feat: add support for filter during merge insert when matched (#948)

Closes #940

node/src/index.ts

@@ -525,8 +525,19 @@ export interface MergeInsertArgs {
    * If there are multiple matches then the behavior is undefined.
    * Currently this causes multiple copies of the row to be created
    * but that behavior is subject to change.
+   *
+   * Optionally, a filter can be specified.  This should be an SQL
+   * filter where fields with the prefix "target." refer to fields
+   * in the target table (old data) and fields with the prefix
+   * "source." refer to fields in the source table (new data).  For
+   * example, the filter "target.lastUpdated < source.lastUpdated" will
+   * only update matched rows when the incoming `lastUpdated` value is
+   * newer.
+   *
+   * Rows that do not match the filter will not be updated.  Rows that
+   * do not match the filter do become "not matched" rows.
    */
-  whenMatchedUpdateAll?: boolean
+  whenMatchedUpdateAll?: string | boolean
   /**
    * If true then rows that exist only in the source table (new data)
    * will be inserted into the target table.
@@ -885,7 +896,14 @@ export class LocalTable<T = number[]> implements Table<T> {
   }
 
   async mergeInsert (on: string, data: Array<Record<string, unknown>> | ArrowTable, args: MergeInsertArgs): Promise<void> {
-    const whenMatchedUpdateAll = args.whenMatchedUpdateAll ?? false
+    let whenMatchedUpdateAll = false
+    let whenMatchedUpdateAllFilt = null
+    if (args.whenMatchedUpdateAll !== undefined && args.whenMatchedUpdateAll !== null) {
+      whenMatchedUpdateAll = true
+      if (args.whenMatchedUpdateAll !== true) {
+        whenMatchedUpdateAllFilt = args.whenMatchedUpdateAll
+      }
+    }
     const whenNotMatchedInsertAll = args.whenNotMatchedInsertAll ?? false
     let whenNotMatchedBySourceDelete = false
     let whenNotMatchedBySourceDeleteFilt = null
@@ -909,6 +927,7 @@ export class LocalTable<T = number[]> implements Table<T> {
       this._tbl,
       on,
       whenMatchedUpdateAll,
+      whenMatchedUpdateAllFilt,
       whenNotMatchedInsertAll,
       whenNotMatchedBySourceDelete,
       whenNotMatchedBySourceDeleteFilt,

node/src/remote/index.ts

@@ -286,8 +286,11 @@ export class RemoteTable<T = number[]> implements Table<T> {
     const queryParams: any = {
       on
     }
-    if (args.whenMatchedUpdateAll ?? false) {
+    if (args.whenMatchedUpdateAll !== false && args.whenMatchedUpdateAll !== null && args.whenMatchedUpdateAll !== undefined) {
       queryParams.when_matched_update_all = 'true'
+      if (typeof args.whenMatchedUpdateAll === 'string') {
+        queryParams.when_matched_update_all_filt = args.whenMatchedUpdateAll
+      }
     } else {
       queryParams.when_matched_update_all = 'false'
     }

node/src/test/test.ts

@@ -540,26 +540,36 @@ describe('LanceDB client', function () {
       const data = [{ id: 1, age: 1 }, { id: 2, age: 1 }]
       const table = await con.createTable('my_table', data)
 
+      // insert if not exists
       let newData = [{ id: 2, age: 2 }, { id: 3, age: 2 }]
       await table.mergeInsert('id', newData, {
         whenNotMatchedInsertAll: true
       })
       assert.equal(await table.countRows(), 3)
-      assert.equal((await table.filter('age = 2').execute()).length, 1)
+      assert.equal(await table.countRows('age = 2'), 1)
 
-      newData = [{ id: 3, age: 3 }, { id: 4, age: 3 }]
+      // conditional update
+      newData = [{ id: 2, age: 3 }, { id: 3, age: 3 }]
+      await table.mergeInsert('id', newData, {
+        whenMatchedUpdateAll: 'target.age = 1'
+      })
+      assert.equal(await table.countRows(), 3)
+      assert.equal(await table.countRows('age = 1'), 1)
+      assert.equal(await table.countRows('age = 3'), 1)
+
+      newData = [{ id: 3, age: 4 }, { id: 4, age: 4 }]
       await table.mergeInsert('id', newData, {
         whenNotMatchedInsertAll: true,
         whenMatchedUpdateAll: true
       })
       assert.equal(await table.countRows(), 4)
-      assert.equal((await table.filter('age = 3').execute()).length, 2)
+      assert.equal((await table.filter('age = 4').execute()).length, 2)
 
-      newData = [{ id: 5, age: 4 }]
+      newData = [{ id: 5, age: 5 }]
       await table.mergeInsert('id', newData, {
         whenNotMatchedInsertAll: true,
         whenMatchedUpdateAll: true,
-        whenNotMatchedBySourceDelete: 'age < 3'
+        whenNotMatchedBySourceDelete: 'age < 4'
       })
       assert.equal(await table.countRows(), 3)