test: update create-index default contract coverage

`pnpm audit` in `nodejs/` reported a number of vulnerable transitive
dependencies. Most were resolved by `pnpm audit --fix`, which bumped the
affected packages in the lockfile; the `minimumReleaseAgeExclude`
additions in `pnpm-workspace.yaml` are its bookkeeping, exempting the
specific patched versions from the repository's 24-hour hold on newly
published packages. Two findings needed handling by hand, because the
vulnerable package could not simply be moved to a newer release in
place.

`@opentelemetry/sdk-metrics` 1.30.1 pins `@opentelemetry/core` to its
own exact version, and the 1.x line is end-of-life, so
GHSA-8988-4f7v-96qf (unbounded memory allocation in W3C Baggage
propagation) has no fix available on 1.x. This PR moves the dependency
to 2.x, which brings in a patched `@opentelemetry/core`. It is a
dev-only dependency with a single consumer, `__test__/otel.test.ts`, and
the parts of the API that test uses are unchanged between 1.x and 2.x.

`@huggingface/transformers` pins `sharp: ^0.33.5`, and no released
version of transformers has moved past `^0.34.5` — every version in
those ranges inherits the libvips CVEs in GHSA-f88m-g3jw-g9cj, so there
is no upstream release to upgrade to. This PR adds a pnpm `overrides`
entry pinning sharp to the patched `^0.35.4` line instead.

`pnpm audit` now reports no known vulnerabilities.

## Not included

The sharp override only applies to this repository's own dependency
tree, since pnpm overrides are not published to npm. Anyone installing
`@lancedb/lancedb` together with the optional
`@huggingface/transformers` still resolves sharp 0.33.5, and will until
transformers itself moves to sharp 0.35. Practical exposure there is
low: the CVEs require decoding untrusted images, and LanceDB's
transformers embedding function is text-only.

`nodejs/examples/` is a separate install with its own lockfile and is
untouched here. It pins `sharp: "0.33.5"` directly and `pnpm audit`
reports 19 findings against it. Bumping sharp there is more involved
than it looks, because sharp 0.35 requires Node >= 20.9 while the
examples tests run on the Node 18/20 CI matrix, so it is left for
separate work.

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Xuanwo <github@xuanwo.io>

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.39.0-beta.0"
+current_version = "0.39.0-beta.1"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

Cargo.lock

@@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arrow-array",
  "rand 0.9.5",
@@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 
 [[package]]
 name = "lance"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -4888,8 +4888,8 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4911,7 +4911,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4925,7 +4925,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -4934,8 +4934,8 @@ dependencies = [
 
 [[package]]
 name = "lance-bitpacking"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arrayref",
  "crunchy",
@@ -4945,8 +4945,8 @@ dependencies = [
 
 [[package]]
 name = "lance-core"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4983,8 +4983,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datafusion"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5000,6 +5000,7 @@ dependencies = [
  "datafusion-functions",
  "datafusion-physical-expr",
  "futures",
+ "half",
  "jsonb",
  "lance-arrow",
  "lance-core",
@@ -5013,8 +5014,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5031,8 +5032,8 @@ dependencies = [
 
 [[package]]
 name = "lance-derive"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5041,8 +5042,8 @@ dependencies = [
 
 [[package]]
 name = "lance-encoding"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -5075,8 +5076,8 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -5107,8 +5108,8 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -5172,8 +5173,8 @@ dependencies = [
 
 [[package]]
 name = "lance-index-core"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5195,8 +5196,8 @@ dependencies = [
 
 [[package]]
 name = "lance-io"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5236,8 +5237,8 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5251,8 +5252,8 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arrow",
  "async-trait",
@@ -5264,8 +5265,8 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-impls"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -5304,9 +5305,9 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-reqwest-client"
-version = "0.11.0"
+version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a030196da1c994b63a96a4f0bf5b0cfa459fe6dadc9e962320246ca328da22a"
+checksum = "1d06b1fbb5d41f93bc652b61e2872af92e8a6c5f6b4ce8839a8ecfa05365d359"
 dependencies = [
  "reqwest 0.12.28",
  "serde",
@@ -5318,8 +5319,8 @@ dependencies = [
 
 [[package]]
 name = "lance-select"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5333,8 +5334,8 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5374,8 +5375,8 @@ dependencies = [
 
 [[package]]
 name = "lance-testing"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5388,8 +5389,8 @@ dependencies = [
 
 [[package]]
 name = "lance-tokenizer"
-version = "12.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
+version = "12.0.0-beta.11"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
 dependencies = [
  "frostem",
  "icu_segmenter",
@@ -5402,7 +5403,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb"
-version = "0.39.0-beta.0"
+version = "0.39.0-beta.1"
 dependencies = [
  "ahash",
  "anyhow",
@@ -5490,7 +5491,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb-nodejs"
-version = "0.39.0-beta.0"
+version = "0.39.0-beta.1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5515,7 +5516,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb-python"
-version = "0.39.0-beta.0"
+version = "0.39.0-beta.1"
 dependencies = [
  "arrow",
  "async-trait",

Cargo.toml

@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"
 
 [workspace.dependencies]
-lance = { "version" = "=12.0.0-beta.5", default-features = false, "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=12.0.0-beta.5", default-features = false, "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=12.0.0-beta.5", default-features = false, "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
 lancedb = { path = "rust/lancedb", default-features = false }
 ahash = "0.8"
 # Note that this one does not include pyarrow

docs/openapi.yml

@@ -446,6 +446,15 @@ paths:
               properties:
                 column:
                   type: string
+                name:
+                  type: string
+                  description: Optional name for the created index.
+                replace:
+                  type: boolean
+                  default: true
+                  description: |
+                    Whether to replace an existing index with the same resolved
+                    name. Defaults to true.
                 metric_type:
                   type: string
                   nullable: false

docs/src/java/java.md

@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-core</artifactId>
-    <version>0.39.0-beta.0</version>
+    <version>0.39.0-beta.1</version>
 </dependency>
 ```
 

docs/src/js/classes/Table.md

@@ -676,9 +676,17 @@ List all the versions of the table
 abstract mergeInsert(on): MergeInsertBuilder
 ```
 
+Create a [MergeInsertBuilder](MergeInsertBuilder.md), which combines new data with the
+existing table in a single transaction — inserting, updating and deleting
+rows depending on how they match.
+
 #### Parameters
 
 * **on**: `string` \| `string`[]
+    The column, or columns, to match source rows against target
+    rows on. Typically a key or id column. Several columns match on the
+    composite key: a source row updates a target row only when it agrees on
+    every one of them.
 
 #### Returns
 

docs/src/js/interfaces/IndexOptions.md

@@ -50,7 +50,7 @@ If this is false, and another index already exists on the same columns
 and the same name, then an error will be returned.  This is true even if
 that index is out of date.
 
-The default is true
+The default is false
 
 ***
 

java/lancedb-core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
       <groupId>com.lancedb</groupId>
       <artifactId>lancedb-parent</artifactId>
-      <version>0.39.0-beta.0</version>
+      <version>0.39.0-beta.1</version>
       <relativePath>../pom.xml</relativePath>
     </parent>
 

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.39.0-beta.0</version>
+    <version>0.39.0-beta.1</version>
     <packaging>pom</packaging>
     <name>${project.artifactId}</name>
     <description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>12.0.0-beta.5</lance-core.version>
+        <lance-core.version>12.0.0-beta.11</lance-core.version>
         <spotless.skip>false</spotless.skip>
         <spotless.version>2.30.0</spotless.version>
         <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.39.0-beta.0"
+version = "0.39.0-beta.1"
 publish = false
 license.workspace = true
 description.workspace = true

nodejs/__test__/table.test.ts

@@ -737,11 +737,12 @@ it("should query documents with LangChain PDF metadata", async () => {
 
 describe("merge insert", () => {
   let tmpDir: tmp.DirResult;
+  let conn: Connection;
   let table: Table;
 
   beforeEach(async () => {
     tmpDir = tmp.dirSync({ unsafeCleanup: true });
-    const conn = await connect(tmpDir.name);
+    conn = await connect(tmpDir.name);
 
     table = await conn.createTable("some_table", [
       { a: 1, b: "a" },
@@ -779,6 +780,38 @@ describe("merge insert", () => {
 
     expect(result.map((row) => ({ ...row }))).toEqual(expected);
   });
+  test("upsert on a composite key", async () => {
+    const composite = await conn.createTable("composite", [
+      { shard: "a", id: 1, val: "x" },
+      { shard: "a", id: 2, val: "y" },
+      { shard: "b", id: 1, val: "z" },
+    ]);
+
+    // ("a", 1) matches an existing row and updates it. ("b", 2) agrees with an
+    // existing row on each key column separately but on neither pair, so it is
+    // an insert.
+    const mergeInsertRes = await composite
+      .mergeInsert(["shard", "id"])
+      .whenMatchedUpdateAll()
+      .whenNotMatchedInsertAll()
+      .execute([
+        { shard: "a", id: 1, val: "X" },
+        { shard: "b", id: 2, val: "W" },
+      ]);
+    expect(mergeInsertRes.numUpdatedRows).toBe(1);
+    expect(mergeInsertRes.numInsertedRows).toBe(1);
+
+    const result = (await composite.toArrow())
+      .toArray()
+      .sort((a, b) => a.shard.localeCompare(b.shard) || a.id - b.id);
+
+    expect(result.map((row) => ({ ...row }))).toEqual([
+      { shard: "a", id: 1, val: "X" },
+      { shard: "a", id: 2, val: "y" },
+      { shard: "b", id: 1, val: "z" },
+      { shard: "b", id: 2, val: "W" },
+    ]);
+  });
   test("conditional update", async () => {
     const newData = [
       { a: 2, b: "x" },
@@ -1493,11 +1526,7 @@ describe("When creating an index", () => {
 
   it("should allow me to replace (or not) an existing index", async () => {
     await tbl.createIndex("id");
-    // Default is replace=true
-    await tbl.createIndex("id");
-    await expect(tbl.createIndex("id", { replace: false })).rejects.toThrow(
-      "already exists",
-    );
+    await expect(tbl.createIndex("id")).rejects.toThrow("already exists");
     await tbl.createIndex("id", { replace: true });
   });
 

nodejs/lancedb/indices.ts

@@ -841,7 +841,7 @@ export interface IndexOptions {
    * and the same name, then an error will be returned.  This is true even if
    * that index is out of date.
    *
-   * The default is true
+   * The default is false
    */
   replace?: boolean;
 

nodejs/lancedb/table.ts

@@ -919,6 +919,16 @@ export abstract class Table {
   /** Return the table as an arrow table */
   abstract toArrow(): Promise<ArrowTable>;
 
+  /**
+   * Create a {@link MergeInsertBuilder}, which combines new data with the
+   * existing table in a single transaction — inserting, updating and deleting
+   * rows depending on how they match.
+   *
+   * @param on - The column, or columns, to match source rows against target
+   * rows on. Typically a key or id column. Several columns match on the
+   * composite key: a source row updates a target row only when it agrees on
+   * every one of them.
+   */
   abstract mergeInsert(on: string | string[]): MergeInsertBuilder;
 
   /** List all the stats of a specified index

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.39.0-beta.0",
+	"version": "0.39.0-beta.1",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.39.0-beta.0",
+	"version": "0.39.0-beta.1",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.39.0-beta.0",
+	"version": "0.39.0-beta.1",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.39.0-beta.0",
+	"version": "0.39.0-beta.1",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.39.0-beta.0",
+	"version": "0.39.0-beta.1",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.39.0-beta.0",
+  "version": "0.39.0-beta.1",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.39.0-beta.0",
+	"version": "0.39.0-beta.1",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.39.0-beta.0",
+  "version": "0.39.0-beta.1",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -44,7 +44,7 @@
     "@biomejs/biome": "^1.7.3",
     "@jest/globals": "^29.7.0",
     "@napi-rs/cli": "3.7.0",
-    "@opentelemetry/sdk-metrics": "^1.30.0",
+    "@opentelemetry/sdk-metrics": "^2.10.0",
     "@types/axios": "^0.14.0",
     "@types/jest": "^29.1.2",
     "@types/node": "22.7.4",
@@ -56,7 +56,7 @@
     "eslint": "^8.57.0",
     "jest": "^29.7.0",
     "shx": "^0.3.4",
-    "tmp": "^0.2.3",
+    "tmp": "^0.2.7",
     "ts-jest": "^29.1.2",
     "typedoc": "0.26.4",
     "typedoc-plugin-markdown": "4.2.1",

nodejs/pnpm-workspace.yaml

@@ -16,3 +16,41 @@ allowBuilds:
   onnxruntime-node: true
   protobufjs: true
   sharp: true
+
+minimumReleaseAgeExclude:
+  - protobufjs@7.5.8
+  - tmp@0.2.6
+  - form-data@4.0.6
+  - tar@7.5.16
+  - markdown-it@14.1.2
+  - linkify-it@5.0.1
+  - js-yaml@3.15.0
+  - js-yaml@4.1.2
+  - protobufjs@7.6.1
+  - protobufjs@7.6.3
+  - '@babel/core@7.29.1'
+  - axios@1.18.0
+  - brace-expansion@2.1.2
+  - brace-expansion@1.1.16
+  - js-yaml@4.3.0
+  - tar@7.5.18
+  - tar@7.5.19
+  - tar@7.5.17
+  - protobufjs@7.6.5
+  - linkify-it@5.0.2
+  - sharp@0.35.0
+  - brace-expansion@1.1.17
+  - brace-expansion@2.1.3
+  - brace-expansion@2.1.4
+  - brace-expansion@1.1.18
+  - js-yaml@3.15.1
+  - js-yaml@4.3.1
+  - tar@7.5.21
+  - '@opentelemetry/core@2.8.0'
+
+# @huggingface/transformers pins sharp ^0.33.5 and no released version has moved
+# past ^0.34.5, all of which inherit the libvips CVEs in GHSA-f88m-g3jw-g9cj.
+# Force the patched line. sharp is only reached by transformers' image pipeline,
+# which LanceDB's text embedding function never uses.
+overrides:
+  sharp: ^0.35.4

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.39.0-beta.0"
+version = "0.39.0-beta.1"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"

python/python/lancedb/remote/table.py

@@ -402,6 +402,7 @@ class RemoteTable(Table):
         /,
         *,
         config: IndexConfigType,
+        replace: bool = ...,
         wait_timeout: Optional[timedelta] = ...,
         name: Optional[str] = ...,
         train: bool = ...,
@@ -416,7 +417,7 @@ class RemoteTable(Table):
         index_cache_size: Optional[int] = ...,
         num_partitions: Optional[int] = ...,
         num_sub_vectors: Optional[int] = ...,
-        replace: Optional[bool] = ...,
+        replace: bool = ...,
         accelerator: Optional[str] = ...,
         index_type: Literal[
             "VECTOR", "IVF_FLAT", "IVF_SQ", "IVF_PQ", "IVF_HNSW_SQ", "IVF_HNSW_PQ"
@@ -435,7 +436,7 @@ class RemoteTable(Table):
         index_cache_size: Optional[int] = None,
         num_partitions: Optional[int] = None,
         num_sub_vectors: Optional[int] = None,
-        replace: Optional[bool] = None,
+        replace: bool = False,
         accelerator: Optional[str] = None,
         index_type="vector",
         wait_timeout: Optional[timedelta] = None,
@@ -479,7 +480,6 @@ class RemoteTable(Table):
             vector_column_name,
             accelerator,
             index_cache_size,
-            replace,
         )
 
         if is_legacy:
@@ -503,12 +503,6 @@ class RemoteTable(Table):
                     "If you have 100M+ vectors to index,"
                     "please contact us at contact@lancedb.com"
                 )
-            if replace is not None:
-                logging.warning(
-                    "replace is not supported on LanceDB cloud."
-                    "Existing indexes will always be replaced."
-                )
-
             idx_type = index_type.upper()
             if idx_type == "VECTOR" or idx_type == "IVF_PQ":
                 config = IvfPq(
@@ -548,6 +542,7 @@ class RemoteTable(Table):
         LOOP.run(
             self._table.create_index(
                 column,
+                replace=replace,
                 config=config,
                 wait_timeout=wait_timeout,
                 name=name,
@@ -560,7 +555,7 @@ class RemoteTable(Table):
         column: str,
         *,
         config: IndexConfigType,
-        replace: Optional[bool] = None,
+        replace: bool = False,
         wait_timeout: Optional[timedelta] = None,
         name: Optional[str] = None,
         train: bool = True,
@@ -592,7 +587,6 @@ class RemoteTable(Table):
         vector_column_name: str,
         accelerator: Optional[str],
         index_cache_size: Optional[int],
-        replace: Optional[bool],
     ) -> bool:
         """Detect if this is a legacy create_index call."""
         if config is not None:
@@ -604,7 +598,6 @@ class RemoteTable(Table):
                 num_sub_vectors,
                 accelerator,
                 index_cache_size,
-                replace,
             )
         ):
             return True

python/python/lancedb/table.py

@@ -1132,7 +1132,7 @@ class Table(ABC):
         num_partitions: Optional[int] = None,
         num_sub_vectors: Optional[int] = None,
         vector_column_name: str = VECTOR_COLUMN_NAME,
-        replace: bool = True,
+        replace: bool = False,
         accelerator: Optional[str] = None,
         index_cache_size: Optional[int] = None,
         *,
@@ -1166,7 +1166,7 @@ class Table(ABC):
             The index configuration object. If provided, uses the new unified API.
             Can be one of: IvfFlat, IvfPq, IvfSq, IvfRq, HnswPq, HnswSq,
             BTree, Bitmap, LabelList, Fm, FTS.
-        replace : bool, default True
+        replace : bool, default False
             Whether to replace an existing index on this column.
         wait_timeout : timedelta, optional
             Timeout to wait for async indexing to complete.
@@ -1198,7 +1198,7 @@ class Table(ABC):
         column: str,
         *,
         config: IndexConfigType,
-        replace: Optional[bool] = None,
+        replace: bool = False,
         wait_timeout: Optional[timedelta] = None,
         name: Optional[str] = None,
         train: bool = True,
@@ -1260,7 +1260,7 @@ class Table(ABC):
         self,
         column: str,
         *,
-        replace: bool = True,
+        replace: bool = False,
         index_type: ScalarIndexType = "BTREE",
         wait_timeout: Optional[timedelta] = None,
         name: Optional[str] = None,
@@ -1272,7 +1272,7 @@ class Table(ABC):
         column : str
             The column to be indexed.  Must be a boolean, integer, float,
             or string column.
-        replace : bool, default True
+        replace : bool, default False
             Replace the existing index if it exists.
         index_type: Literal["BTREE", "BITMAP", "LABEL_LIST"], default "BTREE"
             The type of index to create.
@@ -1547,7 +1547,9 @@ class Table(ABC):
         on: Union[str, Iterable[str]]
             A column (or columns) to join on.  This is how records from the
             source table and target table are matched.  Typically this is some
-            kind of key or id column.
+            kind of key or id column.  Passing several columns matches on the
+            composite key: a source row updates a target row only when it
+            agrees on every one of them.
 
         Examples
         --------
@@ -2986,7 +2988,7 @@ class LanceTable(Table):
         num_partitions: Optional[int] = None,
         num_sub_vectors: Optional[int] = None,
         vector_column_name: str = VECTOR_COLUMN_NAME,
-        replace: bool = True,
+        replace: bool = False,
         accelerator: Optional[str] = None,
         index_cache_size: Optional[int] = None,
         num_bits: int = 8,
@@ -3028,7 +3030,7 @@ class LanceTable(Table):
             The index configuration object. If provided, uses the new unified API.
             Can be one of: IvfFlat, IvfPq, IvfSq, IvfRq, HnswPq, HnswSq,
             BTree, Bitmap, LabelList, Fm, FTS.
-        replace : bool, default True
+        replace : bool, default False
             Whether to replace an existing index on this column.
         wait_timeout : timedelta, optional
             Timeout to wait for async indexing to complete.
@@ -3167,7 +3169,7 @@ class LanceTable(Table):
         column: str,
         *,
         config: IndexConfigType,
-        replace: Optional[bool] = None,
+        replace: bool = False,
         wait_timeout: Optional[timedelta] = None,
         name: Optional[str] = None,
         train: bool = True,
@@ -3422,7 +3424,7 @@ class LanceTable(Table):
         self,
         column: str,
         *,
-        replace: bool = True,
+        replace: bool = False,
         index_type: ScalarIndexType = "BTREE",
         name: Optional[str] = None,
     ):
@@ -5311,7 +5313,7 @@ class AsyncTable:
         self,
         column: str,
         *,
-        replace: Optional[bool] = None,
+        replace: bool = False,
         config: Optional[
             Union[
                 IvfFlat,
@@ -5342,14 +5344,14 @@ class AsyncTable:
         ----------
         column: str
             The column to index.
-        replace: bool, default True
+        replace: bool, default False
             Whether to replace the existing index
 
             If this is false, and another index already exists on the same columns
             and the same name, then an error will be returned.  This is true even if
             that index is out of date.
 
-            The default is True
+            The default is False
         config: default None
             For advanced configuration you can specify the type of index you would
             like to create.   You can also specify index-specific parameters when
@@ -5407,7 +5409,7 @@ class AsyncTable:
         self,
         column: str,
         *,
-        replace: Optional[bool] = None,
+        replace: bool = False,
         config: Optional[
             Union[
                 IvfFlat,
@@ -5701,7 +5703,9 @@ class AsyncTable:
         on: Union[str, Iterable[str]]
             A column (or columns) to join on.  This is how records from the
             source table and target table are matched.  Typically this is some
-            kind of key or id column.
+            kind of key or id column.  Passing several columns matches on the
+            composite key: a source row updates a target row only when it
+            agrees on every one of them.
 
         Examples
         --------

python/python/tests/test_index.py

@@ -97,6 +97,9 @@ async def test_create_index_async_returns_done_job(some_table: AsyncTable):
 async def test_create_scalar_index(some_table: AsyncTable):
     # Can create
     await some_table.create_index("id")
+    # Can't recreate by default
+    with pytest.raises(RuntimeError, match="already exists"):
+        await some_table.create_index("id")
     # Can recreate if replace=True
     await some_table.create_index("id", replace=True)
     indices = await some_table.list_indices()
@@ -110,7 +113,7 @@ async def test_create_scalar_index(some_table: AsyncTable):
     with pytest.raises(RuntimeError, match="already exists"):
         await some_table.create_index("id", replace=False)
     # can also specify index type
-    await some_table.create_index("id", config=BTree())
+    await some_table.create_index("id", config=BTree(), replace=True)
 
     await some_table.drop_index("id_idx")
     indices = await some_table.list_indices()
@@ -351,13 +354,18 @@ async def test_full_text_search_index(some_table: AsyncTable):
 async def test_create_vector_index(some_table: AsyncTable):
     # Can create
     await some_table.create_index("vector")
+    # Can't recreate by default
+    with pytest.raises(RuntimeError, match="already exists"):
+        await some_table.create_index("vector")
     # Can recreate if replace=True
     await some_table.create_index("vector", replace=True)
     # Can't recreate if replace=False
     with pytest.raises(RuntimeError, match="already exists"):
         await some_table.create_index("vector", replace=False)
     # Can also specify index type
-    await some_table.create_index("vector", config=IvfPq(num_partitions=100))
+    await some_table.create_index(
+        "vector", config=IvfPq(num_partitions=100), replace=True
+    )
     indices = await some_table.list_indices()
     assert len(indices) == 1
     assert indices[0].index_type == "IvfPq"

python/python/tests/test_remote_db.py

@@ -820,11 +820,13 @@ def test_table_create_indices():
         scalar_req = received_requests[0]
         assert "name" in scalar_req
         assert scalar_req["name"] == "custom_scalar_idx"
+        assert scalar_req["replace"] is False
 
         # Check FTS index request has custom name
         fts_req = received_requests[1]
         assert "name" in fts_req
         assert fts_req["name"] == "custom_fts_idx"
+        assert fts_req["replace"] is False
         assert fts_req["block_size"] == 256
         assert fts_req["custom_stop_words"] == ["cloud"]
 
@@ -832,6 +834,7 @@ def test_table_create_indices():
         vector_req = received_requests[2]
         assert "name" in vector_req
         assert vector_req["name"] == "custom_vector_idx"
+        assert vector_req["replace"] is False
 
         table.wait_for_index(["custom_scalar_idx"], timedelta(seconds=2))
         table.wait_for_index(
@@ -1104,6 +1107,9 @@ def test_remote_create_index_new_api():
             table.create_index("text", config=FTS(block_size=256))
             # IvfRq via new API
             table.create_index("vector", config=IvfRq(distance_type="l2"))
+            table.create_index(
+                "vector", config=IvfPq(distance_type="l2"), replace=False
+            )
 
         # Legacy index_type="IVF_RQ" routes to IvfRq config under the hood.
         with pytest.warns(DeprecationWarning, match="create_index"):
@@ -1113,15 +1119,17 @@ def test_remote_create_index_new_api():
                 num_partitions=8,
             )
 
-        assert len(received_requests) == 5
+        assert len(received_requests) == 6
         assert [req["column"] for req in received_requests] == [
             "vector",
             "category",
             "text",
             "vector",
             "vector",
+            "vector",
         ]
         assert received_requests[2]["block_size"] == 256
+        assert received_requests[4]["replace"] is False
 
 
 def test_table_wait_for_index_timeout():

python/python/tests/test_table.py

@@ -1600,7 +1600,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
     )
     mock_create_index.assert_called_with(
         "my_vector",
-        replace=True,
+        replace=False,
         config=expected_config,
         wait_timeout=None,
         name=None,
@@ -1620,7 +1620,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
     )
     mock_create_index.assert_called_with(
         "my_vector",
-        replace=True,
+        replace=False,
         config=expected_config,
         wait_timeout=None,
         name=None,
@@ -1646,7 +1646,7 @@ def test_create_index_name_and_train_parameters(
     expected_config = IvfPq()  # Default config
     mock_create_index.assert_called_with(
         "vector",
-        replace=True,
+        replace=False,
         config=expected_config,
         wait_timeout=None,
         name="my_custom_index",
@@ -1657,7 +1657,7 @@ def test_create_index_name_and_train_parameters(
     table.create_index(vector_column_name="vector", train=False)
     mock_create_index.assert_called_with(
         "vector",
-        replace=True,
+        replace=False,
         config=expected_config,
         wait_timeout=None,
         name=None,
@@ -1668,7 +1668,7 @@ def test_create_index_name_and_train_parameters(
     table.create_index(vector_column_name="vector", name="my_index_name", train=True)
     mock_create_index.assert_called_with(
         "vector",
-        replace=True,
+        replace=False,
         config=expected_config,
         wait_timeout=None,
         name="my_index_name",
@@ -1705,7 +1705,7 @@ def test_create_index_new_api(mock_create_index, mem_db: DBConnection):
         table.create_index("vector", config=IvfPq(distance_type="l2"))
     mock_create_index.assert_called_with(
         "vector",
-        replace=True,
+        replace=False,
         config=IvfPq(distance_type="l2"),
         wait_timeout=None,
         name=None,
@@ -1716,7 +1716,7 @@ def test_create_index_new_api(mock_create_index, mem_db: DBConnection):
     table.create_index("category", config=BTree())
     mock_create_index.assert_called_with(
         "category",
-        replace=True,
+        replace=False,
         config=BTree(),
         wait_timeout=None,
         name=None,
@@ -1727,7 +1727,7 @@ def test_create_index_new_api(mock_create_index, mem_db: DBConnection):
     table.create_index("text", config=FTS(with_position=True))
     mock_create_index.assert_called_with(
         "text",
-        replace=True,
+        replace=False,
         config=FTS(with_position=True),
         wait_timeout=None,
         name=None,
@@ -2682,6 +2682,43 @@ def test_merge_insert(mem_db: DBConnection):
         )
 
 
+def test_merge_insert_composite_key(mem_db: DBConnection):
+    table = mem_db.create_table(
+        "my_table",
+        data=pa.table(
+            {
+                "shard": ["a", "a", "b"],
+                "id": [1, 2, 1],
+                "val": ["x", "y", "z"],
+            }
+        ),
+    )
+
+    # ("a", 1) matches an existing row and updates it. ("b", 2) agrees with an
+    # existing row on each key column separately but on neither pair, so it is
+    # an insert.
+    new_data = pa.table({"shard": ["a", "b"], "id": [1, 2], "val": ["X", "W"]})
+    res = (
+        table.merge_insert(["shard", "id"])
+        .when_matched_update_all()
+        .when_not_matched_insert_all()
+        .execute(new_data)
+    )
+    assert res.num_updated_rows == 1
+    assert res.num_inserted_rows == 1
+
+    expected = pa.table(
+        {
+            "shard": ["a", "a", "b", "b"],
+            "id": [1, 2, 1, 2],
+            "val": ["X", "y", "z", "W"],
+        }
+    )
+    assert table.to_arrow().sort_by([("shard", "ascending"), ("id", "ascending")]) == (
+        expected
+    )
+
+
 def test_merge_insert_nullable_pandas_into_pydantic_schema(mem_db: DBConnection):
     # Regression test for https://github.com/lancedb/lancedb/issues/2366
     pd = pytest.importorskip("pandas")

rust/lancedb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.39.0-beta.0"
+version = "0.39.0-beta.1"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true

rust/lancedb/src/index.rs

@@ -200,14 +200,14 @@ impl IndexBuilder {
             parent,
             index,
             columns,
-            replace: true,
+            replace: false,
             train: true,
             wait_timeout: None,
             name: None,
         }
     }
 
-    /// Whether to replace the existing index, the default is `true`.
+    /// Whether to replace the existing index, the default is `false`.
     ///
     /// If this is false, and another index already exists on the same columns
     /// and the same name, then an error will be returned.  This is true even if

rust/lancedb/src/job.rs

@@ -40,6 +40,7 @@ impl TerminalResult {
         }
     }
 
+    #[cfg(feature = "remote")]
     pub(crate) fn remote(value: Option<Value>, request_id: String) -> Self {
         Self {
             value,
@@ -47,30 +48,46 @@ impl TerminalResult {
         }
     }
 
+    #[cfg(feature = "remote")]
     pub(crate) fn value(&self) -> Option<&Value> {
         self.value.as_ref()
     }
 
     fn decode<T: DeserializeOwned>(self) -> Result<T> {
-        let value = self.value.ok_or_else(|| match &self.request_id {
-            Some(request_id) => Error::Http {
-                source: "successful typed job response did not contain a result".into(),
-                request_id: request_id.clone(),
-                status_code: None,
-            },
-            None => Error::Runtime {
+        let value = self.value.ok_or_else(|| {
+            #[cfg(feature = "remote")]
+            if let Some(request_id) = &self.request_id {
+                return Error::Http {
+                    source: "successful typed job response did not contain a result".into(),
+                    request_id: request_id.clone(),
+                    status_code: None,
+                };
+            }
+            Error::Runtime {
                 message: "successful typed job did not contain a result".to_string(),
-            },
+            }
         })?;
-        serde_json::from_value(value).map_err(|error| match self.request_id {
-            Some(request_id) => Error::Http {
-                source: format!("failed to parse typed job result: {error}").into(),
-                request_id,
-                status_code: None,
-            },
-            None => Error::Runtime {
-                message: format!("failed to parse typed job result: {error}"),
-            },
+        serde_json::from_value(value).map_err(|error| {
+            #[cfg(feature = "remote")]
+            {
+                match self.request_id {
+                    Some(request_id) => Error::Http {
+                        source: format!("failed to parse typed job result: {error}").into(),
+                        request_id,
+                        status_code: None,
+                    },
+                    None => Error::Runtime {
+                        message: format!("failed to parse typed job result: {error}"),
+                    },
+                }
+            }
+            #[cfg(not(feature = "remote"))]
+            {
+                let _ = self.request_id;
+                Error::Runtime {
+                    message: format!("failed to parse typed job result: {error}"),
+                }
+            }
         })
     }
 }
@@ -117,6 +134,7 @@ impl Job<()> {
         }
     }
 
+    #[cfg(feature = "remote")]
     pub(crate) fn new(handle: Box<dyn JobHandle>) -> Self {
         Self {
             inner: JobInner::Handle {

rust/lancedb/src/remote/table.rs

@@ -72,7 +72,7 @@ use lance_datafusion::exec::{OneShotExec, execute_plan};
 use reqwest::{RequestBuilder, Response};
 use serde::{Deserialize, Serialize};
 use serde_json::Number;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::io::Cursor;
 use std::pin::Pin;
 use std::sync::{Arc, Mutex};
@@ -524,7 +524,8 @@ impl<S: HttpSend> RemoteTable<S> {
             _ => resolve_arrow_field_path(&schema, &column)?,
         };
         let mut body = serde_json::json!({
-            "column": canonical_column
+            "column": canonical_column,
+            "replace": index.replace,
         });
 
         // Add name parameter if provided (for backwards compatibility, only include if Some)
@@ -3647,7 +3648,12 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
 
 #[derive(Serialize, Clone, Debug)]
 pub struct MergeInsertRequest {
-    on: String,
+    // Sent as one repeated `on` query parameter per column, which is how the
+    // namespace spec encodes an array-valued `on`. serde_urlencoded (which
+    // reqwest's `query()` uses) cannot serialize a sequence nested in a struct,
+    // so this field is emitted separately by [`Self::on_query_params`].
+    #[serde(skip_serializing)]
+    on: Vec<String>,
     when_matched_update_all: bool,
     when_matched_update_all_filt: Option<String>,
     when_not_matched_insert_all: bool,
@@ -3663,6 +3669,17 @@ pub struct MergeInsertRequest {
     use_lsm: Option<bool>,
 }
 
+impl MergeInsertRequest {
+    /// The `on` columns as repeated query parameters: `?on=a&on=b`.
+    ///
+    /// A single column serializes to `?on=a`, exactly what clients sent before
+    /// `on` became a list, so a server that predates composite keys sees no
+    /// change from a single-column caller.
+    pub(crate) fn on_query_params(&self) -> Vec<(&str, &str)> {
+        self.on.iter().map(|col| ("on", col.as_str())).collect()
+    }
+}
+
 fn is_true(b: &bool) -> bool {
     *b
 }
@@ -3675,12 +3692,15 @@ impl TryFrom<MergeInsertBuilder> for MergeInsertRequest {
             return Err(Error::InvalidInput {
                 message: "MergeInsertBuilder missing required 'on' field".into(),
             });
-        } else if value.on.len() > 1 {
-            return Err(Error::NotSupported {
-                message: "MergeInsertBuilder only supports a single 'on' column".into(),
+        }
+        // The server rejects a repeated column with a 400; catching it here
+        // names the offending column and costs no round trip.
+        let mut seen = HashSet::with_capacity(value.on.len());
+        if let Some(dup) = value.on.iter().find(|col| !seen.insert(*col)) {
+            return Err(Error::InvalidInput {
+                message: format!("MergeInsertBuilder 'on' column '{dup}' is repeated"),
             });
         }
-        let on = value.on[0].clone();
 
         let when_matched_update_all_filt = match value.when_matched_update_all_filt {
             Some(MergeFilter::Sql(sql)) => Some(sql),
@@ -3704,7 +3724,7 @@ impl TryFrom<MergeInsertBuilder> for MergeInsertRequest {
             };
 
         Ok(Self {
-            on,
+            on: value.on,
             when_matched_update_all: value.when_matched_update_all,
             when_matched_update_all_filt,
             when_not_matched_insert_all: value.when_not_matched_insert_all,
@@ -4548,6 +4568,76 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn test_merge_insert_composite_key() {
+        let batch = RecordBatch::try_new(
+            Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
+            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
+        )
+        .unwrap();
+        let data: Box<dyn RecordBatchReader + Send> = Box::new(RecordBatchIterator::new(
+            [Ok(batch.clone())],
+            batch.schema(),
+        ));
+
+        let table = Table::new_with_handler("my_table", move |request| {
+            assert_eq!(request.url().path(), "/v1/table/my_table/merge_insert/");
+
+            // One repeated `on` per column, in the order the caller gave them.
+            let on = request
+                .url()
+                .query_pairs()
+                .filter(|(key, _)| key == "on")
+                .map(|(_, value)| value.into_owned())
+                .collect::<Vec<_>>();
+            assert_eq!(on, vec!["shard_key".to_string(), "id".to_string()]);
+
+            let params = request.url().query_pairs().collect::<HashMap<_, _>>();
+            assert_eq!(params["when_matched_update_all"], "true");
+            assert_eq!(params["when_not_matched_insert_all"], "true");
+
+            http::Response::builder()
+                .status(200)
+                .body(r#"{"version": 43, "num_deleted_rows": 0, "num_inserted_rows": 3, "num_updated_rows": 0}"#)
+                .unwrap()
+        });
+
+        let mut merge = table.merge_insert(&["shard_key", "id"]);
+        merge.when_matched_update_all(None);
+        merge.when_not_matched_insert_all();
+        let result = table.base_table().merge_insert(merge, data).await.unwrap();
+
+        assert_eq!(result.num_inserted_rows, 3);
+    }
+
+    #[tokio::test]
+    async fn test_merge_insert_rejects_repeated_on_column() {
+        let batch = RecordBatch::try_new(
+            Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
+            vec![Arc::new(Int32Array::from(vec![1]))],
+        )
+        .unwrap();
+        let data: Box<dyn RecordBatchReader + Send> = Box::new(RecordBatchIterator::new(
+            [Ok(batch.clone())],
+            batch.schema(),
+        ));
+
+        let table = Table::new_with_handler::<&str>("my_table", |request| {
+            panic!("Unexpected request: {}", request.url());
+        });
+
+        let merge = table.merge_insert(&["id", "id"]);
+        let err = table
+            .base_table()
+            .merge_insert(merge, data)
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(&err, Error::InvalidInput { message } if message.contains("'id' is repeated")),
+            "unexpected error: {err}"
+        );
+    }
+
     #[tokio::test]
     async fn test_merge_insert_retries_on_409() {
         let batch = RecordBatch::try_new(
@@ -6216,6 +6306,7 @@ mod tests {
                             let mut expected_body = expected_body.clone();
                             expected_body["column"] = "a".into();
                             expected_body[INDEX_TYPE_KEY] = index_type.into();
+                            expected_body["replace"] = false.into();
 
                             assert_eq!(body, expected_body);
 
@@ -6233,6 +6324,73 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn test_create_index_forwards_default_replace_false_on_existing_route() {
+        let table = Table::new_with_handler("my_table", move |request| {
+            assert_eq!(request.method(), "POST");
+            match request.url().path() {
+                "/v1/table/my_table/describe/" => {
+                    let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+                    http::Response::builder()
+                        .status(200)
+                        .body(describe_response(&schema))
+                        .unwrap()
+                }
+                "/v1/table/my_table/create_index/" => {
+                    let body = request.body().unwrap().as_bytes().unwrap();
+                    let body: serde_json::Value = serde_json::from_slice(body).unwrap();
+                    assert_eq!(body["replace"], json!(false));
+
+                    http::Response::builder()
+                        .status(200)
+                        .body("{}".to_string())
+                        .unwrap()
+                }
+                path => panic!("Unexpected path: {}", path),
+            }
+        });
+
+        table
+            .create_index(&["a"], Index::BTree(Default::default()))
+            .execute()
+            .await
+            .unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_create_index_forwards_explicit_replace_true_on_existing_route() {
+        let table = Table::new_with_handler("my_table", move |request| {
+            assert_eq!(request.method(), "POST");
+            match request.url().path() {
+                "/v1/table/my_table/describe/" => {
+                    let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+                    http::Response::builder()
+                        .status(200)
+                        .body(describe_response(&schema))
+                        .unwrap()
+                }
+                "/v1/table/my_table/create_index/" => {
+                    let body = request.body().unwrap().as_bytes().unwrap();
+                    let body: serde_json::Value = serde_json::from_slice(body).unwrap();
+                    assert_eq!(body["replace"], json!(true));
+
+                    http::Response::builder()
+                        .status(200)
+                        .body("{}".to_string())
+                        .unwrap()
+                }
+                path => panic!("Unexpected path: {}", path),
+            }
+        });
+
+        table
+            .create_index(&["a"], Index::BTree(Default::default()))
+            .replace(true)
+            .execute()
+            .await
+            .unwrap();
+    }
+
     #[tokio::test]
     async fn test_create_index_returns_job() {
         let describe_calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
@@ -6458,38 +6616,46 @@ mod tests {
             json!({
                 "column": "rowId",
                 "index_type": "BTREE",
+                "replace": false,
             }),
             json!({
                 "column": "`row-id`",
                 "index_type": "BTREE",
+                "replace": false,
             }),
             json!({
                 "column": "userId",
                 "index_type": "BTREE",
+                "replace": false,
             }),
             json!({
                 "column": "MetaData.userId",
                 "index_type": "BTREE",
+                "replace": false,
             }),
             json!({
                 "column": "metadata.user_id",
                 "index_type": "BTREE",
+                "replace": false,
             }),
             json!({
                 "column": "image.embedding",
                 "index_type": "IVF_PQ",
                 "metric_type": "l2",
+                "replace": false,
             }),
             {
                 let mut body = serde_json::to_value(InvertedIndexParams::default()).unwrap();
                 body["column"] = "payload.text".into();
                 body["index_type"] = "FTS".into();
+                body["replace"] = false.into();
                 body
             },
             {
                 let mut body = serde_json::to_value(InvertedIndexParams::default()).unwrap();
                 body["column"] = "docs.content".into();
                 body["index_type"] = "FTS".into();
+                body["replace"] = false.into();
                 body
             },
             {
@@ -6497,15 +6663,18 @@ mod tests {
                 body["column"] = "docs.content".into();
                 body["index_type"] = "FTS".into();
                 body["document_granularity"] = "list_element".into();
+                body["replace"] = false.into();
                 body
             },
             json!({
                 "column": "`meta-data`.`user-id`",
                 "index_type": "BTREE",
+                "replace": false,
             }),
             json!({
                 "column": "literal.`a.b`",
                 "index_type": "BTREE",
+                "replace": false,
             }),
         ]);
         let request_idx = Arc::new(AtomicUsize::new(0));

rust/lancedb/src/remote/table/insert.rs

@@ -734,6 +734,7 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
                 WriteOp::MergeInsert { query, timeout } => {
                     let mut request = client
                         .post(&format!("/v1/table/{}/merge_insert/", identifier))
+                        .query(&query.on_query_params())
                         .query(query)
                         .header(CONTENT_TYPE, ARROW_STREAM_CONTENT_TYPE);
                     if let Some(timeout) = timeout {
@@ -1489,7 +1490,7 @@ mod tests {
         });
 
         let query = MergeInsertRequest {
-            on: "id".to_string(),
+            on: vec!["id".to_string()],
             when_matched_update_all: false,
             when_matched_update_all_filt: None,
             when_not_matched_insert_all: false,

rust/lancedb/src/table.rs

@@ -1506,7 +1506,9 @@ impl Table {
     ///
     /// * `on` One or more columns to join on.  This is how records from the
     ///   source table and target table are matched.  Typically this is some
-    ///   kind of key or id column.
+    ///   kind of key or id column.  Several columns match on the composite
+    ///   key: a source row updates a target row only when it agrees on every
+    ///   one of them.
     ///
     /// # Examples
     ///

rust/lancedb/src/table/create_index.rs

@@ -133,7 +133,7 @@ impl NativeTable {
                     ),
                 });
             }
-            (resolved.canonical_path, resolved.field)
+            (resolved.canonical_path, resolved.terminal_field)
         } else {
             Self::resolve_index_field(dataset.schema(), &opts.columns[0])?
         };
@@ -439,7 +439,8 @@ mod tests {
     use arrow_array::record_batch;
     use arrow_array::{
         Array, ArrayRef, BinaryArray, BooleanArray, FixedSizeListArray, Float32Array, Int32Array,
-        LargeBinaryArray, LargeStringArray, RecordBatch, StringArray, StructArray,
+        LargeBinaryArray, LargeStringArray, ListArray, RecordBatch, StringArray, StructArray,
+        UInt32Array,
     };
     use arrow_data::ArrayDataBuilder;
     use arrow_schema::{DataType, Field, Schema};
@@ -458,6 +459,7 @@ mod tests {
     use crate::query::{ExecutableQuery, QueryBase};
     use crate::table::optimize::{CompactionOptions, OptimizeAction};
     use lance_index::scalar::FullTextSearchQuery;
+    use lance_index::scalar::inverted::query::{FtsQuery, MatchQuery};
 
     fn create_fixed_size_list<T: Array>(
         values: T,
@@ -599,6 +601,80 @@ mod tests {
         assert!(invalid_granularity.is_err());
     }
 
+    #[tokio::test]
+    async fn test_nested_list_fts_uses_deepest_document_coordinates() {
+        let conn = connect("memory://").execute().await.unwrap();
+        let mut docs = ListBuilder::new(ListBuilder::new(StringBuilder::new()));
+
+        docs.values().values().append_value("alpha");
+        docs.values().values().append_value("beta");
+        docs.values().append(true);
+        docs.values().values().append_value("gamma");
+        docs.values().values().append_value("alpha delta");
+        docs.values().append(true);
+        docs.append(true);
+
+        docs.values().append(true);
+        docs.values().values().append_value("alpha");
+        docs.values().append(true);
+        docs.append(true);
+
+        let batch = RecordBatch::try_from_iter(vec![
+            ("id", Arc::new(Int32Array::from(vec![0, 1])) as ArrayRef),
+            ("docs", Arc::new(docs.finish()) as ArrayRef),
+        ])
+        .unwrap();
+        let table = conn.create_table("nested", batch).execute().await.unwrap();
+
+        let job = table
+            .create_index(
+                &["docs"],
+                Index::FTS(
+                    FtsIndexBuilder::default()
+                        .document_granularity(DocumentGranularity::ListElement),
+                ),
+            )
+            .execute_async()
+            .await
+            .unwrap();
+        job.wait().await.unwrap();
+
+        let query = FullTextSearchQuery::new_query(FtsQuery::Match(
+            MatchQuery::new("alpha".to_string())
+                .with_column(Some("docs".to_string()))
+                .with_document_granularity(DocumentGranularity::ListElement),
+        ));
+        let batches = table
+            .query()
+            .full_text_search(query)
+            .limit(10)
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+
+        let mut hits = Vec::new();
+        for batch in batches {
+            let ids = batch["id"].as_any().downcast_ref::<Int32Array>().unwrap();
+            let coordinates = batch["_doc_index"]
+                .as_any()
+                .downcast_ref::<ListArray>()
+                .unwrap();
+            for row in 0..batch.num_rows() {
+                let coordinate = coordinates.value(row);
+                let coordinate = coordinate.as_any().downcast_ref::<UInt32Array>().unwrap();
+                hits.push((ids.value(row), coordinate.values().to_vec()));
+            }
+        }
+        hits.sort_unstable();
+        assert_eq!(
+            hits,
+            vec![(0, vec![0, 0]), (0, vec![1, 1]), (1, vec![1, 0])]
+        );
+    }
+
     /// Concurrent waiters, and a wait issued after the job settled, all
     /// succeed once the build does.
     #[tokio::test]
@@ -649,12 +725,11 @@ mod tests {
             .await
             .unwrap();
 
-        // Rebuilding the same index without replace fails once the build
+        // Rebuilding the same index without explicit replace fails once the build
         // starts, so the failure reaches the job rather than execute_async.
         let job = Arc::new(
             table
                 .create_index(&["id"], Index::BTree(BTreeIndexBuilder::default()))
-                .replace(false)
                 .execute_async()
                 .await
                 .unwrap(),
@@ -693,7 +768,6 @@ mod tests {
 
         let job = table
             .create_index(&["id"], Index::BTree(BTreeIndexBuilder::default()))
-            .replace(false)
             .execute_async()
             .await
             .unwrap();
@@ -1030,6 +1104,7 @@ mod tests {
         // Can also specify btree
         table
             .create_index(&["i"], Index::BTree(BTreeIndexBuilder::default()))
+            .replace(true)
             .execute()
             .await
             .unwrap();

rust/lancedb/src/utils/mod.rs

@@ -227,7 +227,7 @@ pub(crate) fn resolve_arrow_field_path(schema: &Schema, column: &str) -> Result<
 
 pub(crate) struct ResolvedFtsField {
     pub canonical_path: String,
-    pub field: Field,
+    pub terminal_field: Field,
     pub list_depth: usize,
 }
 
@@ -309,7 +309,7 @@ pub(crate) fn resolve_lance_fts_field_path(
     );
     Ok(ResolvedFtsField {
         canonical_path,
-        field: Field::from(field),
+        terminal_field: Field::from(terminal),
         list_depth,
     })
 }
@@ -375,7 +375,7 @@ pub(crate) fn resolve_arrow_fts_field_path(
             message: format!("Invalid schema: {}", e),
         })?;
     let resolved = resolve_lance_fts_field_path(&lance_schema, column)?;
-    Ok((resolved.canonical_path, resolved.field))
+    Ok((resolved.canonical_path, resolved.terminal_field))
 }
 
 pub fn supported_btree_data_type(dtype: &DataType) -> bool {
@@ -647,8 +647,9 @@ mod tests {
             Field::new("docs", text_list(), true),
         ]);
 
-        let (path, _) = resolve_arrow_fts_field_path(&schema, "docs.content").unwrap();
+        let (path, field) = resolve_arrow_fts_field_path(&schema, "docs.content").unwrap();
         assert_eq!(path, "docs.content");
+        assert_eq!(field.data_type(), &DataType::Utf8);
 
         let lance_schema = lance_core::datatypes::Schema::try_from(&schema).unwrap();
         let field_id = lance_schema