Merge branch 'main' into gatekeeper/fix-2369-1

test: cover HNSW index optimization
2026-09-04 12:38:38 +00:00 · 2026-08-21 16:12:55 -07:00 · 2026-08-05 21:57:08 +00:00
23 changed files with 241 additions and 5460 deletions
@@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"

 [[package]]
 name = "fsst"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arrow-array",
 "rand 0.9.5",
@@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"

 [[package]]
 name = "lance"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -4888,8 +4888,8 @@ dependencies = [

 [[package]]
 name = "lance-arrow"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4911,7 +4911,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4925,7 +4925,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -4934,8 +4934,8 @@ dependencies = [

 [[package]]
 name = "lance-bitpacking"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arrayref",
 "crunchy",
@@ -4945,8 +4945,8 @@ dependencies = [

 [[package]]
 name = "lance-core"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4983,8 +4983,8 @@ dependencies = [

 [[package]]
 name = "lance-datafusion"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5013,8 +5013,8 @@ dependencies = [

 [[package]]
 name = "lance-datagen"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5031,8 +5031,8 @@ dependencies = [

 [[package]]
 name = "lance-derive"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -5041,8 +5041,8 @@ dependencies = [

 [[package]]
 name = "lance-encoding"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5075,8 +5075,8 @@ dependencies = [

 [[package]]
 name = "lance-file"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5107,8 +5107,8 @@ dependencies = [

 [[package]]
 name = "lance-index"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -5172,8 +5172,8 @@ dependencies = [

 [[package]]
 name = "lance-index-core"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5195,8 +5195,8 @@ dependencies = [

 [[package]]
 name = "lance-io"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5232,8 +5232,8 @@ dependencies = [

 [[package]]
 name = "lance-linalg"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5247,8 +5247,8 @@ dependencies = [

 [[package]]
 name = "lance-namespace"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arrow",
 "async-trait",
@@ -5260,8 +5260,8 @@ dependencies = [

 [[package]]
 name = "lance-namespace-impls"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arrow",
 "arrow-ipc",
@@ -5314,8 +5314,8 @@ dependencies = [

 [[package]]
 name = "lance-select"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5329,8 +5329,8 @@ dependencies = [

 [[package]]
 name = "lance-table"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5370,8 +5370,8 @@ dependencies = [

 [[package]]
 name = "lance-testing"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5384,8 +5384,8 @@ dependencies = [

 [[package]]
 name = "lance-tokenizer"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.18"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
 dependencies = [
 "frostem",
 "icu_segmenter",
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"

 [workspace.dependencies]
-lance = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
 lancedb = { path = "rust/lancedb", default-features = false }
 ahash = "0.8"
 # Note that this one does not include pyarrow
@@ -28,7 +28,7 @@
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>11.0.0-beta.19</lance-core.version>
+        <lance-core.version>11.0.0-beta.18</lance-core.version>
        <spotless.skip>false</spotless.skip>
        <spotless.version>2.30.0</spotless.version>
        <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
@@ -26,9 +26,7 @@ lance-namespace-impls.workspace = true
 lance-io.workspace = true
 env_logger.workspace = true
 log.workspace = true
-# Maturin enables extension-module mode for Python builds. Keeping it out of
-# Cargo features lets Rust unit tests link against libpython.
-pyo3 = { version = "0.28", features = ["abi3-py310", "chrono"] }
+pyo3 = { version = "0.28", features = ["extension-module", "abi3-py310", "chrono"] }
 chrono.workspace = true
 pyo3-async-runtimes = { version = "0.28", features = [
    "attributes",
@@ -43,7 +41,10 @@ tokio.workspace = true
 libc = "0.2"

 [build-dependencies]
-pyo3-build-config = { version = "0.28", features = ["abi3-py310"] }
+pyo3-build-config = { version = "0.28", features = [
+    "extension-module",
+    "abi3-py310",
+] }

 [features]
 default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/cos", "lancedb/goosefs", "lancedb/metrics-otel"]
@@ -38,25 +38,6 @@ Stable releases are created about every 2 weeks. For the latest features and bug
 pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb
 ```

-### Threading in CPU-limited containers
-
-LanceDB uses separate pools for compute work and storage I/O. On a container with
-two visible CPUs, current releases intentionally use one compute worker by default;
-no manual configuration is needed. If every query logs an I/O core reservation
-warning on a two-CPU container, upgrade from LanceDB 0.21.1 or earlier.
-
-The two commonly tuned environment variables control different resources:
-
- `LANCE_CPU_THREADS` overrides the number of compute workers. One worker is the
-  appropriate setting for a two-CPU container when an explicit override is needed.
- `LANCE_IO_THREADS` controls concurrent storage operations, not reserved CPU
-  cores. Its default can be greater than the number of CPUs because I/O workers
-  spend much of their time waiting for storage.
-
-Keep the defaults unless measurements show that the workload benefits from an
-override. See the [Lance threading model](https://lance.org/guide/performance/#threading-model)
-for the current defaults and tuning guidance.
-
 ## Usage

 ### Basic Example
@@ -103,7 +103,7 @@ python-source = "python"
 module-name = "lancedb._lancedb"

 [build-system]
-requires = ["maturin>=1.9.4"]
+requires = ["maturin>=1.4"]
 build-backend = "maturin"

 [tool.ruff.lint]
@@ -163,15 +163,6 @@ class FTS:
        The number of documents per compressed posting block. Supported values
        are 128 and 256. A value of 256 uses the experimental FTS V3 format
        and may introduce breaking changes.
-    memory_limit : int, optional
-        The total memory limit in MiB for the local FTS build stage. The limit
-        is divided evenly among indexing workers. This build-only setting is
-        not persisted with the index and does not apply to remote tables.
-    num_workers : int, optional
-        The number of workers for a local FTS build. By default Lance uses
-        roughly half of the available CPU cores. The effective value is
-        limited by the available compute capacity. This build-only setting is
-        not persisted with the index and does not apply to remote tables.

    Notes
    -----
@@ -194,8 +185,6 @@ class FTS:
    prefix_only: bool = False
    block_size: int = 128
    custom_stop_words: Optional[List[str]] = None
-    memory_limit: Optional[int] = None
-    num_workers: Optional[int] = None


@dataclass
@@ -245,14 +245,6 @@ def test_create_inverted_index_rejects_invalid_block_size(table):
        table.create_index("text", config=FTS(block_size=129))


-def test_create_inverted_index_respects_build_memory_limit(table):
-    with pytest.raises(ValueError, match="exceeds worker memory limit"):
-        table.create_index(
-            "text",
-            config=FTS(memory_limit=0, num_workers=1),
-        )
-
-
 def test_custom_stop_words_list(table):
    table.create_index(
        "text",
@@ -42,7 +42,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
            "Fm" => Ok(LanceDbIndex::Fm(FmIndexBuilder::default())),
            "FTS" => {
                let params = source.extract::<FtsParams>()?;
-                let mut inner_opts = FtsIndexBuilder::default()
+                let inner_opts = FtsIndexBuilder::default()
                    .base_tokenizer(params.base_tokenizer)
                    .language(&params.language)
                    .map_err(|_| {
@@ -61,12 +61,6 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                    .ngram_max_length(params.ngram_max_length)
                    .ngram_prefix_only(params.prefix_only)
                    .custom_stop_words(params.custom_stop_words);
-                if let Some(memory_limit) = params.memory_limit {
-                    inner_opts = inner_opts.memory_limit_mb(memory_limit);
-                }
-                if let Some(num_workers) = params.num_workers {
-                    inner_opts = inner_opts.num_workers(num_workers);
-                }
                let inner_opts = inner_opts
                    .block_size(params.block_size)
                    .map_err(|err| PyValueError::new_err(err.to_string()))?;
@@ -219,8 +213,6 @@ struct FtsParams {
    ngram_max_length: u32,
    prefix_only: bool,
    block_size: usize,
-    memory_limit: Option<u64>,
-    num_workers: Option<usize>,
 }

 #[derive(FromPyObject)]
@@ -452,51 +444,3 @@ impl IndexConfig {
        }
    }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use pyo3::types::{PyDict, PyDictMethods};
-    use serde_json::json;
-
-    #[test]
-    fn fts_build_controls_are_forwarded() {
-        Python::initialize();
-        Python::attach(|py| {
-            let locals = PyDict::new(py);
-            py.run(
-                c"class FTS:
-    with_position = True
-    base_tokenizer = 'simple'
-    language = 'English'
-    max_token_length = None
-    lower_case = True
-    stem = False
-    remove_stop_words = False
-    custom_stop_words = None
-    ascii_folding = False
-    ngram_min_length = 3
-    ngram_max_length = 3
-    prefix_only = False
-    block_size = 128
-    memory_limit = 2048
-    num_workers = 7
-
-config = FTS()",
-                None,
-                Some(&locals),
-            )
-            .unwrap();
-
-            let config = locals.get_item("config").unwrap().unwrap();
-            let index = extract_index_params(&Some(config)).unwrap();
-            let LanceDbIndex::FTS(params) = index else {
-                panic!("expected FTS index parameters");
-            };
-            let training_json = params.to_training_json().unwrap();
-
-            assert_eq!(training_json.get("memory_limit"), Some(&json!(2048)));
-            assert_eq!(training_json.get("num_workers"), Some(&json!(7)));
-        });
-    }
-}
@@ -41,7 +41,7 @@ use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};

 mod create_table;

-pub(crate) fn merge_storage_options(
+fn merge_storage_options(
    store_params: &mut ObjectStoreParams,
    pairs: impl IntoIterator<Item = (String, String)>,
 ) {
@@ -765,13 +765,60 @@ impl ListingDatabase {
        }
    }

+    /// Extract storage option overrides from the request
+    fn extract_storage_overrides(
+        &self,
+        request: &CreateTableRequest,
+    ) -> Result<(Option<LanceFileVersion>, Option<bool>, Option<bool>)> {
+        let storage_options = request
+            .write_options
+            .lance_write_params
+            .as_ref()
+            .and_then(|p| p.store_params.as_ref())
+            .and_then(|sp| sp.storage_options());
+
+        let storage_version_override = storage_options
+            .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION))
+            .map(|s| s.parse::<LanceFileVersion>())
+            .transpose()?;
+
+        let v2_manifest_override = storage_options
+            .and_then(|opts| opts.get(OPT_NEW_TABLE_V2_MANIFEST_PATHS))
+            .map(|s| s.parse::<bool>())
+            .transpose()
+            .map_err(|_| Error::InvalidInput {
+                message: "enable_v2_manifest_paths must be a boolean".to_string(),
+            })?;
+
+        let stable_row_ids_override = storage_options
+            .and_then(|opts| opts.get(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS))
+            .map(|s| s.parse::<bool>())
+            .transpose()
+            .map_err(|_| Error::InvalidInput {
+                message: "enable_stable_row_ids must be a boolean".to_string(),
+            })?;
+
+        Ok((
+            storage_version_override,
+            v2_manifest_override,
+            stable_row_ids_override,
+        ))
+    }
+
    /// Prepare write parameters for table creation
    fn prepare_write_params(
        &self,
        request: &CreateTableRequest,
-        mut write_params: lance::dataset::WriteParams,
-        overrides: NewTableConfig,
+        storage_version_override: Option<LanceFileVersion>,
+        v2_manifest_override: Option<bool>,
+        stable_row_ids_override: Option<bool>,
    ) -> lance::dataset::WriteParams {
+        let mut write_params = request
+            .write_options
+            .lance_write_params
+            .clone()
+            .unwrap_or_default();
+
        // Only modify the storage options if we actually have something to
        // inherit. There is a difference between storage_options=None and
        // storage_options=Some({}). Using storage_options=None will cause the
@@ -795,21 +842,18 @@ impl ListingDatabase {
            store_params.storage_options_accessor = Some(Arc::new(accessor));
        }

-        write_params.data_storage_version = overrides
-            .data_storage_version
+        write_params.data_storage_version = storage_version_override
            .or(write_params.data_storage_version)
            .or(self.new_table_config.data_storage_version);

-        if let Some(enable_v2_manifest_paths) = overrides
-            .enable_v2_manifest_paths
-            .or(self.new_table_config.enable_v2_manifest_paths)
+        if let Some(enable_v2_manifest_paths) =
+            v2_manifest_override.or(self.new_table_config.enable_v2_manifest_paths)
        {
            write_params.enable_v2_manifest_paths = enable_v2_manifest_paths;
        }

        let data_schema = request.data.arrow_schema();
-        if let Some(enable_stable_row_ids) = overrides
-            .enable_stable_row_ids
+        if let Some(enable_stable_row_ids) = stable_row_ids_override
            .or(self.new_table_config.enable_stable_row_ids)
            .or(has_blob_columns(&data_schema).then_some(true))
        {
@@ -1004,13 +1048,15 @@ impl Database for ListingDatabase {
            .clone()
            .unwrap_or_else(|| self.table_uri(&request.name).unwrap());

-        let mut write_params = request
-            .write_options
-            .lance_write_params
-            .clone()
-            .unwrap_or_default();
-        let overrides = take_request_creation_overrides(&mut write_params)?;
-        let write_params = self.prepare_write_params(&request, write_params, overrides);
+        let (storage_version_override, v2_manifest_override, stable_row_ids_override) =
+            self.extract_storage_overrides(&request)?;
+
+        let write_params = self.prepare_write_params(
+            &request,
+            storage_version_override,
+            v2_manifest_override,
+            stable_row_ids_override,
+        );

        let data_schema = request.data.arrow_schema();

@@ -1242,232 +1288,8 @@ impl Database for ListingDatabase {
    }
 }

-/// Parse the request-level `new_table_*` creation keys into overrides and
-/// strip them from the store options in one step: every create path that
-/// honors them must also keep them out of the object store.
-pub(crate) fn take_request_creation_overrides(
-    params: &mut lance::dataset::WriteParams,
-) -> Result<NewTableConfig> {
-    let storage_options = params
-        .store_params
-        .as_ref()
-        .and_then(|sp| sp.storage_options());
-    let overrides = NewTableConfig {
-        data_storage_version: storage_options
-            .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION))
-            .map(|s| s.parse::<LanceFileVersion>())
-            .transpose()?,
-        enable_v2_manifest_paths: storage_options
-            .and_then(|opts| opts.get(OPT_NEW_TABLE_V2_MANIFEST_PATHS))
-            .map(|s| s.parse::<bool>())
-            .transpose()
-            .map_err(|_| Error::InvalidInput {
-                message: "enable_v2_manifest_paths must be a boolean".to_string(),
-            })?,
-        enable_stable_row_ids: storage_options
-            .and_then(|opts| opts.get(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS))
-            .map(|s| s.parse::<bool>())
-            .transpose()
-            .map_err(|_| Error::InvalidInput {
-                message: "enable_stable_row_ids must be a boolean".to_string(),
-            })?,
-    };
-    if let Some(store_params) = params.store_params.as_mut() {
-        strip_new_table_creation_keys(store_params);
-    }
-    Ok(overrides)
-}
-
-/// Strip the `new_table_*` creation keys from request store options: they are
-/// creation config, not credentials, and left in place they fork a fresh
-/// store connection for the request.
-fn strip_new_table_creation_keys(store_params: &mut ObjectStoreParams) {
-    let mut options = store_params.storage_options().cloned().unwrap_or_default();
-    let mut removed = false;
-    for key in [
-        OPT_NEW_TABLE_STORAGE_VERSION,
-        OPT_NEW_TABLE_V2_MANIFEST_PATHS,
-        OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS,
-    ] {
-        removed |= options.remove(key).is_some();
-    }
-    if !removed {
-        return;
-    }
-    let provider = store_params
-        .storage_options_accessor
-        .as_ref()
-        .and_then(|accessor| accessor.provider().cloned());
-    store_params.storage_options_accessor = match (options.is_empty(), provider) {
-        (true, None) => None,
-        (true, Some(provider)) => Some(Arc::new(StorageOptionsAccessor::with_provider(provider))),
-        (false, Some(provider)) => Some(Arc::new(
-            StorageOptionsAccessor::with_initial_and_provider(options, provider),
-        )),
-        (false, None) => Some(Arc::new(StorageOptionsAccessor::with_static_options(
-            options,
-        ))),
-    };
-}
-
 #[cfg(test)]
 mod tests {
-    #[tokio::test]
-    async fn request_level_creation_keys_do_not_fork_the_store() {
-        use crate::query::ExecutableQuery;
-        use futures::TryStreamExt;
-
-        let db = crate::connect("memory://").execute().await.unwrap();
-        let batch = arrow_array::record_batch!(("x", Int32, [1, 2])).unwrap();
-        let store_params = ObjectStoreParams {
-            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
-                HashMap::from([(
-                    OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
-                    "true".to_string(),
-                )]),
-            ))),
-            ..Default::default()
-        };
-        db.create_table("t", batch)
-            .write_options(crate::table::WriteOptions {
-                lance_write_params: Some(lance::dataset::WriteParams {
-                    store_params: Some(store_params),
-                    ..Default::default()
-                }),
-            })
-            .execute()
-            .await
-            .unwrap();
-
-        let table = db.open_table("t").execute().await.unwrap();
-        let rows: usize = table
-            .query()
-            .execute()
-            .await
-            .unwrap()
-            .try_collect::<Vec<_>>()
-            .await
-            .unwrap()
-            .iter()
-            .map(|b| b.num_rows())
-            .sum();
-        assert_eq!(rows, 2, "the table must live in the session's store");
-    }
-
-    mod strip_new_table_creation_keys {
-        use super::super::*;
-
-        #[derive(Debug)]
-        struct EmptyProvider;
-
-        #[async_trait::async_trait]
-        impl StorageOptionsProvider for EmptyProvider {
-            async fn fetch_storage_options(
-                &self,
-            ) -> lance_core::Result<Option<HashMap<String, String>>> {
-                Ok(Some(HashMap::new()))
-            }
-
-            fn provider_id(&self) -> String {
-                "empty-test-provider".into()
-            }
-        }
-
-        fn params_with_static(options: &[(&str, &str)]) -> ObjectStoreParams {
-            ObjectStoreParams {
-                storage_options_accessor: Some(Arc::new(
-                    StorageOptionsAccessor::with_static_options(
-                        options
-                            .iter()
-                            .map(|(k, v)| (k.to_string(), v.to_string()))
-                            .collect(),
-                    ),
-                )),
-                ..Default::default()
-            }
-        }
-
-        #[test]
-        fn creation_keys_are_removed_and_store_keys_kept() {
-            let mut params = params_with_static(&[
-                ("region", "us-west-2"),
-                (OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, "true"),
-            ]);
-            strip_new_table_creation_keys(&mut params);
-            let options = params.storage_options().cloned().unwrap();
-            assert_eq!(options.get("region").map(String::as_str), Some("us-west-2"));
-            assert!(!options.contains_key(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS));
-
-            // Creation keys alone: no accessor survives to fork a store.
-            let mut params = params_with_static(&[(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, "true")]);
-            strip_new_table_creation_keys(&mut params);
-            assert!(params.storage_options_accessor.is_none());
-        }
-
-        /// A provider must survive every shape of strip: untouched accessors
-        /// keep their identity, emptied ones still fetch, and residual
-        /// statics ride along.
-        #[test]
-        fn provider_accessors_survive_the_strip() {
-            let accessor = Arc::new(StorageOptionsAccessor::with_provider(Arc::new(
-                EmptyProvider,
-            )));
-            let mut params = ObjectStoreParams {
-                storage_options_accessor: Some(accessor.clone()),
-                ..Default::default()
-            };
-            strip_new_table_creation_keys(&mut params);
-            assert!(Arc::ptr_eq(
-                params.storage_options_accessor.as_ref().unwrap(),
-                &accessor
-            ));
-
-            let mut params = ObjectStoreParams {
-                storage_options_accessor: Some(Arc::new(
-                    StorageOptionsAccessor::with_initial_and_provider(
-                        HashMap::from([
-                            ("region".to_string(), "us-west-2".to_string()),
-                            (
-                                OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
-                                "true".to_string(),
-                            ),
-                        ]),
-                        Arc::new(EmptyProvider),
-                    ),
-                )),
-                ..Default::default()
-            };
-            strip_new_table_creation_keys(&mut params);
-            let accessor = params.storage_options_accessor.unwrap();
-            assert!(accessor.has_provider());
-            assert_eq!(
-                accessor
-                    .initial_storage_options()
-                    .and_then(|o| o.get("region").cloned())
-                    .as_deref(),
-                Some("us-west-2")
-            );
-
-            // Emptied entirely: a first-fetch accessor, not one caching {}.
-            let mut params = ObjectStoreParams {
-                storage_options_accessor: Some(Arc::new(
-                    StorageOptionsAccessor::with_initial_and_provider(
-                        HashMap::from([(
-                            OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
-                            "true".to_string(),
-                        )]),
-                        Arc::new(EmptyProvider),
-                    ),
-                )),
-                ..Default::default()
-            };
-            strip_new_table_creation_keys(&mut params);
-            let accessor = params.storage_options_accessor.unwrap();
-            assert!(accessor.has_provider());
-            assert!(accessor.initial_storage_options().is_none());
-        }
-    }
-
    use super::*;
    use crate::Table;
    use crate::arrow::{SendableRecordBatchStream, SimpleRecordBatchStream};
@@ -26,7 +26,10 @@ use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler;
 use crate::blob::{ensure_blob_storage_version, has_blob_columns};
 use crate::connection::NamespaceClientPushdownOperation;
 use crate::database::ReadConsistency;
-use crate::database::listing::{NewTableConfig, take_request_creation_overrides};
+use crate::database::listing::{
+    NewTableConfig, OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, OPT_NEW_TABLE_STORAGE_VERSION,
+    OPT_NEW_TABLE_V2_MANIFEST_PATHS,
+};
 use crate::database::read_freshness::{
    FreshnessBaselines, ReadFreshnessContextProvider, TableFreshness,
 };
@@ -194,28 +197,69 @@ impl LanceNamespaceDatabase {
        TableFreshness::new(self.freshness_baselines.clone(), key)
    }

+    fn extract_storage_overrides(
+        &self,
+        request: &DbCreateTableRequest,
+    ) -> Result<(
+        Option<lance_file::version::LanceFileVersion>,
+        Option<bool>,
+        Option<bool>,
+    )> {
+        let storage_options = request
+            .write_options
+            .lance_write_params
+            .as_ref()
+            .and_then(|p| p.store_params.as_ref())
+            .and_then(|sp| sp.storage_options());
+
+        let storage_version_override = storage_options
+            .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION))
+            .map(|s| s.parse::<lance_file::version::LanceFileVersion>())
+            .transpose()?;
+
+        let v2_manifest_override = storage_options
+            .and_then(|opts| opts.get(OPT_NEW_TABLE_V2_MANIFEST_PATHS))
+            .map(|s| s.parse::<bool>())
+            .transpose()
+            .map_err(|_| Error::InvalidInput {
+                message: "enable_v2_manifest_paths must be a boolean".to_string(),
+            })?;
+
+        let stable_row_ids_override = storage_options
+            .and_then(|opts| opts.get(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS))
+            .map(|s| s.parse::<bool>())
+            .transpose()
+            .map_err(|_| Error::InvalidInput {
+                message: "enable_stable_row_ids must be a boolean".to_string(),
+            })?;
+
+        Ok((
+            storage_version_override,
+            v2_manifest_override,
+            stable_row_ids_override,
+        ))
+    }
+
    fn apply_new_table_config(
        &self,
        params: &mut lance::dataset::WriteParams,
        request: &DbCreateTableRequest,
    ) -> Result<()> {
-        let overrides = take_request_creation_overrides(params)?;
+        let (storage_version_override, v2_manifest_override, stable_row_ids_override) =
+            self.extract_storage_overrides(request)?;

-        params.data_storage_version = overrides
-            .data_storage_version
+        params.data_storage_version = storage_version_override
            .or(params.data_storage_version)
            .or(self.new_table_config.data_storage_version);

-        if let Some(enable_v2_manifest_paths) = overrides
-            .enable_v2_manifest_paths
-            .or(self.new_table_config.enable_v2_manifest_paths)
+        if let Some(enable_v2_manifest_paths) =
+            v2_manifest_override.or(self.new_table_config.enable_v2_manifest_paths)
        {
            params.enable_v2_manifest_paths = enable_v2_manifest_paths;
        }

        let data_schema = request.data.schema();
-        if let Some(enable_stable_row_ids) = overrides
-            .enable_stable_row_ids
+        if let Some(enable_stable_row_ids) = stable_row_ids_override
            .or(self.new_table_config.enable_stable_row_ids)
            .or(has_blob_columns(data_schema.as_ref()).then_some(true))
        {
@@ -600,146 +644,6 @@ mod tests {
        RecordBatch::try_new(schema, vec![Arc::new(id_array), Arc::new(name_array)]).unwrap()
    }

-    /// The shared parse-and-sanitize boundary is wired into this path: the
-    /// request-level creation key must act as an override (the strip itself
-    /// is covered by the listing tests).
-    #[tokio::test]
-    async fn request_level_creation_keys_are_taken_as_overrides() {
-        use crate::database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS;
-
-        let tmp_dir = tempdir().unwrap();
-        let mut properties = HashMap::new();
-        properties.insert(
-            "root".to_string(),
-            tmp_dir.path().to_str().unwrap().to_string(),
-        );
-        let db = connect_namespace("dir", properties)
-            .execute()
-            .await
-            .unwrap();
-
-        let store_params = ObjectStoreParams {
-            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
-                HashMap::from([(
-                    OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
-                    "true".to_string(),
-                )]),
-            ))),
-            ..Default::default()
-        };
-        let table = db
-            .create_table("t", create_test_data())
-            .write_options(crate::table::WriteOptions {
-                lance_write_params: Some(lance::dataset::WriteParams {
-                    store_params: Some(store_params),
-                    ..Default::default()
-                }),
-            })
-            .execute()
-            .await
-            .unwrap();
-        let native = table.as_native().unwrap();
-        assert!(
-            native
-                .dataset
-                .get()
-                .await
-                .unwrap()
-                .manifest
-                .uses_stable_row_ids(),
-            "the creation key must be honored as an override"
-        );
-
-        let table = db.open_table("t").execute().await.unwrap();
-        let rows: usize = table
-            .query()
-            .execute()
-            .await
-            .unwrap()
-            .try_collect::<Vec<_>>()
-            .await
-            .unwrap()
-            .iter()
-            .map(|b| b.num_rows())
-            .sum();
-        assert_eq!(rows, 5);
-    }
-
-    /// Sanitation on this path: apply must strip the creation keys from the
-    /// store options while genuine options and the provider survive.
-    #[tokio::test]
-    async fn apply_new_table_config_sanitizes_request_store_options() {
-        use crate::database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS;
-        use lance_io::object_store::StorageOptionsProvider;
-
-        #[derive(Debug)]
-        struct EmptyProvider;
-
-        #[async_trait::async_trait]
-        impl StorageOptionsProvider for EmptyProvider {
-            async fn fetch_storage_options(
-                &self,
-            ) -> lance_core::Result<Option<HashMap<String, String>>> {
-                Ok(Some(HashMap::new()))
-            }
-
-            fn provider_id(&self) -> String {
-                "empty-test-provider".into()
-            }
-        }
-
-        let tmp_dir = tempdir().unwrap();
-        let mut properties = HashMap::new();
-        properties.insert(
-            "root".to_string(),
-            tmp_dir.path().to_str().unwrap().to_string(),
-        );
-        let db = LanceNamespaceDatabase::connect_with_new_table_config(
-            "dir",
-            properties,
-            HashMap::new(),
-            None,
-            None,
-            HashSet::new(),
-            NewTableConfig::default(),
-        )
-        .await
-        .unwrap();
-
-        let request = DbCreateTableRequest::new("t".to_string(), Box::new(create_test_data()));
-        let mut params = lance::dataset::WriteParams {
-            store_params: Some(ObjectStoreParams {
-                storage_options_accessor: Some(Arc::new(
-                    StorageOptionsAccessor::with_initial_and_provider(
-                        HashMap::from([
-                            ("region".to_string(), "us-west-2".to_string()),
-                            (
-                                OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
-                                "true".to_string(),
-                            ),
-                        ]),
-                        Arc::new(EmptyProvider),
-                    ),
-                )),
-                ..Default::default()
-            }),
-            ..Default::default()
-        };
-        db.apply_new_table_config(&mut params, &request).unwrap();
-
-        assert!(params.enable_stable_row_ids);
-        let store_params = params.store_params.unwrap();
-        let options = store_params.storage_options().cloned().unwrap();
-        assert!(!options.contains_key(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS));
-        assert_eq!(options.get("region").map(String::as_str), Some("us-west-2"));
-        assert!(
-            store_params
-                .storage_options_accessor
-                .unwrap()
-                .has_provider()
-        );
-    }
-
    #[tokio::test]
    async fn test_namespace_connection_simple() {
        // Test that namespace connections work with simple connect_namespace(impl_type, properties)
@@ -77,8 +77,6 @@ pub enum Error {
    ColumnAlreadyExists { name: String },
    #[snafu(display("Column '{name}' is not a computed column"))]
    NotAComputedColumn { name: String },
-    #[snafu(display("Table '{name}' is not a materialized view"))]
-    NotAMaterializedView { name: String },
    #[snafu(display("Invalid expression for column '{column}': {message}"))]
    InvalidExpression { column: String, message: String },

@@ -186,7 +186,6 @@ pub mod index;
 pub mod io;
 pub mod ipc;
 pub mod job;
-pub mod materialized_view;
 #[cfg(feature = "metrics-otel")]
 pub mod metrics_otel;
 #[cfg(feature = "polars")]
@@ -211,9 +210,6 @@ pub use function::FunctionVersion;
 pub use job::Job;
 use lance_index::vector::ApproxMode as LanceApproxMode;
 use lance_linalg::distance::DistanceType as LanceDistanceType;
-pub use materialized_view::{
-    MaterializedView, MaterializedViewDefinition, RefreshMaterializedViewResult, RefreshMode,
-};
 /// Re-export of the [`metrics`](https://docs.rs/metrics) crate facade. Enable
 /// the `metrics` feature to publish LanceDB's internal metrics; install any
 /// `metrics`-compatible recorder to collect them. See also [`metrics_otel`] for
@@ -10410,20 +10410,6 @@ mod tests {
        );
    }

-    #[tokio::test]
-    async fn test_materialized_view_refused_without_a_request() {
-        // Materialized views are local-only. The table-level entry the
-        // bindings use must refuse a remote table before reading its schema,
-        // so the panicking handler is the assertion.
-        let table = Table::new_with_handler("my_table", |request| -> http::Response<String> {
-            panic!("unexpected request: {}", request.url().path())
-        });
-        let err = crate::MaterializedView::from_table(table)
-            .await
-            .unwrap_err();
-        assert!(matches!(err, Error::NotSupported { .. }), "got {err:?}");
-    }
-
    #[tokio::test]
    async fn test_create_branch_empty_name_rejected_client_side() {
        use lance::dataset::refs::Ref;
@@ -1066,11 +1066,6 @@ impl Table {
        self.database.as_ref().unwrap()
    }

-    /// The database this handle was opened through, when it was.
-    pub fn database_opt(&self) -> Option<&Arc<dyn Database>> {
-        self.database.as_ref()
-    }
-
    pub fn embedding_registry(&self) -> &Arc<dyn EmbeddingRegistry> {
        &self.embedding_registry
    }
@@ -12,7 +12,6 @@ use arrow_schema::{DataType, Field};
 use lance::index::DatasetIndexExt;
 use lance::index::vector::VectorIndexParams;
 use lance::index::vector::utils::infer_vector_dim;
-use lance_arrow::json::is_json_field;
 use lance_index::IndexType;
 use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
 use lance_index::vector::bq::RQBuildParams;
@@ -220,14 +219,6 @@ impl NativeTable {
                )))
            }
            Index::Bitmap(_) => {
-                if is_json_field(field) {
-                    return Err(Error::Schema {
-                        message: format!(
-                            "A BITMAP index cannot be created on the whole-document lance.json field `{}`. Create a JSON-path scalar index for structured equality or range predicates, or use FTS for document search",
-                            field.name()
-                        ),
-                    });
-                }
                Self::validate_index_type(field, "Bitmap", supported_bitmap_data_type)?;
                Ok(Box::new(ScalarIndexParams::for_builtin(
                    BuiltinIndexType::Bitmap,
@@ -1474,35 +1465,6 @@ mod tests {
        assert_eq!(stats.distance_type, None);
    }

-    #[tokio::test]
-    async fn test_create_bitmap_index_rejects_lance_json() {
-        let conn = connect("memory://").execute().await.unwrap();
-        let schema = Arc::new(Schema::new(vec![lance_arrow::json::json_field(
-            "metadata", true,
-        )]));
-        let table = conn
-            .create_empty_table("json_bitmap", schema)
-            .execute()
-            .await
-            .unwrap();
-
-        let err = table
-            .create_index(&["metadata"], Index::Bitmap(Default::default()))
-            .execute()
-            .await
-            .expect_err("a whole-document lance.json field must not support a bitmap index");
-        let message = err.to_string();
-        assert!(
-            message.contains("lance.json"),
-            "unexpected error: {message}"
-        );
-        assert!(
-            message.contains("JSON-path scalar index"),
-            "unexpected error: {message}"
-        );
-        assert!(message.contains("FTS"), "unexpected error: {message}");
-    }
-
    #[tokio::test]
    async fn test_create_label_list_index() {
        let conn = connect("memory://").execute().await.unwrap();
@@ -321,8 +321,7 @@ pub(crate) async fn execute_merge_insert(
 mod tests {
    use arrow_array::builder::FixedSizeBinaryBuilder;
    use arrow_array::{
-        FixedSizeListArray, Int32Array, NullArray, RecordBatch, RecordBatchIterator,
-        RecordBatchReader, StringArray, UInt32Array, UInt64Array,
+        Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray, UInt64Array,
    };
    use arrow_schema::{DataType, Field, Schema};
    use std::sync::Arc;
@@ -530,74 +529,6 @@ mod tests {
        assert_eq!(result.num_deleted_rows, 5);
        assert_eq!(table.count_rows(None).await.unwrap(), 5);
    }
-
-    #[tokio::test]
-    async fn test_merge_insert_fixed_size_list_above_u32_child_count() {
-        // Arrow's FixedSizeList take kernel uses u32 child indices. Previously,
-        // delete-by-source materialized the target payload in a full outer join,
-        // causing the final list below to overflow those indices and panic.
-        // A Null child keeps this boundary test small in memory.
-        const LIST_SIZE: i32 = 65_536;
-        const ROW_COUNT: usize = (u32::MAX as usize / LIST_SIZE as usize) + 1;
-        const BATCH_SIZE: usize = 8_192;
-
-        let item = Arc::new(Field::new("item", DataType::Null, true));
-        let schema = Arc::new(Schema::new(vec![
-            Field::new("id", DataType::UInt32, false),
-            Field::new(
-                "vector",
-                DataType::FixedSizeList(item.clone(), LIST_SIZE),
-                false,
-            ),
-        ]));
-        let batch = |start: usize, len: usize| {
-            RecordBatch::try_new(
-                schema.clone(),
-                vec![
-                    Arc::new(UInt32Array::from_iter_values(
-                        start as u32..(start + len) as u32,
-                    )),
-                    Arc::new(FixedSizeListArray::new(
-                        item.clone(),
-                        LIST_SIZE,
-                        Arc::new(NullArray::new(len * LIST_SIZE as usize)),
-                        None,
-                    )),
-                ],
-            )
-            .unwrap()
-        };
-
-        let target_batches = (0..ROW_COUNT)
-            .step_by(BATCH_SIZE)
-            .map(|start| {
-                let len = (ROW_COUNT - start).min(BATCH_SIZE);
-                Ok(batch(start, len))
-            })
-            .collect::<Vec<_>>();
-        let target_data: Box<dyn RecordBatchReader + Send> =
-            Box::new(RecordBatchIterator::new(target_batches, schema.clone()));
-        let conn = connect("memory://").execute().await.unwrap();
-        let table = conn
-            .create_table("fixed_size_list_overflow", target_data)
-            .execute()
-            .await
-            .unwrap();
-
-        let source = batch(ROW_COUNT - 1, 1);
-        let mut merge = table.merge_insert(&["id"]);
-        merge
-            .when_matched_update_all(None)
-            .when_not_matched_by_source_delete(None);
-        let result = merge
-            .execute(Box::new(RecordBatchIterator::new([Ok(source)], schema)))
-            .await
-            .unwrap();
-
-        assert_eq!(result.num_updated_rows, 1);
-        assert_eq!(result.num_deleted_rows, (ROW_COUNT - 1) as u64);
-        assert_eq!(table.count_rows(None).await.unwrap(), 1);
-    }
 }

 #[cfg(test)]
@@ -104,13 +104,6 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
                .into(),
        });
    }
-    if crate::materialized_view::materialized_view_kind(&dataset.schema().metadata)?.is_some() {
-        return Err(Error::NotSupported {
-            message: "an LSM write spec cannot be installed on a materialized view: \
-                      rows in un-compacted tiers are invisible to refresh"
-                .into(),
-        });
-    }
    let mut builder = dataset.initialize_mem_wal();
    let writer_config_defaults = match spec {
        LsmWriteSpec::Bucket {
@@ -224,8 +224,10 @@ mod tests {

    use crate::connect;
    use crate::database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS;
-    use crate::index::vector::IvfRqIndexBuilder;
-    use crate::index::{Index, scalar::BTreeIndexBuilder};
+    use crate::index::{
+        Index, scalar::BTreeIndexBuilder,
+        vector::{IvfRqIndexBuilder, IvfHnswSqIndexBuilder},
+    };
    use crate::query::ExecutableQuery;
    use crate::table::{CompactionOptions, OptimizeAction, OptimizeStats};
    use futures::TryStreamExt;
@@ -650,6 +652,60 @@ mod tests {
        assert_eq!(all_values, expected);
    }

+    #[tokio::test]
+    async fn test_optimize_all_with_ivf_hnsw_sq_index() {
+        let conn = connect("memory://").execute().await.unwrap();
+
+        let dimension = 8;
+        let item_field = Arc::new(Field::new("item", DataType::Float32, true));
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "vector",
+            DataType::FixedSizeList(item_field.clone(), dimension),
+            false,
+        )]));
+
+        let make_batch = |offset: usize| {
+            let values = Float32Array::from_iter_values(
+                (offset * dimension as usize..(offset + 128) * dimension as usize)
+                    .map(|value| value as f32),
+            );
+            let vectors =
+                FixedSizeListArray::try_new(item_field.clone(), dimension, Arc::new(values), None)
+                    .unwrap();
+            RecordBatch::try_new(schema.clone(), vec![Arc::new(vectors)]).unwrap()
+        };
+
+        let table = conn
+            .create_table("test_hnsw_optimize", make_batch(0))
+            .execute()
+            .await
+            .unwrap();
+        for offset in [128, 256, 384] {
+            table.add(make_batch(offset)).execute().await.unwrap();
+        }
+
+        table
+            .create_index(
+                &["vector"],
+                Index::IvfHnswSq(IvfHnswSqIndexBuilder::default()),
+            )
+            .execute()
+            .await
+            .unwrap();
+
+        let stats = table.optimize(OptimizeAction::All).await.unwrap();
+        assert!(stats.compaction.unwrap().fragments_removed > 0);
+
+        let indices = table.list_indices().await.unwrap();
+        assert_eq!(indices.len(), 1);
+        assert_eq!(indices[0].index_type, crate::index::IndexType::IvfHnswSq);
+
+        let index_stats = table.index_stats(&indices[0].name).await.unwrap().unwrap();
+        assert_eq!(index_stats.num_indexed_rows, 512);
+        assert_eq!(index_stats.num_unindexed_rows, 0);
+        assert_eq!(table.count_rows(None).await.unwrap(), 512);
+    }
+
    #[tokio::test]
    async fn test_optimize_default_action() {
        // Verify that default action is All
@@ -193,7 +193,7 @@ fn declared_expression(dataset: &Dataset, column: &str) -> Result<String> {
 ///
 /// Lance's dialect delimits with backticks, so a double-quoted name would
 /// parse as a string literal rather than a column.
-pub(crate) fn quote_identifier(name: &str) -> String {
+fn quote_identifier(name: &str) -> String {
    format!("`{}`", name.replace('`', "``"))
 }

@@ -597,7 +597,7 @@ mod tests {

    /// A fragment spanning several scan batches exercises the streamed fill:
    /// the probe buffers only until the first gained value and the rest flows
-    /// through write_columns a batch at a time.
+    /// through write_column a batch at a time.
    #[tokio::test]
    async fn test_refresh_streams_a_multi_batch_fragment() {
        let values: Vec<i32> = (0..20_000).collect();
Author	SHA1	Message	Date
Will Jones	10dac899e5	Merge branch 'main' into gatekeeper/fix-2369-1	2026-08-21 16:12:55 -07:00
Gatefixer	10c8894fe8	test: cover HNSW index optimization	2026-08-05 21:57:08 +00:00