docs: add Cursor Cloud dev environment setup notes to AGENTS.md

Co-authored-by: Xuanwo <github@xuanwo.io>
2026-08-28 08:58:41 +00:00 · 2026-08-07 08:02:38 +00:00
80 changed files with 529 additions and 4762 deletions
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.38.0-beta.0"
+current_version = "0.37.1-beta.0"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
@@ -4,14 +4,14 @@ on:
  workflow_call:
    inputs:
      tag:
-        description: "Tag name from Lance (e.g. `v7.2.0-beta.1`). If omitted, the newest release is resolved automatically — stable releases are preferred over pre-releases — and the run is skipped if it is not newer than the version currently pinned in Cargo.toml."
+        description: "Tag name from Lance. If omitted, the skill will use the latest Lance release that needs an update."
        required: false
        default: ""
        type: string
  workflow_dispatch:
    inputs:
      tag:
-        description: "Tag name from Lance (e.g. `v7.2.0-beta.1`). Leave empty to resolve the newest release automatically — stable releases are preferred over pre-releases — and skip the run if it is not newer than the version currently pinned in Cargo.toml."
+        description: "Tag name from Lance. Leave empty to use the latest Lance release that needs an update."
        required: false
        default: ""
        type: string
@@ -1,243 +0,0 @@
-name: Check doc links
-
-# Checking external links is inherently noisy: third-party sites rate-limit
-# automated clients, reject non-browser user agents, and go down temporarily.
-# Blocking pull requests on that trades a lot of false failures for very little
-# signal, so this runs on a schedule and reports findings in a single tracking
-# issue instead of failing anyone's build.
-on:
-  schedule:
-    - cron: "0 7 * * *"
-  workflow_dispatch:
-
-# The report lives in one repository-global issue, so runs must not overlap: a
-# lookup racing a create produces duplicate issues, and a healthy run closing
-# the issue while a failing run only rewrites its body would leave a broken
-# report closed. The group is deliberately ref-independent so that a manual
-# dispatch serializes against the scheduled run.
-concurrency:
-  group: docs-link-check
-  cancel-in-progress: false
-
-permissions: {}
-
-env:
-  REPORT_TITLE: "Docs link checker report"
-
-jobs:
-  scan:
-    name: Scan links
-    runs-on: ubuntu-24.04
-    # lychee-action is pinned by SHA, but its wrapper downloads the lychee
-    # release tarball at run time without verifying a digest, and hands the
-    # resulting binary a GitHub token. Release assets remain replaceable, so
-    # that binary is confined to a job whose token can only read public
-    # content; everything that writes runs in the report job below.
-    permissions:
-      contents: read
-    outputs:
-      checker_outcome: ${{ steps.lychee.outcome }}
-      exit_code: ${{ steps.lychee.outputs.exit_code }}
-      status: ${{ steps.validate.outputs.status }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-        with:
-          # workflow_dispatch can run from any ref, but the report is
-          # repository-global. Always measure the default branch so a manual
-          # run from a topic branch cannot close a report that main warrants,
-          # or overwrite it with branch-only findings.
-          ref: ${{ github.event.repository.default_branch }}
-          persist-credentials: false
-
-      - name: Check links
-        id: lychee
-        continue-on-error: true
-        uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0
-        with:
-          # Restricted to http(s) on purpose. Much of docs/src is generated
-          # API reference (the js/ tree comes from `npm run docs` in nodejs)
-          # and the hand-written pages use mkdocstrings cross-references and
-          # nav-relative paths that only resolve in the site mkdocs builds,
-          # not in this checkout, so relative links would be reported as
-          # broken on every run.
-          args: >-
-            --scheme https
-            --scheme http
-            --no-progress
-            --max-retries 3
-            --timeout 20
-            'docs/src/**/*.md'
-          format: json
-          output: ./lychee/out.json
-          jobSummary: false
-          # The report issue, not a red workflow run, is the signal for link
-          # findings and checker failures alike.
-          fail: false
-
-      - name: Validate report
-        id: validate
-        # lychee does not reserve exit code 2 for broken links: its CLI
-        # parser also exits 2 on an invalid option, before any link was
-        # checked or any report written. Only a parseable report whose
-        # counts agree with a completed exit code (0 or 2) counts as a link
-        # verdict. Everything else becomes a checker-error report instead of
-        # failing the workflow. Exit 2 covers timeouts as well as errors, and a
-        # timed-out host is exactly the transient unavailability this report
-        # exists to surface, so both count as findings. Requiring total > 0
-        # also catches a glob that silently stopped matching any file.
-        if: always()
-        env:
-          CHECKER_OUTCOME: ${{ steps.lychee.outcome }}
-          EXIT_CODE: ${{ steps.lychee.outputs.exit_code }}
-        run: |
-          status=checker-error
-          if [[ "$CHECKER_OUTCOME" == success ]] &&
-             [[ "$EXIT_CODE" == 0 || "$EXIT_CODE" == 2 ]] &&
-             jq -e --argjson code "$EXIT_CODE" '
-               (.total > 0) and
-               (if $code == 0
-                then .errors == 0 and .timeouts == 0
-                  and (.error_map | length == 0) and (.timeout_map | length == 0)
-                else (.errors + .timeouts) > 0
-                  and ((.error_map | length) + (.timeout_map | length)) > 0
-                end)
-             ' ./lychee/out.json
-          then
-            if [[ "$EXIT_CODE" == 0 ]]; then
-              status=healthy
-            else
-              status=findings
-            fi
-          fi
-          echo "status=$status" >> "$GITHUB_OUTPUT"
-          echo "Validated link check as $status"
-
-      - name: Upload report
-        if: steps.validate.outputs.status == 'findings'
-        uses: actions/upload-artifact@v7
-        with:
-          name: link-report
-          path: ./lychee/out.json
-          retention-days: 7
-
-  report:
-    name: Update report issue
-    needs: scan
-    runs-on: ubuntu-24.04
-    # Deliberately no checkout: this job needs the report artifact and the
-    # issues API, not the repository contents.
-    permissions:
-      issues: write
-    env:
-      CHECKER_OUTCOME: ${{ needs.scan.outputs.checker_outcome }}
-      EXIT_CODE: ${{ needs.scan.outputs.exit_code }}
-      STATUS: ${{ needs.scan.outputs.status }}
-      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    steps:
-      - name: Find existing report issue
-        id: report
-        # Matched on title alone, and through search rather than a listing:
-        # the issue action applies labels in a separate call after creating the
-        # issue, so a label filter misses a half-created report, and this
-        # repository has far more open issues than one listing page holds.
-        # Closed issues are included because a healthy run closes the report:
-        # an open-only lookup would forget that identity and the next failing
-        # run would open a duplicate. The oldest match stays the canonical
-        # report and is reopened below when a problem recurs.
-        run: |
-          match=$(gh issue list --repo "$GITHUB_REPOSITORY" --state all \
-            --search "in:title \"$REPORT_TITLE\" author:app/github-actions" \
-            --limit 50 --json number,title,state \
-            --jq "[.[] | select(.title == \"$REPORT_TITLE\")] | sort_by(.number) | first // empty")
-          echo "number=$(jq -r '.number // empty' <<<"$match")" >> "$GITHUB_OUTPUT"
-          echo "state=$(jq -r '.state // empty' <<<"$match")" >> "$GITHUB_OUTPUT"
-
-      - name: Download report
-        if: env.STATUS == 'findings'
-        uses: actions/download-artifact@v8
-        with:
-          name: link-report
-          path: ./lychee
-
-      - name: Compose report
-        if: env.STATUS == 'findings'
-        run: |
-          run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
-          {
-            echo "Broken documentation links found by [\`$GITHUB_WORKFLOW\`]($run_url)."
-            echo
-            echo "This issue is rewritten by every scheduled run and closed automatically once all links resolve."
-            echo
-            echo "Entries can be false positives: some sites rate-limit or block automated clients while working fine in a browser. Confirm before editing the docs, and add persistent offenders to \`--exclude\` in \`.github/workflows/docs-link-check.yml\`."
-            echo
-            # Timeouts are reported alongside errors: entries land in
-            # timeout_map with a status text instead of an HTTP code.
-            jq -r '
-              "\(.errors) of \(.total) links failed, \(.timeouts) timed out.",
-              "",
-              ([(.error_map | to_entries[]), (.timeout_map | to_entries[])]
-               | group_by(.key)[] |
-                "### Errors in \(.[0].key)",
-                "",
-                (map(.value[])[] | "* [\(.status.code // .status.text // "ERR")] <\(.url)> — \(.status.details // .status.text // "unknown error")"),
-                "")
-            ' ./lychee/out.json
-          } > ./lychee/issue.md
-
-      - name: Compose checker error report
-        if: env.STATUS == 'checker-error'
-        run: |
-          mkdir -p ./lychee
-          run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
-          {
-            echo "The documentation link check did not complete in [the latest run]($run_url)."
-            echo
-            echo "This issue is rewritten by every scheduled run and closed automatically once a trustworthy run finds that all links resolve."
-            echo
-            echo "The checker did not produce a trustworthy link verdict. Treat the previous result, if any, as stale until a later run completes."
-            echo
-            echo "* Action outcome: \`$CHECKER_OUTCOME\`"
-            echo "* Exit code: \`${EXIT_CODE:-not reported}\`"
-            echo "* Verdict validation: \`failed\`"
-          } > ./lychee/issue.md
-
-      - name: Reopen report issue
-        # A healthy run closes the report, and the issue action below only
-        # rewrites the body of whatever number it is given. Without an
-        # explicit reopen, a later finding or checker error would rewrite a
-        # closed issue. A CLOSED state implies the lookup found a canonical
-        # issue, so no separate emptiness check.
-        if: >-
-          env.STATUS != 'healthy' &&
-          steps.report.outputs.state == 'CLOSED'
-        env:
-          ISSUE_NUMBER: ${{ steps.report.outputs.number }}
-        run: |
-          run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
-          gh issue reopen "$ISSUE_NUMBER" --repo "$GITHUB_REPOSITORY" \
-            --comment "The documentation link checker reported a problem again in [the latest run]($run_url)."
-
-      - name: Report link-check problem
-        if: env.STATUS != 'healthy'
-        uses: peter-evans/create-issue-from-file@fca9117c27cdc29c6c4db3b86c48e4115a786710 # v6.0.0
-        with:
-          # Empty on the first failing run, which creates the issue; afterwards
-          # the same issue is updated in place.
-          issue-number: ${{ steps.report.outputs.number }}
-          title: ${{ env.REPORT_TITLE }}
-          content-filepath: ./lychee/issue.md
-          labels: documentation
-
-      - name: Close report issue once links are healthy
-        # An OPEN state implies the lookup found a canonical issue; a report
-        # that is already closed needs nothing.
-        if: >-
-          env.STATUS == 'healthy' &&
-          steps.report.outputs.state == 'OPEN'
-        env:
-          ISSUE_NUMBER: ${{ steps.report.outputs.number }}
-        run: |
-          run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
-          gh issue close "$ISSUE_NUMBER" --repo "$GITHUB_REPOSITORY" \
-            --comment "All documentation links resolved in [the latest run]($run_url)."
@@ -69,16 +69,6 @@ jobs:
        uses: actions/setup-python@v6
        with:
          python-version: "3.10"
-      - name: Add swap for Arm fat LTO
-        if: matrix.config.platform == 'aarch64'
-        shell: bash
-        run: |
-          swap_file="$RUNNER_TEMP/lancedb-swap"
-          sudo fallocate --length 16G "$swap_file"
-          sudo chmod 600 "$swap_file"
-          sudo mkswap "$swap_file"
-          sudo swapon "$swap_file"
-          free -h
      - uses: ./.github/workflows/build_linux_wheel
        with:
          python-minor-version: 10
@@ -152,3 +152,54 @@ Please consider the following when reviewing code contributions.
 ### Documentation
 * New features must include updates to the rust documentation comments. Link to
  relevant structs and methods to increase the value of documentation.
+
+## Cursor Cloud specific instructions
+
+The VM snapshot already has the Rust `1.97.0` toolchain (auto-selected by
+`rust-toolchain.toml`), `protoc`, `uv` (on `PATH` via `~/.bashrc`), the Rust
+debug build artifacts, the Python editable extension, and `nodejs/node_modules`.
+The startup update script only refreshes dependencies (`uv sync` for Python and
+`pnpm install` for Node); it deliberately does NOT rebuild the native
+extensions. After changing Rust or PyO3/napi binding code you must rebuild the
+affected binding yourself (see per-binding rebuild commands below).
+
+Non-obvious caveats discovered during setup:
+
+* The documented Python bootstrap `uv run --extra tests --extra dev maturin
+  develop --extras tests,dev` does not work as-is here: `maturin` is not
+  installed as a CLI in the uv environment, and `maturin develop --extras`
+  runs its own dependency resolution that cannot find the prerelease
+  `pylance==9.0.0rc1` (it lacks the extra package index that `uv` uses via
+  `uv.lock`). Because `uv run --extra tests --extra dev` already installs those
+  extras, the working command is:
+  `cd python && uv run --extra tests --extra dev --with maturin maturin develop`
+  (note: `--with maturin`, and no `--extras`). This is the Python binding
+  rebuild command.
+* Rust core, the Python extension (maturin), and the Node addon (napi) all
+  compile into the SHARED `/workspace/target`. Cargo feature unification differs
+  between `maturin develop` and `pnpm build`, so alternating between building
+  the Python and Node bindings forces a full recompile of shared crates
+  (`lancedb`, `datafusion`, `lance-*`) — roughly 6-7 min each way on this
+  4-core VM. Build one binding at a time to avoid the churn.
+* The `_lancedb` release build (triggered when `uv run`/`uv sync` installs the
+  `lancedb` project itself) uses `lto = "fat"` + `opt-level = 3`, needs ~11 GB
+  RAM, and takes ~20 min cold on this VM. To avoid it, the update script uses
+  `uv sync --no-install-project --inexact` (the `--inexact` flag is required so
+  the sync does not uninstall the editable extension). Prefer the debug
+  `maturin develop` (~6 min cold, seconds when warm) for iteration.
+* `cargo check` only produces metadata, so the first `cargo run --example ...`
+  or `cargo test` after a check triggers a large codegen/link compile.
+* Node binding rebuild: `cd nodejs && pnpm build` (napi debug build + `tsc`).
+  The native addon lands at `nodejs/dist/lancedb.linux-x64-gnu.node`.
+
+Verified working (local backend, no cloud credentials needed):
+
+* Rust: `cargo check/clippy --features remote --tests --examples`,
+  `cargo test --features remote -p lancedb --lib`, `cargo run --features remote
+  --example simple`.
+* Python: `cd python && uv run --extra tests pytest python/tests/test_table.py`,
+  `uv run --directory python --extra dev ruff check python`.
+* Node: `cd nodejs && pnpm lint`, `pnpm test __test__/connection.test.ts`.
+
+Java (`java/`) is optional; its integration tests need LanceDB Cloud
+credentials (`LANCEDB_DB`, `LANCEDB_API_KEY`) and were not set up here.
@@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"

 [[package]]
 name = "fsst"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arrow-array",
 "rand 0.9.5",
@@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"

 [[package]]
 name = "lance"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -4832,6 +4832,7 @@ dependencies = [
 "async-recursion",
 "async-trait",
 "async_cell",
+ "aws-credential-types",
 "aws-sdk-dynamodb",
 "byteorder",
 "bytes",
@@ -4847,6 +4848,7 @@ dependencies = [
 "either",
 "fst",
 "futures",
+ "half",
 "humantime",
 "itertools 0.14.0",
 "lance-arrow",
@@ -4888,8 +4890,8 @@ dependencies = [

 [[package]]
 name = "lance-arrow"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4911,7 +4913,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4925,7 +4927,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -4934,8 +4936,8 @@ dependencies = [

 [[package]]
 name = "lance-bitpacking"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arrayref",
 "crunchy",
@@ -4945,8 +4947,8 @@ dependencies = [

 [[package]]
 name = "lance-core"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4954,10 +4956,12 @@ dependencies = [
 "arrow-schema",
 "async-trait",
 "blake3",
+ "byteorder",
 "bytes",
 "datafusion-common",
 "datafusion-sql",
 "futures",
+ "itertools 0.14.0",
 "lance-arrow",
 "lance-derive",
 "libc",
@@ -4975,6 +4979,7 @@ dependencies = [
 "snafu 0.9.0",
 "tempfile",
 "tokio",
+ "tokio-stream",
 "tokio-util",
 "tracing",
 "twox-hash",
@@ -4983,8 +4988,8 @@ dependencies = [

 [[package]]
 name = "lance-datafusion"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5014,8 +5019,8 @@ dependencies = [

 [[package]]
 name = "lance-datagen"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5032,8 +5037,8 @@ dependencies = [

 [[package]]
 name = "lance-derive"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -5042,8 +5047,8 @@ dependencies = [

 [[package]]
 name = "lance-encoding"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5068,6 +5073,7 @@ dependencies = [
 "num-traits",
 "prost",
 "prost-build",
+ "rand 0.9.5",
 "tokio",
 "tracing",
 "xxhash-rust",
@@ -5076,8 +5082,8 @@ dependencies = [

 [[package]]
 name = "lance-file"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5108,8 +5114,8 @@ dependencies = [

 [[package]]
 name = "lance-index"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -5124,6 +5130,7 @@ dependencies = [
 "async-trait",
 "bitvec",
 "bytes",
+ "chrono",
 "crossbeam-queue",
 "datafusion",
 "datafusion-common",
@@ -5141,6 +5148,7 @@ dependencies = [
 "lance-bitpacking",
 "lance-core",
 "lance-datafusion",
+ "lance-datagen",
 "lance-encoding",
 "lance-file",
 "lance-index-core",
@@ -5169,12 +5177,13 @@ dependencies = [
 "tempfile",
 "tokio",
 "tracing",
+ "uuid",
 ]

 [[package]]
 name = "lance-index-core"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5196,8 +5205,8 @@ dependencies = [

 [[package]]
 name = "lance-io"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5211,6 +5220,7 @@ dependencies = [
 "futures",
 "http 1.5.0",
 "io-uring",
+ "lance-arrow",
 "lance-core",
 "lance-namespace",
 "log",
@@ -5228,28 +5238,29 @@ dependencies = [
 "tokio",
 "tracing",
 "url",
- "uuid",
 ]

 [[package]]
 name = "lance-linalg"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arrow-array",
+ "arrow-buffer",
 "arrow-schema",
 "cc",
 "half",
 "lance-arrow",
 "lance-core",
 "num-traits",
+ "rand 0.9.5",
 "rayon",
 ]

 [[package]]
 name = "lance-namespace"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arrow",
 "async-trait",
@@ -5261,8 +5272,8 @@ dependencies = [

 [[package]]
 name = "lance-namespace-impls"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arrow",
 "arrow-ipc",
@@ -5292,6 +5303,7 @@ dependencies = [
 "serde",
 "serde_json",
 "sha2 0.10.9",
+ "time",
 "tokio",
 "tower",
 "tower-http 0.5.2",
@@ -5315,13 +5327,14 @@ dependencies = [

 [[package]]
 name = "lance-select"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
 "arrow-schema",
 "byteorder",
+ "bytes",
 "itertools 0.14.0",
 "lance-core",
 "roaring",
@@ -5330,8 +5343,8 @@ dependencies = [

 [[package]]
 name = "lance-table"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5341,7 +5354,6 @@ dependencies = [
 "async-trait",
 "aws-credential-types",
 "aws-sdk-dynamodb",
- "blake3",
 "byteorder",
 "bytes",
 "chrono",
@@ -5371,8 +5383,8 @@ dependencies = [

 [[package]]
 name = "lance-testing"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5385,8 +5397,8 @@ dependencies = [

 [[package]]
 name = "lance-tokenizer"
-version = "11.0.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
+version = "11.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
 "frostem",
 "icu_segmenter",
@@ -5399,7 +5411,7 @@ dependencies = [

 [[package]]
 name = "lancedb"
-version = "0.37.1-beta.1"
+version = "0.37.1-beta.0"
 dependencies = [
 "ahash",
 "anyhow",
@@ -5435,6 +5447,7 @@ dependencies = [
 "datafusion-physical-plan",
 "datafusion-sql",
 "futures",
+ "goosefs-sdk",
 "half",
 "hf-hub",
 "http 1.5.0",
@@ -5467,7 +5480,6 @@ dependencies = [
 "random_word",
 "regex",
 "reqwest 0.12.28",
- "roaring",
 "rstest",
 "semver",
 "serde",
@@ -5487,7 +5499,7 @@ dependencies = [

 [[package]]
 name = "lancedb-nodejs"
-version = "0.37.1-beta.1"
+version = "0.37.1-beta.0"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5512,7 +5524,7 @@ dependencies = [

 [[package]]
 name = "lancedb-python"
-version = "0.37.1-beta.1"
+version = "0.37.1-beta.0"
 dependencies = [
 "arrow",
 "async-trait",
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"

 [workspace.dependencies]
-lance = { "version" = "=11.0.0-beta.8", default-features = false, "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=11.0.0-beta.8", default-features = false, "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=11.0.0-beta.8", default-features = false, "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=11.0.0-beta.2", default-features = false, "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=11.0.0-beta.2", default-features = false, "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=11.0.0-beta.2", default-features = false, "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
 arrow = { version = "58.0.0", optional = false }
@@ -52,7 +52,7 @@ env_logger = "0.11"
 half = { "version" = "2.7.1", default-features = false, features = [
    "num-traits",
 ] }
-futures = "0.3"
+futures = "0"
 log = "0.4"
 metrics = "0.24"
 metrics-util = "0.19"
@@ -101,13 +101,6 @@ ignore = [
    # https://rustsec.org/advisories/RUSTSEC-2026-0195
    { id = "RUSTSEC-2026-0194", reason = "transitive via inferno/lance/opendal; XML from trusted cloud endpoints, not attacker-controlled" },
    { id = "RUSTSEC-2026-0195", reason = "transitive via inferno/lance/opendal; XML from trusted cloud endpoints, not attacker-controlled" },
-    # smartstring: unmaintained — the repository was archived by its author on
-    # 2026-05-03. Not a vulnerability. Reached only transitively through polars
-    # (polars-core/-io/-ops/-time/-utils); nothing in LanceDB depends on it directly.
-    # The advisory states no safe upgrade is available: upstream recommends
-    # compact_str/smol_str, so clearing this requires polars to migrate.
-    # https://rustsec.org/advisories/RUSTSEC-2026-0249
-    { id = "RUSTSEC-2026-0249", reason = "smartstring unmaintained via polars; no fixed upstream release" },
 ]

 # ---------------------------------------------------------------------------
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-core</artifactId>
-    <version>0.38.0-beta.0</version>
+    <version>0.37.1-beta.0</version>
 </dependency>
 ```

@@ -386,29 +386,6 @@ Drop an existing table.

 ***

-### dropTableAsync()
-
-```ts
-abstract dropTableAsync(name, namespacePath?): Promise<Job>
-```
-
-Start dropping a table and return its cleanup job.
-
-The table may become unavailable before its data files are removed. Wait
-on the returned job to know when cleanup has finished.
-
-#### Parameters
-
-* **name**: `string`
-
-* **namespacePath?**: `string`[]
-
-#### Returns
-
-`Promise`&lt;[`Job`](Job.md)&gt;
-
-***
-
 ### getJob()

 ```ts
@@ -431,10 +431,9 @@ Read the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) currently installed on th

 Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
 spec has been set, or it was removed with [Table#unsetLsmWriteSpec](Table.md#unsetlsmwritespec)).
-The returned spec mirrors what was passed to
-[Table#setLsmWriteSpec](Table.md#setlsmwritespec), except that `maintainedIndexes` always
-reports the concrete list resolved when the spec was set — `undefined`
-never round-trips.
+The returned spec — including its `maintainedIndexes` and
+`writerConfigDefaults` — mirrors what was passed to
+[Table#setLsmWriteSpec](Table.md#setlsmwritespec).

 #### Returns

@@ -807,11 +806,6 @@ All variants require the table to have an unenforced primary key
 ([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally
 requires it to be the single column being bucketed.

-Omitting `maintainedIndexes` maintains every index on the table, resolved
-here, failing if one cannot be maintained — name them to install anyway.
-Naming them pins an exact set, and a still-building index is rejected
-rather than quietly omitted.
-
 #### Parameters

 * **spec**: [`LsmWriteSpec`](../interfaces/LsmWriteSpec.md)
@@ -34,9 +34,7 @@ Bucket and identity variants: the sharding column.
 optional maintainedIndexes: string[];
 ```

-Indexes the MemWAL keeps up to date. Omit to maintain every supported
-index, resolved on install — a snapshot, so indexes created later are not
-maintained. Pass `[]` for none.
+Names of indexes the MemWAL should keep up to date during writes.

 ***

@@ -44,7 +44,4 @@ The number of rows in the table
 totalBytes: number;
 ```

-The total size, in bytes, of the table's data files, index files, and
-overlay files
-
-Read from the manifest, so this excludes deletion files and manifests.
+The total number of bytes in the table
@@ -31,7 +31,7 @@ is also an [asynchronous API client](#connections-asynchronous).
 ## Namespaces (Synchronous)

 A namespace-backed connection resolves tables through a
-[Lance namespace](https://lance-format.github.io/lance-namespace/) service instead of
+[Lance namespace](https://lancedb.github.io/lance-namespace/) service instead of
 listing a storage directory.

 ::: lancedb.connect_namespace
@@ -8,7 +8,7 @@
    <parent>
      <groupId>com.lancedb</groupId>
      <artifactId>lancedb-parent</artifactId>
-      <version>0.38.0-beta.0</version>
+      <version>0.37.1-beta.0</version>
      <relativePath>../pom.xml</relativePath>
    </parent>

@@ -6,7 +6,7 @@

    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-parent</artifactId>
-    <version>0.38.0-beta.0</version>
+    <version>0.37.1-beta.0</version>
    <packaging>pom</packaging>
    <name>${project.artifactId}</name>
    <description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>11.0.0-beta.8</lance-core.version>
+        <lance-core.version>11.0.0-beta.2</lance-core.version>
        <spotless.skip>false</spotless.skip>
        <spotless.version>2.30.0</spotless.version>
        <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.38.0-beta.0"
+version = "0.37.1-beta.0"
 publish = false
 license.workspace = true
 description.workspace = true
@@ -6,9 +6,7 @@ import * as arrow17 from "apache-arrow-17";
 import * as arrow18 from "apache-arrow-18";

 import {
-  Vector as CurrentVector,
  convertToTable,
-  tableFromIPC as currentTableFromIPC,
  fromBufferToRecordBatch,
  fromDataToBuffer,
  fromRecordBatchToBuffer,
@@ -21,7 +19,6 @@ import {
  FunctionOptions,
 } from "../lancedb/embedding/embedding_function";
 import { EmbeddingFunctionConfig } from "../lancedb/embedding/registry";
-import { sanitizeTable } from "../lancedb/sanitize";

 // biome-ignore lint/suspicious/noExplicitAny: skip
 function sampleRecords(): Array<Record<string, any>> {
@@ -67,11 +64,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
      tableFromIPC,
      DataType,
      Dictionary,
-      RecordBatch: ArrowRecordBatch,
-      Table: ArrowTable,
      Uint8: ArrowUint8,
-      makeData: arrowMakeData,
-      vectorFromArray,
      // biome-ignore lint/suspicious/noExplicitAny: <explanation>
    } = <any>arrow;
    type Schema = ApacheArrow["Schema"];
@@ -1061,114 +1054,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
    });

    describe("when using two versions of arrow", function () {
-      it("preserves a dictionary shared by multiple fields", async function () {
-        const values = ["alpha", "beta", "alpha"];
-        const dictionaryVector = vectorFromArray(values);
-        const batch = new ArrowRecordBatch({
-          first: dictionaryVector.data[0],
-          second: dictionaryVector.data[0],
-        });
-        const table = new ArrowTable([batch]);
-
-        const sanitized = sanitizeTable(table);
-        expect([...sanitized.getChild("first")!]).toEqual(values);
-        expect([...sanitized.getChild("second")!]).toEqual(values);
-        const firstType = sanitized.schema.fields[0].type as {
-          dictionary: unknown;
-        };
-        const secondType = sanitized.schema.fields[1].type as {
-          dictionary: unknown;
-        };
-        expect(secondType.dictionary).toBe(firstType.dictionary);
-        expect(sanitized.batches[0].data.children[1].dictionary).toBe(
-          sanitized.batches[0].data.children[0].dictionary,
-        );
-
-        const buf = await fromDataToBuffer(table);
-        const actual = currentTableFromIPC(buf);
-        expect([...actual.getChild("first")!]).toEqual(values);
-        expect([...actual.getChild("second")!]).toEqual(values);
-      });
-
-      it("preserves shared dictionary data from another Arrow version", async function () {
-        const values = ["alpha", "beta", "alpha"];
-        const dictionaryVector = vectorFromArray(values);
-        const firstBatch = new ArrowRecordBatch({
-          label: dictionaryVector.slice(0, 2).data[0],
-        });
-        const secondBatch = new ArrowRecordBatch({
-          label: dictionaryVector.slice(2).data[0],
-        });
-        const table = new ArrowTable([firstBatch, secondBatch]);
-
-        const sanitized = sanitizeTable(table);
-        expect([...sanitized.getChild("label")!]).toEqual(values);
-
-        const dictionaries = sanitized.batches.map(
-          (batch) => batch.data.children[0].dictionary,
-        );
-        expect(dictionaries[0]).toBeInstanceOf(CurrentVector);
-        expect(dictionaries[1]).toBe(dictionaries[0]);
-
-        const buf = await fromDataToBuffer(table);
-        const actual = currentTableFromIPC(buf);
-        expect([...actual.getChild("label")!]).toEqual(values);
-      });
-
-      it("preserves shared chunks in growing dictionaries", async function () {
-        const type = new Dictionary(new Utf8(), new Int32(), 42, false);
-        const firstDictionary = vectorFromArray(["alpha", "beta"], new Utf8());
-        const secondDictionary = firstDictionary.concat(
-          vectorFromArray(["gamma"], new Utf8()),
-        );
-        const firstData = arrowMakeData({
-          type,
-          data: Int32Array.from([0, 1]),
-          dictionary: firstDictionary,
-        });
-        const secondData = arrowMakeData({
-          type,
-          data: Int32Array.from([2]),
-          dictionary: secondDictionary,
-        });
-        const table = new ArrowTable([
-          new ArrowRecordBatch({ label: firstData }),
-          new ArrowRecordBatch({ label: secondData }),
-        ]);
-
-        const sanitized = sanitizeTable(table);
-        const expected = ["alpha", "beta", "gamma"];
-        expect([...sanitized.getChild("label")!]).toEqual(expected);
-        const firstLocalDictionary =
-          sanitized.batches[0].data.children[0].dictionary!;
-        const secondLocalDictionary =
-          sanitized.batches[1].data.children[0].dictionary!;
-        expect(secondLocalDictionary.data[0]).toBe(
-          firstLocalDictionary.data[0],
-        );
-
-        const buf = await fromTableToBuffer(sanitized);
-        const actual = currentTableFromIPC(buf);
-        expect([...actual.getChild("label")!]).toEqual(expected);
-      });
-
-      it("can serialize list data from another Arrow version", async function () {
-        const values = [["anime", "action"], [], null];
-        const vector = vectorFromArray(
-          values,
-          new List(new Field("item", new Utf8(), true)),
-        );
-        const table = new ArrowTable({ tags: vector });
-
-        const buf = await fromDataToBuffer(table);
-        const actual = currentTableFromIPC(buf);
-        const actualTags = actual.getChild("tags");
-
-        expect(actualTags?.get(0)?.toJSON()).toEqual(values[0]);
-        expect(actualTags?.get(1)?.toJSON()).toEqual(values[1]);
-        expect(actualTags?.get(2)).toBeNull();
-      });
-
      it("can still import data", async function () {
        const schema = new arrow15.Schema([
          new arrow15.Field("id", new arrow15.Int32()),
@@ -89,16 +89,6 @@ describe("given a connection", () => {
    await db.createTable("test4", [{ id: 1 }, { id: 2 }]);
  });

-  it("should return a completed job when dropping a local table", async () => {
-    await db.createTable("async-drop", [{ id: 1 }]);
-
-    const job = await db.dropTableAsync("async-drop");
-    expect(job.id).toBeNull();
-    await expect(job.status()).resolves.toBe("finished");
-    await job.wait();
-    await expect(db.tableNames()).resolves.toEqual([]);
-  });
-
  it("should fail if creating table twice, unless overwrite is true", async () => {
    let tbl = await db.createTable("test", [{ id: 1 }, { id: 2 }]);
    await expect(tbl.countRows()).resolves.toBe(2);
@@ -11,11 +11,8 @@ import {
  Float16,
  Float32,
  Float64,
-  Int32,
  Schema,
  Utf8,
-  fromDataToBuffer,
-  tableFromIPC,
 } from "../lancedb/arrow";
 import { EmbeddingFunction, LanceSchema } from "../lancedb/embedding";
 import { getRegistry, register } from "../lancedb/embedding/registry";
@@ -187,63 +184,6 @@ describe("embedding functions", () => {
    const vector0 = JSON.parse(JSON.stringify(arr[0].vector));
    expect(vector0).toEqual([1, 2, 3]);
  });
-
-  it("should append generated vectors to a non-nullable schema", async () => {
-    @register("non_nullable_schema_test")
-    class MockEmbeddingFunction extends EmbeddingFunction<string> {
-      ndims() {
-        return 3;
-      }
-      embeddingDataType(): Float {
-        return new Float64();
-      }
-      async computeSourceEmbeddings(data: string[]) {
-        return data.map(() => [1, 2, 3]);
-      }
-    }
-
-    const schema = new Schema([
-      new Field("id", new Int32()),
-      new Field("text", new Utf8()),
-      new Field("type", new Utf8()),
-      new Field(
-        "vector",
-        new FixedSizeList(3, new Field("item", new Float64())),
-      ),
-    ]);
-    const func = new MockEmbeddingFunction();
-    const db = await connect(tmpDir.name);
-    const table = await db.createEmptyTable("test_non_nullable", schema, {
-      embeddingFunction: {
-        function: func,
-        sourceColumn: "text",
-      },
-    });
-
-    const data = [
-      { id: 1, text: "Carrot", type: "vegetable" },
-      { id: 2, text: "Apple", type: "fruit" },
-    ];
-    const buffer = await fromDataToBuffer(
-      data,
-      undefined,
-      await table.schema(),
-    );
-    const generatedTable = tableFromIPC(buffer);
-    const vectorField = generatedTable.schema.fields.find(
-      (field) => field.name === "vector",
-    );
-    expect(vectorField?.nullable).toBe(false);
-
-    await table.add(data);
-
-    const rows = await table.query().toArray();
-    expect(rows).toHaveLength(2);
-    for (const row of rows) {
-      expect([...row.vector]).toEqual([1, 2, 3]);
-    }
-  });
-
  it("should error when appending to a table with an unregistered embedding function", async () => {
    @register("mock")
    class MockEmbeddingFunction extends EmbeddingFunction<string> {
@@ -1,14 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-import packageJson = require("../package.json");
-
-describe("package metadata", () => {
-  it("requires Node.js type declarations compatible with the runtime", () => {
-    expect(packageJson.engines.node).toBe(">= 18");
-    expect(packageJson.peerDependencies["@types/node"]).toBe(">=18");
-    expect(packageJson.peerDependenciesMeta["@types/node"]).toEqual({
-      optional: true,
-    });
-  });
-});
@@ -110,81 +110,6 @@ describe("Query outputSchema", () => {
  });
 });

-describe("Search pagination", () => {
-  let tmpDir: tmp.DirResult;
-  let table: Table;
-
-  beforeEach(async () => {
-    tmpDir = tmp.dirSync({ unsafeCleanup: true });
-    const db = await connect(tmpDir.name);
-    const schema = new Schema([
-      new Field("id", new Int64(), false),
-      new Field("text", new Utf8(), false),
-      new Field(
-        "vector",
-        new FixedSizeList(2, new Field("item", new Float32())),
-        false,
-      ),
-    ]);
-    const data = makeArrowTable(
-      [
-        { id: 1n, text: "common", vector: [0, 0] },
-        { id: 2n, text: "common common", vector: [1, 1] },
-        { id: 3n, text: "common common common", vector: [2, 2] },
-        { id: 4n, text: "common common common common", vector: [3, 3] },
-      ],
-      { schema },
-    );
-    table = await db.createTable("test", data);
-  });
-
-  afterEach(() => {
-    tmpDir.removeCallback();
-  });
-
-  it("applies offset after the vector search limit", async () => {
-    const allResults = await table
-      .vectorSearch([0, 0])
-      .select(["id"])
-      .limit(4)
-      .toArray();
-    const secondPage = await table
-      .vectorSearch([0, 0])
-      .select(["id"])
-      .limit(2)
-      .offset(2)
-      .toArray();
-
-    expect(allResults).toHaveLength(4);
-    expect(secondPage).toHaveLength(2);
-    expect(secondPage.map((row) => row.id)).toEqual(
-      allResults.slice(2, 4).map((row) => row.id),
-    );
-  });
-
-  it("applies offset after the full-text search limit", async () => {
-    await table.createIndex("text", { config: Index.fts() });
-
-    const allResults = await table
-      .search("common", "fts")
-      .select(["id"])
-      .limit(4)
-      .toArray();
-    const secondPage = await table
-      .search("common", "fts")
-      .select(["id"])
-      .limit(2)
-      .offset(2)
-      .toArray();
-
-    expect(allResults).toHaveLength(4);
-    expect(secondPage).toHaveLength(2);
-    expect(secondPage.map((row) => row.id)).toEqual(
-      allResults.slice(2, 4).map((row) => row.id),
-    );
-  });
-});
-
 describe("Query orderBy", () => {
  let tmpDir: tmp.DirResult;
  let table: Table;
@@ -277,16 +277,8 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
        },
        numIndices: 0,
        numRows: 3,
-        // Full on-disk size of the two data files, footers and metadata included.
-        totalBytes: 684,
+        totalBytes: 44,
      });
-
-      // Index files count toward totalBytes too (only deletion files and
-      // manifests are excluded).
-      await table.createIndex("id", { config: Index.btree() });
-      const statsWithIndex = await table.stats();
-      expect(statsWithIndex.numIndices).toBe(1);
-      expect(statsWithIndex.totalBytes).toBeGreaterThan(684);
    });

    it("should overwrite data if asked", async () => {
@@ -327,14 +327,6 @@ export abstract class Connection {
   */
  abstract dropTable(name: string, namespacePath?: string[]): Promise<void>;

-  /**
-   * Start dropping a table and return its cleanup job.
-   *
-   * The table may become unavailable before its data files are removed. Wait
-   * on the returned job to know when cleanup has finished.
-   */
-  abstract dropTableAsync(name: string, namespacePath?: string[]): Promise<Job>;
-
  /**
   * Drop all tables in the database.
   * @param {string[]} namespacePath The namespace path to drop tables from (defaults to root namespace).
@@ -713,10 +705,6 @@ export class LocalConnection extends Connection {
    return this.inner.dropTable(name, namespacePath ?? []);
  }

-  async dropTableAsync(name: string, namespacePath?: string[]): Promise<Job> {
-    return this.inner.dropTableAsync(name, namespacePath ?? []);
-  }
-
  async dropAllTables(namespacePath?: string[]): Promise<void> {
    return this.inner.dropAllTables(namespacePath ?? []);
  }
@@ -9,7 +9,7 @@
 // comes from the exact same library instance.  This is not always the case
 // and so we must sanitize the input to ensure that it is compatible.

-import { BufferType, Data, Vector } from "apache-arrow";
+import { BufferType, Data } from "apache-arrow";
 import type { IntBitWidth, TKeys, TimeBitWidth } from "apache-arrow/type";
 import {
  Binary,
@@ -74,20 +74,6 @@ import {
  Utf8,
 } from "./arrow";

-type SanitizationContext = {
-  types: WeakMap<object, DataType>;
-  vectors: WeakMap<object, Vector>;
-  data: WeakMap<object, Data<DataType>>;
-};
-
-function createSanitizationContext(): SanitizationContext {
-  return {
-    types: new WeakMap(),
-    vectors: new WeakMap(),
-    data: new WeakMap(),
-  };
-}
-
 export function sanitizeMetadata(
  metadataLike?: unknown,
 ): Map<string, string> | undefined {
@@ -200,13 +186,6 @@ export function sanitizeInterval(typeLike: object) {
 }

 export function sanitizeList(typeLike: object) {
-  return sanitizeListWithContext(typeLike, createSanitizationContext());
-}
-
-function sanitizeListWithContext(
-  typeLike: object,
-  context: SanitizationContext,
-) {
  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
    throw Error(
      "Expected a List type to have an array-like `children` property",
@@ -215,35 +194,19 @@ function sanitizeListWithContext(
  if (typeLike.children.length !== 1) {
    throw Error("Expected a List type to have exactly one child");
  }
-  return new List(sanitizeFieldWithContext(typeLike.children[0], context));
+  return new List(sanitizeField(typeLike.children[0]));
 }

 export function sanitizeStruct(typeLike: object) {
-  return sanitizeStructWithContext(typeLike, createSanitizationContext());
-}
-
-function sanitizeStructWithContext(
-  typeLike: object,
-  context: SanitizationContext,
-) {
  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
    throw Error(
      "Expected a Struct type to have an array-like `children` property",
    );
  }
-  return new Struct(
-    typeLike.children.map((child) => sanitizeFieldWithContext(child, context)),
-  );
+  return new Struct(typeLike.children.map((child) => sanitizeField(child)));
 }

 export function sanitizeUnion(typeLike: object) {
-  return sanitizeUnionWithContext(typeLike, createSanitizationContext());
-}
-
-function sanitizeUnionWithContext(
-  typeLike: object,
-  context: SanitizationContext,
-) {
  if (
    !("typeIds" in typeLike) ||
    !("mode" in typeLike) ||
@@ -263,7 +226,7 @@ function sanitizeUnionWithContext(
    typeLike.mode,
    // biome-ignore lint/suspicious/noExplicitAny: skip
    typeLike.typeIds as any,
-    typeLike.children.map((child) => sanitizeFieldWithContext(child, context)),
+    typeLike.children.map((child) => sanitizeField(child)),
  );
 }

@@ -271,19 +234,6 @@ export function sanitizeTypedUnion(
  typeLike: object,
  // eslint-disable-next-line @typescript-eslint/naming-convention
  UnionType: typeof DenseUnion | typeof SparseUnion,
-) {
-  return sanitizeTypedUnionWithContext(
-    typeLike,
-    UnionType,
-    createSanitizationContext(),
-  );
-}
-
-function sanitizeTypedUnionWithContext(
-  typeLike: object,
-  // eslint-disable-next-line @typescript-eslint/naming-convention
-  UnionType: typeof DenseUnion | typeof SparseUnion,
-  context: SanitizationContext,
 ) {
  if (!("typeIds" in typeLike)) {
    throw Error(
@@ -298,7 +248,7 @@ function sanitizeTypedUnionWithContext(

  return new UnionType(
    typeLike.typeIds as Int32Array | number[],
-    typeLike.children.map((child) => sanitizeFieldWithContext(child, context)),
+    typeLike.children.map((child) => sanitizeField(child)),
  );
 }

@@ -312,16 +262,6 @@ export function sanitizeFixedSizeBinary(typeLike: object) {
 }

 export function sanitizeFixedSizeList(typeLike: object) {
-  return sanitizeFixedSizeListWithContext(
-    typeLike,
-    createSanitizationContext(),
-  );
-}
-
-function sanitizeFixedSizeListWithContext(
-  typeLike: object,
-  context: SanitizationContext,
-) {
  if (!("listSize" in typeLike) || typeof typeLike.listSize !== "number") {
    throw Error("Expected a FixedSizeList type to have a `listSize` property");
  }
@@ -335,18 +275,11 @@ function sanitizeFixedSizeListWithContext(
  }
  return new FixedSizeList(
    typeLike.listSize,
-    sanitizeFieldWithContext(typeLike.children[0], context),
+    sanitizeField(typeLike.children[0]),
  );
 }

 export function sanitizeMap(typeLike: object) {
-  return sanitizeMapWithContext(typeLike, createSanitizationContext());
-}
-
-function sanitizeMapWithContext(
-  typeLike: object,
-  context: SanitizationContext,
-) {
  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
    throw Error(
      "Expected a Map type to have an array-like `children` property",
@@ -359,10 +292,7 @@ function sanitizeMapWithContext(
    throw Error("Expected a Map type to have exactly one child");
  }

-  return new Map_(
-    sanitizeFieldWithContext(typeLike.children[0], context),
-    typeLike.keysSorted,
-  );
+  return new Map_(sanitizeField(typeLike.children[0]), typeLike.keysSorted);
 }

 export function sanitizeDuration(typeLike: object) {
@@ -373,13 +303,6 @@ export function sanitizeDuration(typeLike: object) {
 }

 export function sanitizeDictionary(typeLike: object) {
-  return sanitizeDictionaryWithContext(typeLike, createSanitizationContext());
-}
-
-function sanitizeDictionaryWithContext(
-  typeLike: object,
-  context: SanitizationContext,
-) {
  if (!("id" in typeLike) || typeof typeLike.id !== "number") {
    throw Error("Expected a Dictionary type to have an `id` property");
  }
@@ -393,8 +316,8 @@ function sanitizeDictionaryWithContext(
    throw Error("Expected a Dictionary type to have an `isOrdered` property");
  }
  return new Dictionary(
-    sanitizeTypeWithContext(typeLike.dictionary, context),
-    sanitizeTypeWithContext(typeLike.indices, context) as TKeys,
+    sanitizeType(typeLike.dictionary),
+    sanitizeType(typeLike.indices) as TKeys,
    typeLike.id,
    typeLike.isOrdered,
  );
@@ -402,23 +325,12 @@ function sanitizeDictionaryWithContext(

 // biome-ignore lint/suspicious/noExplicitAny: skip
 export function sanitizeType(typeLike: unknown): DataType<any> {
-  return sanitizeTypeWithContext(typeLike, createSanitizationContext());
-}
-
-function sanitizeTypeWithContext(
-  typeLike: unknown,
-  context: SanitizationContext,
-): DataType {
  if (typeof typeLike === "string") {
    return dataTypeFromName(typeLike);
  }
  if (typeof typeLike !== "object" || typeLike === null) {
    throw Error("Expected a Type but object was null/undefined");
  }
-  const cached = context.types.get(typeLike);
-  if (cached !== undefined) {
-    return cached;
-  }
  if (
    !("typeId" in typeLike) ||
    !(
@@ -437,16 +349,6 @@ function sanitizeTypeWithContext(
    throw Error("Type's typeId property was not a function or number");
  }

-  const type = sanitizeTypeById(typeLike, typeId, context);
-  context.types.set(typeLike, type);
-  return type;
-}
-
-function sanitizeTypeById(
-  typeLike: object,
-  typeId: Type,
-  context: SanitizationContext,
-): DataType {
  switch (typeId) {
    case Type.NONE:
      throw Error("Received a Type with a typeId of NONE");
@@ -473,21 +375,21 @@ function sanitizeTypeById(
    case Type.Interval:
      return sanitizeInterval(typeLike);
    case Type.List:
-      return sanitizeListWithContext(typeLike, context);
+      return sanitizeList(typeLike);
    case Type.Struct:
-      return sanitizeStructWithContext(typeLike, context);
+      return sanitizeStruct(typeLike);
    case Type.Union:
-      return sanitizeUnionWithContext(typeLike, context);
+      return sanitizeUnion(typeLike);
    case Type.FixedSizeBinary:
      return sanitizeFixedSizeBinary(typeLike);
    case Type.FixedSizeList:
-      return sanitizeFixedSizeListWithContext(typeLike, context);
+      return sanitizeFixedSizeList(typeLike);
    case Type.Map:
-      return sanitizeMapWithContext(typeLike, context);
+      return sanitizeMap(typeLike);
    case Type.Duration:
      return sanitizeDuration(typeLike);
    case Type.Dictionary:
-      return sanitizeDictionaryWithContext(typeLike, context);
+      return sanitizeDictionary(typeLike);
    case Type.Int8:
      return new Int8();
    case Type.Int16:
@@ -531,9 +433,9 @@ function sanitizeTypeById(
    case Type.TimestampSecond:
      return sanitizeTypedTimestamp(typeLike, TimestampSecond);
    case Type.DenseUnion:
-      return sanitizeTypedUnionWithContext(typeLike, DenseUnion, context);
+      return sanitizeTypedUnion(typeLike, DenseUnion);
    case Type.SparseUnion:
-      return sanitizeTypedUnionWithContext(typeLike, SparseUnion, context);
+      return sanitizeTypedUnion(typeLike, SparseUnion);
    case Type.IntervalDayTime:
      return new IntervalDayTime();
    case Type.IntervalYearMonth:
@@ -552,13 +454,6 @@ function sanitizeTypeById(
 }

 export function sanitizeField(fieldLike: unknown): Field {
-  return sanitizeFieldWithContext(fieldLike, createSanitizationContext());
-}
-
-function sanitizeFieldWithContext(
-  fieldLike: unknown,
-  context: SanitizationContext,
-): Field {
  if (fieldLike instanceof Field) {
    return fieldLike;
  }
@@ -576,7 +471,7 @@ function sanitizeFieldWithContext(
  }
  let type: DataType;
  try {
-    type = sanitizeTypeWithContext(fieldLike.type, context);
+    type = sanitizeType(fieldLike.type);
  } catch (error: unknown) {
    throw Error(
      `Unable to sanitize type for field: ${fieldLike.name} due to error: ${error}`,
@@ -606,13 +501,6 @@ function sanitizeFieldWithContext(
 * than lancedb is using.
 */
 export function sanitizeSchema(schemaLike: SchemaLike): Schema {
-  return sanitizeSchemaWithContext(schemaLike, createSanitizationContext());
-}
-
-function sanitizeSchemaWithContext(
-  schemaLike: SchemaLike,
-  context: SanitizationContext,
-): Schema {
  if (schemaLike instanceof Schema) {
    return schemaLike;
  }
@@ -634,7 +522,7 @@ function sanitizeSchemaWithContext(
    );
  }
  const sanitizedFields = schemaLike.fields.map((field) =>
-    sanitizeFieldWithContext(field, context),
+    sanitizeField(field),
  );
  return new Schema(sanitizedFields, metadata);
 }
@@ -656,18 +544,13 @@ export function sanitizeTable(tableLike: TableLike): Table {
      "The table passed in does not appear to be a table (no 'columns' property)",
    );
  }
-  const context = createSanitizationContext();
-  const schema = sanitizeSchemaWithContext(tableLike.schema, context);
-  const batches = tableLike.batches.map((batch) =>
-    sanitizeRecordBatch(batch, context),
-  );
+  const schema = sanitizeSchema(tableLike.schema);
+
+  const batches = tableLike.batches.map(sanitizeRecordBatch);
  return new Table(schema, batches);
 }

-function sanitizeRecordBatch(
-  batchLike: RecordBatchLike,
-  context: SanitizationContext,
-): RecordBatch {
+function sanitizeRecordBatch(batchLike: RecordBatchLike): RecordBatch {
  if (batchLike instanceof RecordBatch) {
    return batchLike;
  }
@@ -684,43 +567,19 @@ function sanitizeRecordBatch(
      "The record batch passed in does not appear to be a record batch (no 'data' property)",
    );
  }
-  const schema = sanitizeSchemaWithContext(batchLike.schema, context);
-  const data = sanitizeData(batchLike.data, context) as Data<Struct>;
+  const schema = sanitizeSchema(batchLike.schema);
+  const data = sanitizeData(batchLike.data);
  return new RecordBatch(schema, data);
 }
-
-type DictionaryVectorLike = {
-  data: readonly DataLike[];
-};
-
-type DictionaryDataLike = DataLike & {
-  dictionary?: DictionaryVectorLike;
-};
-
 function sanitizeData(
  dataLike: DataLike,
-  context: SanitizationContext,
-): Data<DataType> {
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+): import("apache-arrow").Data<Struct<any>> {
  if (dataLike instanceof Data) {
    return dataLike;
  }
-  const cachedData = context.data.get(dataLike);
-  if (cachedData !== undefined) {
-    return cachedData;
-  }
-  const dictionaryLike = (dataLike as DictionaryDataLike).dictionary;
-  let dictionary: Vector | undefined;
-  if (dictionaryLike !== undefined) {
-    dictionary = context.vectors.get(dictionaryLike);
-    if (dictionary === undefined) {
-      dictionary = new Vector(
-        dictionaryLike.data.map((data) => sanitizeData(data, context)),
-      );
-      context.vectors.set(dictionaryLike, dictionary);
-    }
-  }
-  const data = new Data(
-    sanitizeTypeWithContext(dataLike.type, context),
+  return new Data(
+    dataLike.type,
    dataLike.offset,
    dataLike.length,
    dataLike.nullCount,
@@ -730,11 +589,7 @@ function sanitizeData(
      [BufferType.VALIDITY]: dataLike.nullBitmap,
      [BufferType.TYPE]: dataLike.typeIds,
    },
-    dataLike.children.map((child) => sanitizeData(child, context)),
-    dictionary,
  );
-  context.data.set(dataLike, data);
-  return data;
 }

 const constructorsByTypeName = {
@@ -197,11 +197,7 @@ export interface LsmWriteSpec {
  column?: string;
  /** Bucket variant: the number of buckets, in `[1, 1024]`. */
  numBuckets?: number;
-  /**
-   * Indexes the MemWAL keeps up to date. Omit to maintain every supported
-   * index, resolved on install — a snapshot, so indexes created later are not
-   * maintained. Pass `[]` for none.
-   */
+  /** Names of indexes the MemWAL should keep up to date during writes. */
  maintainedIndexes?: string[];
  /** Default `ShardWriter` configuration recorded in the MemWAL index. */
  writerConfigDefaults?: Record<string, string>;
@@ -599,11 +595,6 @@ export abstract class Table {
   * All variants require the table to have an unenforced primary key
   * ({@link Table#setUnenforcedPrimaryKey}); bucket sharding additionally
   * requires it to be the single column being bucketed.
-   *
-   * Omitting `maintainedIndexes` maintains every index on the table, resolved
-   * here, failing if one cannot be maintained — name them to install anyway.
-   * Naming them pins an exact set, and a still-building index is rejected
-   * rather than quietly omitted.
   * @param {LsmWriteSpec} spec The sharding spec to install.
   * @returns {Promise<void>}
   * @example
@@ -631,10 +622,9 @@ export abstract class Table {
   *
   * Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
   * spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}).
-   * The returned spec mirrors what was passed to
-   * {@link Table#setLsmWriteSpec}, except that `maintainedIndexes` always
-   * reports the concrete list resolved when the spec was set — `undefined`
-   * never round-trips.
+   * The returned spec — including its `maintainedIndexes` and
+   * `writerConfigDefaults` — mirrors what was passed to
+   * {@link Table#setLsmWriteSpec}.
   * @returns {Promise<LsmWriteSpec | undefined>}
   */
  abstract getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.38.0-beta.0",
+	"version": "0.37.1-beta.0",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.38.0-beta.0",
+	"version": "0.37.1-beta.0",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.38.0-beta.0",
+	"version": "0.37.1-beta.0",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.38.0-beta.0",
+	"version": "0.37.1-beta.0",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.38.0-beta.0",
+	"version": "0.37.1-beta.0",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",
@@ -1,6 +1,6 @@
 {
  "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.38.0-beta.0",
+  "version": "0.37.1-beta.0",
  "os": [
    "win32"
  ],
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.38.0-beta.0",
+	"version": "0.37.1-beta.0",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",
@@ -1,12 +1,12 @@
 {
  "name": "@lancedb/lancedb",
-  "version": "0.37.1-beta.1",
+  "version": "0.37.1-beta.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "@lancedb/lancedb",
-      "version": "0.37.1-beta.1",
+      "version": "0.37.1-beta.0",
      "cpu": [
        "x64",
        "arm64"
@@ -55,13 +55,7 @@
        "openai": "4.29.2"
      },
      "peerDependencies": {
-        "@types/node": ">=18",
        "apache-arrow": ">=15.0.0 <=18.1.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/node": {
-          "optional": true
-        }
      }
    },
    "node_modules/@aws-crypto/crc32": {
@@ -11,7 +11,7 @@
    "ann"
  ],
  "private": false,
-  "version": "0.38.0-beta.0",
+  "version": "0.37.1-beta.0",
  "main": "dist/index.js",
  "exports": {
    ".": "./dist/index.js",
@@ -101,12 +101,6 @@
    "openai": "4.29.2"
  },
  "peerDependencies": {
-    "@types/node": ">=18",
    "apache-arrow": ">=15.0.0 <=18.1.0"
-  },
-  "peerDependenciesMeta": {
-    "@types/node": {
-      "optional": true
-    }
  }
 }
@@ -334,22 +334,6 @@ impl Connection {
            .default_error()
    }

-    /// Start dropping a table and return its cleanup job.
-    #[napi(catch_unwind)]
-    pub async fn drop_table_async(
-        &self,
-        name: String,
-        namespace_path: Option<Vec<String>>,
-    ) -> napi::Result<crate::job::Job> {
-        let ns = namespace_path.unwrap_or_default();
-        let job = self
-            .get_inner()?
-            .drop_table_async(&name, &ns)
-            .await
-            .default_error()?;
-        Ok(crate::job::Job::new(job))
-    }
-
    #[napi(catch_unwind)]
    pub async fn drop_all_tables(&self, namespace_path: Option<Vec<String>>) -> napi::Result<()> {
        let ns = namespace_path.unwrap_or_default();
@@ -772,8 +772,7 @@ pub struct LsmWriteSpec {
    pub column: Option<String>,
    /// Bucket variant: the number of buckets, in `[1, 1024]`.
    pub num_buckets: Option<u32>,
-    /// Indexes the MemWAL keeps up to date. Omitted resolves every
-    /// maintainable index on install; an empty array means none.
+    /// Names of indexes the MemWAL should keep up to date during writes.
    pub maintained_indexes: Option<Vec<String>>,
    /// Default `ShardWriter` configuration recorded in the MemWAL index.
    pub writer_config_defaults: Option<HashMap<String, String>>,
@@ -783,6 +782,7 @@ impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
    type Error = napi::Error;

    fn try_from(value: LsmWriteSpec) -> napi::Result<Self> {
+        let maintained = value.maintained_indexes.unwrap_or_default();
        let writer_config_defaults = value.writer_config_defaults.unwrap_or_default();
        let spec = match value.spec_type.as_str() {
            "bucket" => {
@@ -809,7 +809,7 @@ impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
            }
        };
        Ok(spec
-            .with_maintained_indexes(value.maintained_indexes)
+            .with_maintained_indexes(maintained)
            .with_writer_config_defaults(writer_config_defaults))
    }
 }
@@ -827,7 +827,7 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
                spec_type: "bucket".to_string(),
                column: Some(column),
                num_buckets: Some(num_buckets),
-                maintained_indexes,
+                maintained_indexes: Some(maintained_indexes),
                writer_config_defaults: Some(writer_config_defaults),
            },
            Native::Identity {
@@ -838,7 +838,7 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
                spec_type: "identity".to_string(),
                column: Some(column),
                num_buckets: None,
-                maintained_indexes,
+                maintained_indexes: Some(maintained_indexes),
                writer_config_defaults: Some(writer_config_defaults),
            },
            Native::Unsharded {
@@ -848,7 +848,7 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
                spec_type: "unsharded".to_string(),
                column: None,
                num_buckets: None,
-                maintained_indexes,
+                maintained_indexes: Some(maintained_indexes),
                writer_config_defaults: Some(writer_config_defaults),
            },
        }
@@ -1043,10 +1043,7 @@ impl From<lancedb::index::IndexStatistics> for IndexStatistics {

 #[napi(object)]
 pub struct TableStatistics {
-    /// The total size, in bytes, of the table's data files, index files, and
-    /// overlay files
-    ///
-    /// Read from the manifest, so this excludes deletion files and manifests.
+    /// The total number of bytes in the table
    pub total_bytes: i64,

    /// The number of rows in the table
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.38.0-beta.0"
+version = "0.37.1-beta.0"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"
@@ -60,7 +60,7 @@ tests = [
    "pytest-asyncio>=0.21",
    "duckdb>=0.9.0",
    "pytz>=2023.3",
-    "polars>=0.19, <=1.32.3",
+    "polars>=0.19, <=1.3.0",
    "pyarrow<25",
    "pyarrow-stubs>=16.0",
    "pylance==9.0.0rc1",
@@ -140,7 +140,6 @@ include = [
    "python/lancedb/remote/errors.py",
    "python/lancedb/embeddings/__init__.py",
    "python/lancedb/_lancedb.pyi",
-    "python/type_tests/connect.py",
 ]
 exclude = ["python/tests/"]
 pythonVersion = "3.13"
@@ -198,9 +198,6 @@ class Connection(object):
    async def drop_table(
        self, name: str, namespace_path: Optional[List[str]] = None
    ) -> None: ...
-    async def drop_table_async(
-        self, name: str, namespace_path: Optional[List[str]] = None
-    ) -> Job: ...
    async def drop_all_tables(
        self, namespace_path: Optional[List[str]] = None
    ) -> None: ...
@@ -358,10 +355,6 @@ class Table:
    async def set_lsm_write_spec(self, spec: LsmWriteSpec) -> None: ...
    async def unset_lsm_write_spec(self) -> None: ...
    async def get_lsm_write_spec(self) -> Optional[LsmWriteSpec]: ...
-    async def checkpoint_lsm(self) -> None: ...
-    async def flush_lsm(self) -> None: ...
-    async def compact_lsm(self) -> None: ...
-    async def get_lsm_stats(self, include_generation_rows: bool) -> Optional[dict]: ...
    async def close_lsm_writers(self) -> None: ...
    @property
    def tags(self) -> Tags: ...
@@ -656,10 +649,9 @@ class LsmWriteSpec:
    def identity(column: str) -> "LsmWriteSpec": ...
    @staticmethod
    def unsharded() -> "LsmWriteSpec": ...
-    def with_maintained_indexes(self, indexes: Optional[List[str]]) -> "LsmWriteSpec":
-        """Set which indexes the MemWAL keeps up to date. None resolves every
-        index on the table at install, failing if one cannot be maintained;
-        a list is verbatim, empty means none."""
+    def with_maintained_indexes(self, indexes: List[str]) -> "LsmWriteSpec":
+        """Return a copy of this spec asking the MemWAL to keep the named
+        indexes up to date as rows are appended."""
        ...
    def with_writer_config_defaults(self, defaults: Dict[str, str]) -> "LsmWriteSpec":
        """Return a copy of this spec recording the given default
@@ -674,9 +666,7 @@ class LsmWriteSpec:
    @property
    def num_buckets(self) -> Optional[int]: ...
    @property
-    def maintained_indexes(self) -> Optional[List[str]]:
-        """Indexes the MemWAL keeps up to date, or None for every supported one."""
-        ...
+    def maintained_indexes(self) -> List[str]: ...
    @property
    def writer_config_defaults(self) -> Dict[str, str]: ...

@@ -524,12 +524,6 @@ class DBConnection(EnforceOverrides):
            namespace_path = []
        raise NotImplementedError

-    def drop_table_async(
-        self, name: str, namespace_path: Optional[List[str]] = None
-    ) -> Job:
-        """Start dropping a table and return its cleanup job."""
-        raise NotImplementedError
-
    def rename_table(
        self,
        cur_name: str,
@@ -1192,20 +1186,6 @@ class LanceDBConnection(DBConnection):
            )
        )

-    @override
-    def drop_table_async(
-        self, name: str, namespace_path: Optional[List[str]] = None
-    ) -> Job:
-        """Start dropping a table and return its cleanup job.
-
-        The table may become unavailable before its data files are removed.
-        Call :meth:`Job.wait` to wait for cleanup to finish.
-        """
-        if namespace_path is None:
-            namespace_path = []
-        job = LOOP.run(self._conn.drop_table_async(name, namespace_path=namespace_path))
-        return Job(job if isinstance(job, AsyncJob) else AsyncJob(job))
-
    @override
    def drop_all_tables(self, namespace_path: Optional[List[str]] = None):
        if namespace_path is None:
@@ -1983,23 +1963,6 @@ class AsyncConnection(object):
            if f"Table '{name}' was not found" not in str(e):
                raise e

-    async def drop_table_async(
-        self,
-        name: str,
-        *,
-        namespace_path: Optional[List[str]] = None,
-    ) -> AsyncJob:
-        """Start dropping a table and return its cleanup job.
-
-        The table may become unavailable before its data files are removed.
-        Await :meth:`AsyncJob.wait` to wait for cleanup to finish.
-        """
-        if namespace_path is None:
-            namespace_path = []
-        return AsyncJob(
-            await self._inner.drop_table_async(name, namespace_path=namespace_path)
-        )
-
    async def drop_all_tables(self, namespace_path: Optional[List[str]] = None):
        """Drop all tables from the database.

@@ -87,13 +87,12 @@ class JinaEmbeddings(EmbeddingFunction):
        if isinstance(image, bytes):
            image_dict = {"image": base64.b64encode(image).decode("utf-8")}
        elif isinstance(image, (str, Path)):
-            parsed = urlparse(str(image))
+            parsed = urlparse.urlparse(image)
+            # TODO handle drive letter on windows.
            PIL_Image = attempt_import_or_raise("PIL.Image", "pillow")
            if parsed.scheme == "file":
                pil_image = PIL_Image.open(parsed.path)
-            elif parsed.scheme == "" or (os.name == "nt" and len(parsed.scheme) == 1):
-                # A Windows drive letter parses as a one-character scheme
-                # ("C:\\img.png" -> scheme="c"), so treat it as a local path.
+            elif parsed.scheme == "":
                pil_image = PIL_Image.open(image if os.name == "nt" else parsed.path)
            elif parsed.scheme.startswith("http"):
                pil_image = PIL_Image.open(io.BytesIO(url_retrieve(image)))
@@ -49,7 +49,6 @@ from lancedb._lancedb import (
 )
 from lancedb.background_loop import LOOP
 from lancedb.db import AsyncConnection, DBConnection
-from lancedb.job import AsyncJob, Job
 from lance_namespace import (
    LanceNamespace,
    connect as namespace_connect,
@@ -625,18 +624,6 @@ class LanceNamespaceDBConnection(DBConnection):
            namespace_path = []
        LOOP.run(self._inner.drop_table(name, namespace_path=namespace_path))

-    @override
-    def drop_table_async(
-        self, name: str, namespace_path: Optional[List[str]] = None
-    ) -> Job:
-        """Start dropping a table and return its cleanup job."""
-        if namespace_path is None:
-            namespace_path = []
-        job = LOOP.run(
-            self._inner.drop_table_async(name, namespace_path=namespace_path)
-        )
-        return Job(job if isinstance(job, AsyncJob) else AsyncJob(job))
-
    @override
    def rename_table(
        self,
@@ -1147,14 +1134,6 @@ class AsyncLanceNamespaceDBConnection:
            namespace_path = []
        await self._inner.drop_table(name, namespace_path=namespace_path)

-    async def drop_table_async(
-        self, name: str, namespace_path: Optional[List[str]] = None
-    ) -> AsyncJob:
-        """Start dropping a table and return its cleanup job."""
-        if namespace_path is None:
-            namespace_path = []
-        return await self._inner.drop_table_async(name, namespace_path=namespace_path)
-
    async def rename_table(
        self,
        cur_name: str,
@@ -1 +0,0 @@
-
@@ -153,16 +153,6 @@ def Vector(
    return FixedSizeList


-def _raise_bare_vector_error(*_args):
-    raise TypeError("Vector must be parameterized with a dimension, e.g. Vector(128).")
-
-
-# Pydantic v1 and v2 otherwise treat the bare Vector factory as a field validator
-# and inspect its signature, which produces misleading errors about internal types.
-setattr(Vector, "__get_validators__", _raise_bare_vector_error)
-setattr(Vector, "__get_pydantic_core_schema__", _raise_bare_vector_error)
-
-
 def MultiVector(
    dim: int, value_type: pa.DataType = pa.float32(), nullable: bool = True
 ) -> Type:
@@ -23,7 +23,7 @@ import pyarrow as pa

 from ..common import DATA
 from ..db import DBConnection, LOOP
-from ..job import AsyncJob, Job
+from ..job import Job

 if TYPE_CHECKING:
    from .._lancedb import JobDescription, JobInfo
@@ -663,16 +663,6 @@ class RemoteDBConnection(DBConnection):
            namespace_path = []
        LOOP.run(self._conn.drop_table(name, namespace_path=namespace_path))

-    @override
-    def drop_table_async(
-        self, name: str, namespace_path: Optional[List[str]] = None
-    ) -> Job:
-        """Start dropping a table and return its cleanup job."""
-        if namespace_path is None:
-            namespace_path = []
-        job = LOOP.run(self._conn.drop_table_async(name, namespace_path=namespace_path))
-        return Job(job if isinstance(job, AsyncJob) else AsyncJob(job))
-
    @override
    def rename_table(
        self,
@@ -11,11 +11,6 @@ Provides StreamingDataset, a PyTorch IterableDataset that guarantees:
 - **Resumability**: state_dict / load_state_dict capture per-split consumption
  counts so training can resume from an exact mid-epoch position even when the
  distributed topology changes between runs.
-
-Transform failures on bad rows (e.g. nulls or NaNs from incomplete data) can
-be tolerated with ``on_transform_error="skip"``; see the parameter
-documentation on StreamingDataset for how this interacts with the guarantees
-above.
 """

 import ctypes
@@ -27,7 +22,7 @@ import time
 from collections import deque
 from concurrent.futures import ThreadPoolExecutor
 from multiprocessing import RawArray
-from typing import Any, Callable, Iterator, Optional, Union
+from typing import Any, Callable, Iterator, Optional

 from torch.utils.data import IterableDataset, get_worker_info

@@ -132,49 +127,6 @@ class StreamingDataset(IterableDataset):
        Maximum number of transforms to run concurrently.  Must be greater
        than zero.  When ``None`` (the default), uses ``os.cpu_count()`` or 1
        when the CPU count is unavailable.
-    on_transform_error:
-        What to do when the transform raises an exception:
-
-        - ``"raise"`` (the default): the exception propagates and iteration
-          aborts.
-        - ``"skip"``: the failing rows are dropped and iteration continues.
-        - ``"warn"``: like ``"skip"``, but a warning is logged for each
-          failing batch.
-        - a callable ``handler(exc) -> bool``: called with the exception;
-          return ``True`` to skip the failing rows or ``False`` to re-raise.
-          Useful to skip only expected error types (compatible with
-          ``webdataset.handlers`` style handlers).
-
-        When a batch fails, the transform is re-invoked on each single-row
-        slice of the batch so that only the rows that actually fail are
-        dropped.  Transforms should therefore be deterministic and accept
-        batches of any size (including one row).  Skipped rows are counted in
-        ``rows_skipped``.
-
-        Skipping weakens the elastic-determinism guarantee at the end of the
-        epoch: splits that lose more rows than others run dry earlier, and
-        each rank's iterator ends at the last cycle where every split *it
-        owns* still has a row.  Because bad rows are not distributed evenly
-        across splits, this means one rank's iterator can yield noticeably
-        fewer or more steps than another rank's *in the same run* — there is
-        no cross-rank coordination that stops every rank at the same global
-        step.  This is generally safe for asynchronous or single-rank use,
-        but synchronous distributed training (e.g. ranks that call
-        ``all_reduce`` every step) can hang or deadlock if one rank's
-        iterator is exhausted while others are still stepping; callers doing
-        synchronous multi-rank training with ``on_transform_error != "raise"``
-        are responsible for their own cross-rank stopping mechanism (e.g.
-        broadcasting a stop signal on ``StopIteration``).  The final few
-        global steps can also differ across topologies (bounded by the skew
-        in bad-row counts across splits).  The sequence of samples yielded
-        from each split remains deterministic.  Mid-epoch
-        checkpoints remain exact provided the transform fails
-        deterministically; in multi-rank training each rank must save its
-        own ``state_dict`` and the states must be combined with
-        ``merge_state_dicts`` before resuming on a different topology.
-        Prefer the ``filter`` parameter when bad rows can be expressed as a
-        SQL predicate (e.g. ``"col IS NOT NULL"``) — filtering happens before
-        splits are built, so every guarantee is fully preserved.
    worker_info_override:
        If set, used in place of ``torch.utils.data.get_worker_info()`` to
        determine the DataLoader worker assignment.  Intended for unit tests
@@ -200,7 +152,6 @@ class StreamingDataset(IterableDataset):
        filter: Optional[str] = None,
        transform: Optional[Callable] = None,
        transform_parallelism: Optional[int] = None,
-        on_transform_error: Union[str, Callable[[Exception], bool]] = "raise",
        connection_factory: Optional[Callable[[str], Any]] = None,
        worker_info_override=None,
    ):
@@ -216,13 +167,6 @@ class StreamingDataset(IterableDataset):
            )
        if transform_parallelism is not None and transform_parallelism <= 0:
            raise ValueError("transform_parallelism must be greater than 0")
-        if on_transform_error not in ("raise", "skip", "warn") and not callable(
-            on_transform_error
-        ):
-            raise ValueError(
-                "on_transform_error must be 'raise', 'skip', 'warn', or a "
-                f"callable, got {on_transform_error!r}"
-            )

        self._table = table
        self._num_splits = num_splits
@@ -238,7 +182,6 @@ class StreamingDataset(IterableDataset):
        self._filter = filter
        self._transform = transform
        self._transform_parallelism = transform_parallelism
-        self._on_transform_error = on_transform_error
        self._connection_factory = connection_factory
        self._worker_info_override = worker_info_override

@@ -256,28 +199,19 @@ class StreamingDataset(IterableDataset):
        # in the main process.  RawArray is picklable via the forkserver
        # reduction protocol so it survives the dataset pickle round-trip.
        # Layout: [unscanned_rows, raw_rows, cooked_rows, consumed_rows,
-        #          bytes_loaded, fetch_time_us, transform_time_us,
-        #          rows_skipped]
-        self._worker_stats: RawArray = RawArray(ctypes.c_int64, 8)
+        #          bytes_loaded, fetch_time_us, transform_time_us]
+        self._worker_stats: RawArray = RawArray(ctypes.c_int64, 7)

        # Cumulative bytes of Arrow buffer data fetched across all iterations.
        self._bytes_loaded: int = 0
        # Cumulative seconds spent in LanceDB I/O and in transform functions.
        self._fetch_time: float = 0.0
        self._transform_time: float = 0.0
-        # Cumulative rows dropped by on_transform_error across all iterations.
-        self._rows_skipped: int = 0

        # Number of samples each split has already been consumed.  At global
        # step boundaries all splits have consumed this many samples, so a
        # single scalar captures the topology-independent checkpoint state.
        self._resume_offset: int = 0
-        # Permutation position each split has consumed through, keyed by
-        # global split index.  Equal to _resume_offset for every split unless
-        # on_transform_error skipped rows, in which case skipped positions
-        # push the watermark of the affected splits further ahead.  Splits
-        # this instance has never iterated have no entry.
-        self._resume_positions: dict[int, int] = {}

        # Build the permutation table once, deterministically.
        builder = permutation_builder(table)
@@ -341,7 +275,6 @@ class StreamingDataset(IterableDataset):
        # Set identity transform on each Permutation so __getitems__ returns
        # the raw RecordBatch.  Stage 2 applies the real transform.
        permutations: list[Permutation] = []
-        initial_positions: list[int] = []
        for split_idx in my_splits:
            perm = Permutation.from_tables(
                self._table, self._perm_table, split=split_idx
@@ -349,20 +282,14 @@ class StreamingDataset(IterableDataset):
            if self._columns is not None:
                perm = perm.select_columns(self._columns)
            perm = perm.with_transform(lambda batch: batch)
-            start_pos = self._resume_positions.get(split_idx, self._resume_offset)
-            if start_pos > 0:
-                perm = perm.with_skip(start_pos)
-            initial_positions.append(start_pos)
+            if self._resume_offset > 0:
+                perm = perm.with_skip(self._resume_offset)
            permutations.append(perm)

        n = len(permutations)
        split_sizes = [perm.num_rows for perm in permutations]
        initial_offset = self._resume_offset
        local_consumed = [0] * n
-        # Permutation position each split has consumed through (absolute,
-        # i.e. counted from the start of the unskipped split).  Runs ahead of
-        # initial + local_consumed when rows are skipped.
-        pos_consumed = list(initial_positions)

        batch_size = self._read_batch_size
        max_prefetch = self._prefetch_batches
@@ -375,14 +302,12 @@ class StreamingDataset(IterableDataset):
            self._transform if self._transform is not None else Transforms.arrow2python
        )

-        # Per-split pipeline state.  Batches are paired with the absolute
-        # permutation position of their first row so that skipped rows can be
-        # accounted for in pos_consumed.
+        # Per-split pipeline state.
        fetch_head = [0] * n
-        io_pending = [deque() for _ in range(n)]  # (abs_start, Future[RecordBatch])
-        raw_batches = [deque() for _ in range(n)]  # (abs_start, RecordBatch)
-        tx_pending = [deque() for _ in range(n)]  # Future[list[(abs_pos, row)]]
-        cooked = [deque() for _ in range(n)]  # (abs_pos, row) ready to yield
+        io_pending = [deque() for _ in range(n)]  # Future[RecordBatch]
+        raw_batches = [deque() for _ in range(n)]  # RecordBatch — fetched, awaiting tx
+        tx_pending = [deque() for _ in range(n)]  # Future[list[Any]]
+        cooked = [deque() for _ in range(n)]  # rows ready to yield

        # Limit simultaneous transforms to transform_workers across all splits.
        tx_semaphore = threading.Semaphore(transform_workers)
@@ -405,8 +330,7 @@ class StreamingDataset(IterableDataset):
            fetch_head[i] += fetch
            perm_i = permutations[i]
            indices = list(range(start, start + fetch))
-            abs_start = initial_positions[i] + start
-            io_pending[i].append((abs_start, io_pool.submit(_io_call, perm_i, indices)))
+            io_pending[i].append(io_pool.submit(_io_call, perm_i, indices))

        def _fill_io(i: int) -> None:
            while len(io_pending[i]) < max_prefetch and fetch_head[i] < split_sizes[i]:
@@ -414,72 +338,15 @@ class StreamingDataset(IterableDataset):

        def _drain_io(i: int) -> None:
            """Move completed I/O futures into raw_batches non-blockingly."""
-            while io_pending[i] and io_pending[i][0][1].done():
-                abs_start, fut = io_pending[i].popleft()
-                raw_batches[i].append((abs_start, fut.result()))
+            while io_pending[i] and io_pending[i][0].done():
+                raw_batches[i].append(io_pending[i].popleft().result())

        # ── Stage 2 helpers ───────────────────────────────────────────────────

-        on_error = self._on_transform_error
-
-        def _should_skip(exc: Exception) -> bool:
-            if on_error == "raise":
-                return False
-            if callable(on_error):
-                return bool(on_error(exc))
-            return True  # "skip" or "warn"
-
-        def _check_row_count(rows: list, num_rows: int) -> None:
-            if len(rows) != num_rows:
-                raise ValueError(
-                    f"transform returned {len(rows)} rows for a batch of "
-                    f"{num_rows}; transforms must return exactly one output "
-                    "row per input row.  To drop bad rows, raise inside the "
-                    "transform and pass on_transform_error='skip'."
-                )
-
-        def _transform_isolated(abs_start, batch, batch_exc):
-            """Re-run the transform on single-row slices, dropping failures."""
-            out = []
-            skipped = 0
-            first_exc = None
-            for j in range(batch.num_rows):
-                try:
-                    rows = list(final_transform(batch.slice(j, 1)))
-                except Exception as exc:
-                    if not _should_skip(exc):
-                        raise
-                    skipped += 1
-                    if first_exc is None:
-                        first_exc = exc
-                    continue
-                _check_row_count(rows, 1)
-                out.append((abs_start + j, rows[0]))
-            self._rows_skipped += skipped
-            if skipped and on_error == "warn":
-                logger.warning(
-                    "Skipped %d of %d rows whose transform failed (first error: %r)",
-                    skipped,
-                    batch.num_rows,
-                    first_exc if first_exc is not None else batch_exc,
-                )
-            return out
-
-        def _transform_batch(abs_start, batch):
-            """Apply the transform, returning [(abs_pos, row), ...]."""
-            try:
-                rows = list(final_transform(batch))
-            except Exception as exc:
-                if not _should_skip(exc):
-                    raise
-                return _transform_isolated(abs_start, batch, exc)
-            _check_row_count(rows, batch.num_rows)
-            return [(abs_start + j, row) for j, row in enumerate(rows)]
-
-        def _tx_call_guarded(abs_start, batch):
+        def _tx_call_guarded(batch):
            try:
                t0 = time.perf_counter()
-                result = _transform_batch(abs_start, batch)
+                result = final_transform(batch)
                self._transform_time += time.perf_counter() - t0
                return result
            finally:
@@ -488,8 +355,8 @@ class StreamingDataset(IterableDataset):
        def _try_submit_tx(i: int) -> None:
            """Submit transforms for raw_batches[i] up to available capacity."""
            while raw_batches[i] and tx_semaphore.acquire(blocking=False):
-                abs_start, batch = raw_batches[i].popleft()
-                tx_pending[i].append(tx_pool.submit(_tx_call_guarded, abs_start, batch))
+                batch = raw_batches[i].popleft()
+                tx_pending[i].append(tx_pool.submit(_tx_call_guarded, batch))

        def _drain_tx(i: int) -> None:
            """Move completed transform futures into cooked non-blockingly."""
@@ -517,14 +384,11 @@ class StreamingDataset(IterableDataset):
                    # Acquire a transform slot (may block briefly if all
                    # transform_workers are busy with other splits).
                    tx_semaphore.acquire()
-                    abs_start, batch = raw_batches[i].popleft()
-                    tx_pending[i].append(
-                        tx_pool.submit(_tx_call_guarded, abs_start, batch)
-                    )
+                    batch = raw_batches[i].popleft()
+                    tx_pending[i].append(tx_pool.submit(_tx_call_guarded, batch))
                elif io_pending[i]:
                    # Block on the oldest in-flight I/O fetch.
-                    abs_start, fut = io_pending[i].popleft()
-                    raw_batches[i].append((abs_start, fut.result()))
+                    raw_batches[i].append(io_pending[i].popleft().result())
                    _advance(i)
                else:
                    break  # split exhausted
@@ -543,28 +407,15 @@ class StreamingDataset(IterableDataset):
                        _fill_io(i)

                    while True:
-                        # A cycle only runs if every split can still produce a
-                        # row.  Without skips all splits exhaust simultaneously
-                        # (equal split sizes + round-robin); when
-                        # on_transform_error drops rows a split can run dry
-                        # early, ending the epoch at the last complete cycle.
-                        # This check only sees splits owned by this rank/worker
-                        # (my_splits) — there is no cross-rank coordination, so
-                        # a different rank with fewer skipped rows keeps going;
-                        # see the on_transform_error docstring.
-                        exhausted = False
-                        for i in range(n):
-                            _ensure_cooked(i)
-                            if not cooked[i]:
-                                exhausted = True
-                                break
-                        if exhausted:
+                        # Stop when any split is exhausted (all exhaust
+                        # simultaneously: equal split sizes + round-robin).
+                        if any(local_consumed[i] >= split_sizes[i] for i in range(n)):
                            break

                        for i in range(n):
-                            pos, row = cooked[i].popleft()
+                            _ensure_cooked(i)
+                            row = cooked[i].popleft()
                            local_consumed[i] += 1
-                            pos_consumed[i] = pos + 1
                            _advance(i)

                            # After the last split in each cycle: update the
@@ -573,39 +424,21 @@ class StreamingDataset(IterableDataset):
                            # even when __iter__ runs in a worker process.
                            if i == n - 1:
                                self._resume_offset = initial_offset + local_consumed[i]
-                                for j, split_idx in enumerate(my_splits):
-                                    self._resume_positions[split_idx] = pos_consumed[j]
                                ws = self._worker_stats
                                ws[0] = sum(
                                    split_sizes[j] - fetch_head[j] for j in range(n)
                                )
                                ws[1] = sum(
-                                    batch.num_rows
-                                    for q in raw_batches
-                                    for _, batch in q
+                                    batch.num_rows for q in raw_batches for batch in q
                                )
                                ws[2] = sum(len(q) for q in cooked)
                                ws[3] = sum(local_consumed)
                                ws[4] = self._bytes_loaded
                                ws[5] = int(self._fetch_time * 1_000_000)
                                ws[6] = int(self._transform_time * 1_000_000)
-                                ws[7] = self._rows_skipped

                            yield row
                finally:
-                    # Final stats flush: the per-cycle write above never runs
-                    # when iteration ends mid-cycle (e.g. a split whose rows
-                    # were all skipped before completing a single cycle), so
-                    # counters like rows_skipped would otherwise be stale.
-                    ws = self._worker_stats
-                    ws[0] = sum(split_sizes[j] - fetch_head[j] for j in range(n))
-                    ws[1] = 0  # queue-depth properties document 0 when idle
-                    ws[2] = 0
-                    ws[3] = sum(local_consumed)
-                    ws[4] = self._bytes_loaded
-                    ws[5] = int(self._fetch_time * 1_000_000)
-                    ws[6] = int(self._transform_time * 1_000_000)
-                    ws[7] = self._rows_skipped
                    self._raw_batches_ref = None
                    self._cooked_ref = None
                    self._fetch_head_ref = None
@@ -659,7 +492,7 @@ class StreamingDataset(IterableDataset):
        batches.  Returns 0 when not iterating.
        """
        if self._raw_batches_ref is not None:
-            return sum(batch.num_rows for q in self._raw_batches_ref for _, batch in q)
+            return sum(batch.num_rows for q in self._raw_batches_ref for batch in q)
        return int(self._worker_stats[1])

    @property
@@ -689,19 +522,6 @@ class StreamingDataset(IterableDataset):
            )
        return int(self._worker_stats[0])

-    @property
-    def rows_skipped(self) -> int:
-        """Number of rows dropped because their transform raised an exception.
-
-        Only ever non-zero when ``on_transform_error`` is set to ``"skip"``,
-        ``"warn"``, or a callable that returned ``True``.  Accumulates across
-        multiple iterations of the same dataset instance and is never reset
-        automatically.
-        """
-        if self._raw_batches_ref is not None:
-            return self._rows_skipped
-        return int(self._worker_stats[7])
-
    @property
    def consumed_rows(self) -> int:
        """Number of rows already yielded to the caller across all splits.
@@ -767,27 +587,12 @@ class StreamingDataset(IterableDataset):
        every split has been consumed the same number of times (by the
        round-robin design), so the per-split count is a single uniform value
        that is identical across all ranks and DataLoader workers.
-
-        ``positions_consumed_per_split`` records how far into each split's
-        permutation iteration has advanced.  It only differs from
-        ``samples_consumed_per_split`` when ``on_transform_error`` skipped
-        rows, in which case entries are exact for the splits this instance
-        iterated and a lower bound (the sample count) for splits owned by
-        other ranks or workers.  Combine the state dicts from all ranks with
-        [merge_state_dicts][lancedb.streaming.StreamingDataset.merge_state_dicts]
-        to recover the exact value for every split before resuming on a
-        different topology.
        """
-        positions = [
-            self._resume_positions.get(split, self._resume_offset)
-            for split in range(self._num_splits)
-        ]
        return {
            "shuffle_seed": self._shuffle_seed,
            "num_splits": self._num_splits,
            "epoch": self._epoch,
            "samples_consumed_per_split": [self._resume_offset] * self._num_splits,
-            "positions_consumed_per_split": positions,
        }

    def load_state_dict(self, state: dict) -> None:
@@ -813,96 +618,3 @@ class StreamingDataset(IterableDataset):
            self._resume_offset = consumed[0] if consumed else 0
        else:
            self._resume_offset = int(consumed)
-        # Older checkpoints predate positions_consumed_per_split; without
-        # skipped rows positions equal sample counts, so falling back to
-        # _resume_offset (the .get default in __iter__) is exact.
-        positions = state.get("positions_consumed_per_split")
-        if positions is None:
-            self._resume_positions = {}
-        else:
-            self._resume_positions = {
-                split: int(pos) for split, pos in enumerate(positions)
-            }
-
-    @staticmethod
-    def merge_state_dicts(states: list[dict]) -> dict:
-        """Merge state dicts saved by different ranks into one exact state.
-
-        Only needed when ``on_transform_error`` skips rows in multi-rank
-        training: each rank then knows the exact permutation position only for
-        its own splits, and records a lower bound for the rest.  Because
-        exactly one rank owns each split, the elementwise maximum across all
-        ranks' ``positions_consumed_per_split`` recovers the exact position of
-        every split.  Without skipped rows every rank's state is already
-        identical and merging is a no-op.
-
-        Raises ``ValueError`` if the states are empty or were not produced by
-        the same run (mismatched seed, split count, epoch, or sample counts).
-
-        The merge is always all-to-all and topology-agnostic: collect the
-        ``state_dict()`` from every rank of the *previous* run into one list,
-        merge that whole list, and hand the identical merged result to every
-        rank of the *next* run — regardless of whether the rank count grew,
-        shrank, or stayed the same. There is no pairwise or subset merging
-        step, because each split's exact position is only known to whichever
-        rank owned that split, and the elementwise maximum needs every rank's
-        contribution to be correct.
-
-        For example, checkpointing 8 ranks and resuming on 4 (the same
-        pattern applies when growing, e.g. 4 ranks resuming on 8)::
-
-            states = [ds.state_dict() for ds in previous_run_datasets]  # 8
-            merged = StreamingDataset.merge_state_dicts(states)
-            for ds in resumed_datasets:  # now only 4 ranks
-                ds.load_state_dict(merged)  # same dict on every rank
-
-        The rank count on either side never affects the merge itself, since
-        ``merge_state_dicts`` only cares about the list of states it is
-        given.  Each split's position is recovered by elementwise maximum;
-        here rank 0 owned split 0 (and skipped two rows there) while rank 1
-        owned split 1 (and skipped one row):
-
-        >>> rank0 = {
-        ...     "shuffle_seed": 0, "num_splits": 2, "epoch": 0,
-        ...     "samples_consumed_per_split": [3, 3],
-        ...     "positions_consumed_per_split": [5, 3],
-        ... }
-        >>> rank1 = {
-        ...     "shuffle_seed": 0, "num_splits": 2, "epoch": 0,
-        ...     "samples_consumed_per_split": [3, 3],
-        ...     "positions_consumed_per_split": [3, 4],
-        ... }
-        >>> merged = StreamingDataset.merge_state_dicts([rank0, rank1])
-        >>> merged["positions_consumed_per_split"]
-        [5, 4]
-        """
-        if not states:
-            raise ValueError("merge_state_dicts requires at least one state dict")
-        first = states[0]
-        for state in states[1:]:
-            for key in ("shuffle_seed", "num_splits", "epoch"):
-                if state[key] != first[key]:
-                    raise ValueError(
-                        f"{key} mismatch across state dicts: "
-                        f"{state[key]} != {first[key]}"
-                    )
-            if (
-                state["samples_consumed_per_split"]
-                != first["samples_consumed_per_split"]
-            ):
-                raise ValueError(
-                    "samples_consumed_per_split mismatch across state dicts; "
-                    "state_dict() must be called at the same global step "
-                    "boundary on every rank"
-                )
-        merged = dict(first)
-        all_positions = [
-            state.get(
-                "positions_consumed_per_split", state["samples_consumed_per_split"]
-            )
-            for state in states
-        ]
-        merged["positions_consumed_per_split"] = [
-            max(per_split) for per_split in zip(*all_positions)
-        ]
-        return merged
@@ -108,11 +108,6 @@ def _should_push_down_query_table(
    return namespace_client is not None and "QueryTable" in pushdown_operations


-def _polars_predicate_pushdown_barrier(frame: Any) -> Any:
-    """Return a Polars frame unchanged while blocking predicate pushdown."""
-    return frame
-
-
 _MODEL_BACKED_TOKENIZER_PREFIXES = ("jieba", "lindera")
 _MODEL_BACKED_TOKENIZER_ERRORS = (
    "unknown base tokenizer",
@@ -869,18 +864,12 @@ class Table(ABC):
        """
        raise NotImplementedError

-    def to_polars(self, **kwargs) -> "pl.LazyFrame":
-        """Return the table as a Polars LazyFrame.
-
-        Note
-        ----
-        The Polars streaming engine is not supported because it does not currently
-        implement Python PyArrow dataset scans. Use the default engine when collecting
-        this LazyFrame.
+    def to_polars(self, **kwargs) -> "pl.DataFrame":
+        """Return the table as a polars.DataFrame.

        Returns
        -------
-        polars.LazyFrame
+        polars.DataFrame
        """
        raise NotImplementedError

@@ -2580,9 +2569,6 @@ class LanceTable(Table):
        2. Currently we've disabled push-down of the filters from polars
           because polars pushdown into pyarrow uses pyarrow compute
           expressions rather than SQl strings (which LanceDB supports)
-        3. The Polars streaming engine is not supported because it does not
-           currently implement Python PyArrow dataset scans. Use the default
-           engine when collecting this LazyFrame.

        Returns
        -------
@@ -2591,12 +2577,8 @@ class LanceTable(Table):
        from lancedb.integrations.pyarrow import PyarrowDatasetAdapter

        dataset = PyarrowDatasetAdapter(self)
-        # Polars 1.32's non-PyArrow callback path passes batch_size twice.  Keep
-        # the compatible PyArrow path, but block predicates because this adapter
-        # cannot translate PyArrow expressions into LanceDB filters.
-        return pl.scan_pyarrow_dataset(dataset, batch_size=batch_size).map_batches(
-            _polars_predicate_pushdown_barrier,
-            predicate_pushdown=False,
+        return pl.scan_pyarrow_dataset(
+            dataset, allow_pyarrow_filter=False, batch_size=batch_size
        )

    # New unified API overload
@@ -3976,28 +3958,6 @@ class LanceTable(Table):
        [`AsyncTable.get_lsm_write_spec`][lancedb.AsyncTable.get_lsm_write_spec]."""
        return LOOP.run(self._table.get_lsm_write_spec())

-    def checkpoint_lsm(self) -> None:
-        """Synchronous version of
-        [`AsyncTable.checkpoint_lsm`][lancedb.AsyncTable.checkpoint_lsm]."""
-        return LOOP.run(self._table.checkpoint_lsm())
-
-    def flush_lsm(self) -> None:
-        """Synchronous version of
-        [`AsyncTable.flush_lsm`][lancedb.AsyncTable.flush_lsm]."""
-        return LOOP.run(self._table.flush_lsm())
-
-    def compact_lsm(self) -> None:
-        """Synchronous version of
-        [`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm]."""
-        return LOOP.run(self._table.compact_lsm())
-
-    def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]:
-        """Synchronous version of
-        [`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats]."""
-        return LOOP.run(
-            self._table.get_lsm_stats(include_generation_rows=include_generation_rows)
-        )
-
    def close_lsm_writers(self) -> None:
        """Close cached MemWAL shard writers. See
        [`AsyncTable.close_lsm_writers`][lancedb.AsyncTable.close_lsm_writers]."""
@@ -4676,13 +4636,6 @@ class AsyncTable:
        via [`set_unenforced_primary_key`]; bucket sharding additionally
        requires it to be the single column being bucketed.

-        By default the MemWAL maintains every index on the table, resolved
-        here — a snapshot, so an index created afterwards needs the spec unset
-        and set again. This fails if one cannot be maintained; name the set
-        with ``with_maintained_indexes`` to install anyway. That pins an exact
-        set (a still-building index is rejected, not omitted); ``[]`` maintains
-        none.
-
        Parameters
        ----------
        spec : LsmWriteSpec
@@ -4709,73 +4662,12 @@ class AsyncTable:

        Returns ``None`` when the MemWAL LSM write path is not enabled (no
        spec has been set, or it was removed with `unset_lsm_write_spec`).
-        The returned spec mirrors what was passed to `set_lsm_write_spec`,
-        except that ``maintained_indexes`` always reports the concrete list
-        resolved when the spec was set — ``None`` never round-trips.
+        The returned spec — including its ``maintained_indexes`` and
+        ``writer_config_defaults`` — mirrors what was passed to
+        `set_lsm_write_spec`.
        """
        return await self._inner.get_lsm_write_spec()

-    async def checkpoint_lsm(self) -> None:
-        """Converge this table's LSM write path into its base table.
-
-        One flush, sealing every memtable into L0, then compaction triggers
-        until every generation that existed at that moment has reached base.
-        The loop runs client-side, reading progress from ``get_lsm_stats``.
-
-        Best-effort: generations created *while* it runs are deliberately not
-        waited on, which is what lets it terminate on a table taking writes.
-        Idempotent and safe on a cadence.
-
-        There is no deadline, and the caller owns that. It returns when the
-        target generations are gone, raises on a terminal server fault, and
-        otherwise waits however long the server takes. A slow table and a
-        stuck one are the same picture from the client: the compactor pool is
-        shared across every table on the node, so a checkpoint queued behind
-        unrelated work looks exactly like one that is merging. Wrap this in
-        ``asyncio.wait_for`` for a wall-clock bound; abandoning it partway
-        costs nothing.
-        """
-        return await self._inner.checkpoint_lsm()
-
-    async def flush_lsm(self) -> None:
-        """Seal every bucket's active memtable into L0.
-
-        Does not touch the base table — moving L0 into base is
-        `compact_lsm`. On a node that has not claimed this table, this claims
-        it and replays its WAL log first.
-        """
-        return await self._inner.flush_lsm()
-
-    async def compact_lsm(self) -> None:
-        """Trigger a background L0 to base compaction pass per bucket.
-
-        Returns once the passes are dispatched, not once they finish: watch
-        ``get_lsm_stats`` for progress, or use ``checkpoint_lsm`` to loop
-        until the current L0 has reached base.
-        """
-        return await self._inner.compact_lsm()
-
-    async def get_lsm_stats(
-        self, *, include_generation_rows: bool = False
-    ) -> Optional[dict]:
-        """Read live per-bucket LSM state.
-
-        Answers "how far behind is my fresh tier", "which bucket is hot", and
-        "why is my fresh-tier vector search brute-force". Mutates no table
-        state, though on a node that has not claimed this table it claims it,
-        exactly as a read would.
-
-        Returns ``None`` only when the LSM write path is not enabled.
-
-        Parameters
-        ----------
-        include_generation_rows
-            Report a row count per L0 generation. Off by default: each count
-            opens an uncached Lance dataset, and ``checkpoint_lsm`` polls this
-            needing only generation numbers.
-        """
-        return await self._inner.get_lsm_stats(include_generation_rows)
-
    async def close_lsm_writers(self) -> None:
        """Drain and close any cached MemWAL shard writers for this table.

@@ -6341,9 +6233,7 @@ class TableStatistics:
    Attributes
    ----------
    total_bytes: int
-        The total size, in bytes, of the table's data files, index files, and
-        overlay files. Read from the manifest, so this excludes deletion files
-        and manifests.
+        The total number of bytes in the table.
    num_rows: int
        The total number of rows in the table.
    num_indices: int
@@ -6,7 +6,6 @@ import inspect
 import re
 import sys
 from datetime import timedelta
-from importlib import resources
 import os
 from types import SimpleNamespace

@@ -19,10 +18,6 @@ from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
 from lancedb.pydantic import LanceModel, Vector


-def test_package_includes_pep_561_marker():
-    assert resources.files(lancedb).joinpath("py.typed").is_file()
-
-
 def test_basic(tmp_path):
    db = lancedb.connect(tmp_path)

@@ -755,7 +750,8 @@ def test_delete_table(tmp_db: lancedb.DBConnection):
    assert tmp_db.table_names() == []


-def test_drop_table_async(tmp_db: lancedb.DBConnection):
+@pytest.mark.asyncio
+async def test_delete_table_async(tmp_db: lancedb.DBConnection):
    data = pd.DataFrame(
        {
            "vector": [[3.1, 4.1], [5.9, 26.5]],
@@ -771,10 +767,7 @@ def test_drop_table_async(tmp_db: lancedb.DBConnection):

    assert tmp_db.table_names() == ["test"]

-    job = tmp_db.drop_table_async("test")
-    assert job.id is None
-    assert job.status() == "finished"
-    job.wait()
+    tmp_db.drop_table("test")
    assert tmp_db.table_names() == []

    tmp_db.create_table("test", data=data)
@@ -783,17 +776,6 @@ def test_drop_table_async(tmp_db: lancedb.DBConnection):
    tmp_db.drop_table("does_not_exist", ignore_missing=True)


-@pytest.mark.asyncio
-async def test_drop_table_async_connection(tmp_db_async: lancedb.AsyncConnection):
-    await tmp_db_async.create_table("test", data=pa.table({"id": [1, 2]}))
-
-    job = await tmp_db_async.drop_table_async("test")
-    assert job.id is None
-    assert await job.status() == "finished"
-    await job.wait()
-    assert await tmp_db_async.table_names() == []
-
-
 def test_drop_database(tmp_db: lancedb.DBConnection):
    data = pd.DataFrame(
        {
@@ -1456,408 +1456,6 @@ def test_shuffle_clump_size_yields_all_rows(lance_table):
    )


-# ---------------------------------------------------------------------------
-# on_transform_error tests
-# ---------------------------------------------------------------------------
-
-
-class BadRowError(ValueError):
-    """Raised by the failing transforms below when a batch contains a bad id."""
-
-
-def _failing_transform(bad_ids: set):
-    """A transform that raises BadRowError whenever the batch has a bad id.
-
-    Raises on the full batch and on any single-row slice containing a bad id,
-    so per-row isolation drops exactly the bad rows.
-    """
-
-    def transform(batch: pa.RecordBatch) -> list:
-        ids = batch.column("id").to_pylist()
-        bad = sorted(set(ids) & bad_ids)
-        if bad:
-            raise BadRowError(f"bad ids in batch: {bad}")
-        return [{"id": i} for i in ids]
-
-    return transform
-
-
-def _sequential_split_members(table) -> list[list[int]]:
-    """Return each split's ids in yield order for shuffle=False.
-
-    With a single rank and no workers the round-robin yields one row per split
-    per cycle, so item k of a clean run belongs to split k % NUM_SPLITS.
-    """
-    ds = StreamingDataset(table, num_splits=NUM_SPLITS, shuffle=False)
-    members: list[list[int]] = [[] for _ in range(NUM_SPLITS)]
-    for k, row in enumerate(ds):
-        members[k % NUM_SPLITS].append(row["id"])
-    return members
-
-
-def test_on_transform_error_default_raises(lance_table):
-    """By default a transform exception propagates and aborts iteration."""
-    ds = StreamingDataset(
-        lance_table,
-        num_splits=NUM_SPLITS,
-        shuffle_seed=SHUFFLE_SEED,
-        transform=_failing_transform({7}),
-    )
-    with pytest.raises(BadRowError):
-        list(ds)
-
-
-def test_on_transform_error_invalid_value(lance_table):
-    with pytest.raises(ValueError, match="on_transform_error"):
-        StreamingDataset(lance_table, num_splits=NUM_SPLITS, on_transform_error="bogus")
-
-
-def test_on_transform_error_skip_drops_bad_rows(lance_table):
-    """With one bad row per split, 'skip' yields every good row exactly once
-    and counts the dropped rows in rows_skipped."""
-    members = _sequential_split_members(lance_table)
-    bad_ids = {members[i][4] for i in range(NUM_SPLITS)}
-
-    ds = StreamingDataset(
-        lance_table,
-        num_splits=NUM_SPLITS,
-        shuffle=False,
-        transform=_failing_transform(bad_ids),
-        on_transform_error="skip",
-    )
-    assert ds.rows_skipped == 0
-
-    ids = [row["id"] for row in ds]
-
-    assert sorted(ids) == sorted(set(range(NUM_ROWS)) - bad_ids)
-    assert ds.rows_skipped == NUM_SPLITS
-
-
-def test_on_transform_error_skip_uneven_ends_at_last_complete_cycle(lance_table):
-    """When one split loses more rows than the others, the epoch ends at the
-    last cycle where every split still has a row — no crash, no bad rows, and
-    every step remains one sample per split."""
-    members = _sequential_split_members(lance_table)
-    bad_ids = set(members[0][:3])  # all 3 bad rows in split 0
-
-    ds = StreamingDataset(
-        lance_table,
-        num_splits=NUM_SPLITS,
-        shuffle=False,
-        transform=_failing_transform(bad_ids),
-        on_transform_error="skip",
-    )
-    items = [row["id"] for row in ds]
-
-    rows_per_split = NUM_ROWS // NUM_SPLITS
-    expected_cycles = rows_per_split - len(bad_ids)
-    assert len(items) == expected_cycles * NUM_SPLITS
-    assert len(set(items)) == len(items), "duplicate samples yielded"
-    assert not set(items) & bad_ids, "a bad row was yielded"
-    # Split 0 contributed exactly its surviving rows, in order, one per cycle.
-    survivors = [i for i in members[0] if i not in bad_ids]
-    assert items[0::NUM_SPLITS] == survivors[:expected_cycles]
-
-
-def test_on_transform_error_warn_logs(lance_table, caplog):
-    """'warn' skips like 'skip' but logs a warning for the failing batch."""
-    members = _sequential_split_members(lance_table)
-    bad_ids = {members[i][3] for i in range(NUM_SPLITS)}
-
-    ds = StreamingDataset(
-        lance_table,
-        num_splits=NUM_SPLITS,
-        shuffle=False,
-        transform=_failing_transform(bad_ids),
-        on_transform_error="warn",
-    )
-    with caplog.at_level(logging.WARNING, logger="lancedb.streaming"):
-        items = list(ds)
-
-    assert len(items) == NUM_ROWS - NUM_SPLITS
-    assert ds.rows_skipped == NUM_SPLITS
-    assert "Skipped" in caplog.text
-    assert "BadRowError" in caplog.text
-
-
-def test_on_transform_error_callable_selective(lance_table):
-    """A callable handler can skip expected errors and re-raise the rest."""
-    members = _sequential_split_members(lance_table)
-    bad_ids = {members[i][0] for i in range(NUM_SPLITS)}
-
-    handled: list[Exception] = []
-
-    def handler(exc: Exception) -> bool:
-        handled.append(exc)
-        return isinstance(exc, BadRowError)
-
-    ds = StreamingDataset(
-        lance_table,
-        num_splits=NUM_SPLITS,
-        shuffle=False,
-        transform=_failing_transform(bad_ids),
-        on_transform_error=handler,
-    )
-    items = list(ds)
-    assert len(items) == NUM_ROWS - NUM_SPLITS
-    assert handled and all(isinstance(exc, BadRowError) for exc in handled)
-
-    def broken_transform(batch: pa.RecordBatch) -> list:
-        raise TypeError("boom")
-
-    ds2 = StreamingDataset(
-        lance_table,
-        num_splits=NUM_SPLITS,
-        shuffle=False,
-        transform=broken_transform,
-        on_transform_error=handler,
-    )
-    with pytest.raises(TypeError, match="boom"):
-        list(ds2)
-
-
-def test_transform_wrong_row_count_raises(lance_table):
-    """A transform that returns the wrong number of rows is an error even with
-    on_transform_error='skip' — silent shrinkage would corrupt accounting."""
-
-    def drops_rows(batch: pa.RecordBatch) -> list:
-        return batch.column("id").to_pylist()[:-1]
-
-    ds = StreamingDataset(
-        lance_table,
-        num_splits=NUM_SPLITS,
-        shuffle_seed=SHUFFLE_SEED,
-        transform=drops_rows,
-        on_transform_error="skip",
-    )
-    with pytest.raises(ValueError, match="one output row per input row"):
-        list(ds)
-
-
-def test_skip_deterministic_across_runs(lance_table):
-    """With a fixed seed, skipping produces the identical sample sequence on
-    every run — skips are data-dependent, not run-dependent."""
-    bad_ids = {5, 17, 46}
-
-    def run() -> tuple[list[int], int]:
-        ds = StreamingDataset(
-            lance_table,
-            num_splits=NUM_SPLITS,
-            shuffle_seed=SHUFFLE_SEED,
-            transform=_failing_transform(bad_ids),
-            on_transform_error="skip",
-        )
-        return [row["id"] for row in ds], ds.rows_skipped
-
-    ids_a, skipped_a = run()
-    ids_b, skipped_b = run()
-    assert ids_a == ids_b
-    assert skipped_a == skipped_b
-    assert not set(ids_a) & bad_ids
-
-
-def test_skip_elastic_det_across_world_sizes(lance_table):
-    """With equal bad-row counts per split, skipping preserves the full
-    elastic-determinism guarantee: identical global batches at every step for
-    every compatible world_size."""
-    members = _sequential_split_members(lance_table)
-    bad_ids = {members[i][6] for i in range(NUM_SPLITS)}
-
-    def collect(world_size: int) -> list[frozenset[int]]:
-        micro = GLOBAL_BATCH_SIZE // world_size
-        iters = [
-            iter(
-                StreamingDataset(
-                    lance_table,
-                    num_splits=NUM_SPLITS,
-                    shuffle=False,
-                    rank=rank,
-                    world_size=world_size,
-                    transform=_failing_transform(bad_ids),
-                    on_transform_error="skip",
-                )
-            )
-            for rank in range(world_size)
-        ]
-        _STOP = object()
-        batches: list[frozenset[int]] = []
-        while True:
-            step_samples: set[int] = set()
-            exhausted = 0
-            for it in iters:
-                for _ in range(micro):
-                    val = next(it, _STOP)
-                    if val is _STOP:
-                        exhausted += 1
-                        break
-                    step_samples.add(val["id"])
-            if exhausted == len(iters):
-                break
-            assert exhausted == 0, (
-                "Rank iterators exhausted at different steps despite equal "
-                "bad-row counts per split"
-            )
-            batches.append(frozenset(step_samples))
-        return batches
-
-    reference = collect(1)
-    assert len(reference) == NUM_ROWS // NUM_SPLITS - 1
-    for ws in (2, 3, 4):
-        assert collect(ws) == reference, f"world_size={ws} diverged"
-
-
-def test_resumability_with_skips_same_topology(lance_table):
-    """Checkpointing mid-epoch with skipped rows resumes exactly: no sample
-    repeated, no sample lost, skipped rows stay skipped."""
-    members = _sequential_split_members(lance_table)
-    # Uneven skips: positions diverge across splits (2 bad in split 0, 1 in
-    # split 5), which only a position-based checkpoint can resume exactly.
-    bad_ids = {members[0][2], members[0][3], members[5][7]}
-    kwargs = dict(
-        num_splits=NUM_SPLITS,
-        shuffle=False,
-        transform=_failing_transform(bad_ids),
-        on_transform_error="skip",
-    )
-
-    reference = [row["id"] for row in StreamingDataset(lance_table, **kwargs)]
-    rows_per_split = NUM_ROWS // NUM_SPLITS
-    assert len(reference) == (rows_per_split - 2) * NUM_SPLITS
-
-    steps = 3
-    ds = StreamingDataset(lance_table, **kwargs)
-    it = iter(ds)
-    consumed = [next(it)["id"] for _ in range(steps * NUM_SPLITS)]
-    checkpoint = ds.state_dict()
-    it.close()
-
-    # Split 0 skipped positions 2 and 3 within its first 3 yields; split 5's
-    # bad row is beyond the checkpoint.  Everything else is at 3 = the sample
-    # count.
-    positions = checkpoint["positions_consumed_per_split"]
-    assert positions[0] == 5
-    assert positions[1:] == [3] * (NUM_SPLITS - 1)
-    assert checkpoint["samples_consumed_per_split"] == [3] * NUM_SPLITS
-
-    ds2 = StreamingDataset(lance_table, **kwargs)
-    ds2.load_state_dict(checkpoint)
-    resumed = [row["id"] for row in ds2]
-
-    assert consumed == reference[: steps * NUM_SPLITS]
-    assert resumed == reference[steps * NUM_SPLITS :]
-
-
-def test_resumability_with_skips_elastic_merge(lance_table):
-    """Elastic resume with skips: each rank's checkpoint knows exact positions
-    only for its own splits; merge_state_dicts recovers the global state, and
-    a run on a different world_size continues exactly."""
-    members = _sequential_split_members(lance_table)
-    # Bad rows early in split 0 (rank 0) and split 6 (rank 1 of a ws=2 run) so
-    # both ranks' position vectors diverge before the checkpoint.
-    bad_ids = {members[0][0], members[0][2], members[6][1]}
-    kwargs = dict(
-        num_splits=NUM_SPLITS,
-        shuffle=False,
-        transform=_failing_transform(bad_ids),
-        on_transform_error="skip",
-    )
-
-    reference = [row["id"] for row in StreamingDataset(lance_table, **kwargs)]
-
-    steps = 3
-    world_size = 2
-    micro = GLOBAL_BATCH_SIZE // world_size
-    datasets = [
-        StreamingDataset(lance_table, rank=rank, world_size=world_size, **kwargs)
-        for rank in range(world_size)
-    ]
-    iters = [iter(ds) for ds in datasets]
-    seen: list[frozenset[int]] = []
-    for _ in range(steps):
-        step_samples = set()
-        for it in iters:
-            for _ in range(micro):
-                step_samples.add(next(it)["id"])
-        seen.append(frozenset(step_samples))
-    states = [ds.state_dict() for ds in datasets]
-    for it in iters:
-        it.close()
-
-    merged = StreamingDataset.merge_state_dicts(states)
-    expected_positions = [3] * NUM_SPLITS
-    expected_positions[0] = 5  # skipped positions 0 and 2
-    expected_positions[6] = 4  # skipped position 1
-    assert merged["positions_consumed_per_split"] == expected_positions
-
-    # The first 3 global batches match the world_size=1 reference.
-    ref_batches = [
-        frozenset(reference[s * NUM_SPLITS : (s + 1) * NUM_SPLITS])
-        for s in range(len(reference) // NUM_SPLITS)
-    ]
-    assert seen == ref_batches[:steps]
-
-    # Resume on world_size=1 from the merged state.
-    ds_resume = StreamingDataset(lance_table, **kwargs)
-    ds_resume.load_state_dict(merged)
-    resumed = [row["id"] for row in ds_resume]
-    assert resumed == reference[steps * NUM_SPLITS :]
-
-
-def test_rows_skipped_flushed_when_split_entirely_bad(lance_table):
-    """A split whose rows all fail never completes a cycle, so the epoch ends
-    immediately — but rows_skipped must still report the drops after the
-    iterator exits (the shared-memory counter is flushed on exhaustion)."""
-    members = _sequential_split_members(lance_table)
-    bad_ids = set(members[0])  # every row of split 0 is bad
-
-    ds = StreamingDataset(
-        lance_table,
-        num_splits=NUM_SPLITS,
-        shuffle=False,
-        transform=_failing_transform(bad_ids),
-        on_transform_error="skip",
-    )
-    assert list(ds) == []
-    assert ds.rows_skipped == len(bad_ids)
-
-
-def test_merge_state_dicts_validates_consistency(lance_table):
-    ds = StreamingDataset(lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED)
-    state = ds.state_dict()
-    other = dict(state, shuffle_seed=SHUFFLE_SEED + 1)
-    with pytest.raises(ValueError, match="shuffle_seed mismatch"):
-        StreamingDataset.merge_state_dicts([state, other])
-    with pytest.raises(ValueError, match="at least one"):
-        StreamingDataset.merge_state_dicts([])
-
-
-def test_load_state_dict_without_positions_key(lance_table):
-    """Checkpoints from before positions_consumed_per_split existed still
-    resume exactly (positions equal sample counts when nothing is skipped)."""
-    reference = [
-        row["id"]
-        for row in StreamingDataset(
-            lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED
-        )
-    ]
-
-    steps = 4
-    ds = StreamingDataset(lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED)
-    it = iter(ds)
-    for _ in range(steps * NUM_SPLITS):
-        next(it)
-    checkpoint = ds.state_dict()
-    it.close()
-    del checkpoint["positions_consumed_per_split"]
-
-    ds2 = StreamingDataset(
-        lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED
-    )
-    ds2.load_state_dict(checkpoint)
-    resumed = [row["id"] for row in ds2]
-    assert resumed == reference[steps * NUM_SPLITS :]
-
-
 def test_num_splits_defaults_to_world_size(lance_table):
    """Omitting num_splits gives world_size splits (one per rank)."""
    ds = StreamingDataset(
@@ -631,23 +631,3 @@ def test_url_retrieve_downloads_image():
    image_bytes = url_retrieve(image_url)
    img = Image.open(io.BytesIO(image_bytes))
    assert img.size[0] > 0 and img.size[1] > 0
-
-
-def test_jina_generate_image_input_dict_local_path(tmp_path):
-    """
-    JinaEmbeddings._generate_image_input_dict must accept a local image path
-    (str or Path), not just bytes. Previously it crashed with
-    `AttributeError: 'function' object has no attribute 'urlparse'` on any
-    str/Path input because it called `urlparse.urlparse(image)` instead of
-    `urlparse(image)` (urlparse was imported as a function, not a module).
-    """
-    Image = pytest.importorskip("PIL.Image")
-    from lancedb.embeddings.jinaai import JinaEmbeddings
-
-    image_path = tmp_path / "test.png"
-    Image.new("RGB", (4, 4), color="red").save(image_path, format="PNG")
-
-    for image in (str(image_path), image_path):
-        image_dict = JinaEmbeddings._generate_image_input_dict(image)
-        assert "image" in image_dict
-        assert isinstance(image_dict["image"], str) and len(image_dict["image"]) > 0
@@ -83,9 +83,7 @@ def test_lsm_write_spec_repr():
    assert s.spec_type == "bucket"
    assert s.column == "id"
    assert s.num_buckets == 4
-    # A fresh spec defers its maintained set to install time.
-    assert s.maintained_indexes is None
-    assert s.with_maintained_indexes([]).maintained_indexes == []
+    assert s.maintained_indexes == []
    assert "bucket" in repr(s)
    assert "id" in repr(s)
    assert "4" in repr(s)
@@ -171,23 +169,18 @@ def test_get_lsm_write_spec(tmp_path):
    table.unset_lsm_write_spec()
    assert table.get_lsm_write_spec() is None

-    # Identity round-trips (column recovered from the schema). Leaving the
-    # maintained set to be inferred picks up the index on the table, so the
-    # spec reads back naming it rather than as "infer".
+    # Identity round-trips (column recovered from the schema).
    table.set_lsm_write_spec(LsmWriteSpec.identity("id"))
    spec = table.get_lsm_write_spec()
    assert spec.spec_type == "identity"
    assert spec.column == "id"
-    assert spec.maintained_indexes == [idx_name]
    table.unset_lsm_write_spec()

-    # Unsharded round-trips (no routing column). Opting out is distinct from
-    # the inferred default.
-    table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([]))
+    # Unsharded round-trips (no routing column).
+    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
    spec = table.get_lsm_write_spec()
    assert spec.spec_type == "unsharded"
    assert spec.column is None
-    assert spec.maintained_indexes == []


@pytest.mark.asyncio
@@ -544,7 +544,7 @@ def test_lsm_read_fts_unmaintained_index_errors(tmp_path):
    table.create_index("text", config=FTS())
    # No maintained indexes: the active memtable FTS arm cannot serve un-compacted
    # docs, so the search would silently omit them — reject instead.
-    table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([]))
+    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
    with pytest.raises(Exception, match="maintained"):
        table.search("fox", query_type="fts", fts_columns="text").to_arrow()

@@ -631,7 +631,7 @@ def test_lsm_read_vector_unmaintained_index_errors(tmp_path):
    )
    # Spec with NO maintained indexes: the base vector index's catch-up is untracked,
    # so the scanner rejects rather than risk dropping compacted-but-unindexed rows.
-    table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([]))
+    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
    with pytest.raises(Exception, match="maintained"):
        table.search([1.0] * VECTOR_DIM).to_arrow()

@@ -415,17 +415,6 @@ def test_nullable_vector():
    assert schema == pa.schema([pa.field("vec", pa.list_(pa.float32(), 16), True)])


-def test_bare_vector_raises_clear_error():
-    namespace = {
-        "__name__": "test_model_without_pyarrow",
-        "LanceModel": LanceModel,
-        "Vector": Vector,
-    }
-
-    with pytest.raises(TypeError, match=r"Vector must be parameterized.*Vector\(128\)"):
-        exec("class TestModel(LanceModel):\n    vector: Vector", namespace)
-
-
 def test_fixed_size_list_field():
    class TestModel(pydantic.BaseModel):
        vec: Vector(16)
@@ -570,15 +570,6 @@ def test_query_builder(table):
    assert all(np.array(rs[0]["vector"]) == [1, 2])


-def test_query_multiple_vectors(table):
-    results = table.search([np.array([1, 2]), np.array([4, 5])]).limit(1).to_list()
-
-    assert len(results) == 2
-    results_by_query = {result["query_index"]: result for result in results}
-    assert results_by_query[0]["id"] == 1
-    assert results_by_query[1]["id"] == 2
-
-
 def test_with_row_id(table: lancedb.table.Table):
    rs = table.search().with_row_id(True).to_arrow()
    assert "_rowid" in rs.column_names
@@ -929,7 +929,6 @@ def test_polars(mem_db: DBConnection):

    # enter table to polars dataframe
    result = table.to_polars()
-    assert isinstance(result, pl.LazyFrame)
    assert np.allclose(result.collect()["vector"].to_list(), data["vector"])

    # make sure filtering isn't broken
@@ -1846,27 +1845,6 @@ def test_add_with_empty_fixed_size_list_drops_bad_rows(mem_db: DBConnection):
    assert np.allclose(data["embedding"].to_pylist()[0], np.array([0.1] * 16))


-def test_add_nullable_fixed_size_list_with_none(mem_db: DBConnection):
-    """Regression test for issue #2340."""
-    table = mem_db.create_table(
-        "test_nullable_fixed_size_list",
-        schema=pa.schema(
-            [
-                pa.field("id", pa.string()),
-                pa.field("feature", pa.list_(pa.float32(), 256)),
-                pa.field("tags", pa.list_(pa.string())),
-            ]
-        ),
-    )
-
-    table.add([{"id": "1", "feature": None, "tags": ["tag1", "tag2"]}])
-
-    result = table.to_arrow()
-    assert result.to_pylist() == [
-        {"id": "1", "feature": None, "tags": ["tag1", "tag2"]}
-    ]
-
-
 def test_add_nullable_struct_with_none(mem_db: DBConnection):
    """Regression test for issue #2654: a nullable struct column whose
    first batch contains only None values must not crash in
@@ -2218,45 +2196,6 @@ def test_merge(tmp_db: DBConnection, tmp_path):
    table.merge(other_dataset, left_on="id")


-@pytest.mark.parametrize("storage_version", ["legacy", "stable"])
-def test_search_after_merge(tmp_path, storage_version):
-    pytest.importorskip("lance")
-    pd = pytest.importorskip("pandas")
-
-    db = lancedb.connect(
-        tmp_path,
-        storage_options={"new_table_data_storage_version": storage_version},
-    )
-    rng = np.random.default_rng(42)
-    row_count = 512
-    vectors = rng.standard_normal((row_count, 8)).astype(np.float32)
-    table = db.create_table(
-        "search_after_merge",
-        data=pd.DataFrame(
-            {
-                "id": [str(i) for i in range(row_count)],
-                "vector": list(vectors),
-            }
-        ),
-    )
-    table.create_index("vector", config=IvfPq(num_partitions=1, num_sub_vectors=2))
-
-    links = pd.DataFrame(
-        {
-            "id": [str(i) for i in range(row_count // 2)],
-            "link": [f"https://example.com/{i}" for i in range(row_count // 2)],
-        }
-    )
-    table.merge(links, left_on="id")
-
-    query = table.search(vectors[-1]).refine_factor(50).limit(10)
-    assert "ANN" in query.explain_plan(verbose=True)
-
-    result = query.to_arrow()
-    links_by_id = dict(zip(result["id"].to_pylist(), result["link"].to_pylist()))
-    assert links_by_id[str(row_count - 1)] is None
-
-
 def test_delete(mem_db: DBConnection):
    table = mem_db.create_table(
        "my_table",
@@ -2799,40 +2738,15 @@ def test_create_with_embedding_function(mem_db: DBConnection):
    assert actual == expected


-def test_create_f16_table_from_arrow_data(mem_db: DBConnection):
-    dimension = 32
-    num_rows = 512
-    values = pa.array(
-        np.random.default_rng(42)
-        .standard_normal(num_rows * dimension)
-        .astype(np.float16)
-    )
-    df = pa.table(
-        {
-            "text": [f"s-{i}" for i in range(num_rows)],
-            "vector": pa.FixedSizeListArray.from_arrays(values, dimension),
-        }
-    )
-    table = mem_db.create_table("f16_tbl", data=df)
-    assert table.schema.field("vector").type == pa.list_(pa.float16(), dimension)
-    table.create_index(num_partitions=2, num_sub_vectors=2)
-
-    query = df["vector"][2].as_py()
-    expected = table.search(query).limit(2).to_arrow()
-
-    assert "s-2" in expected["text"].to_pylist()
-
-
 def test_create_f16_table(mem_db: DBConnection):
    class MyTable(LanceModel):
        text: str
        vector: Vector(32, value_type=pa.float16())

-    rng = np.random.default_rng(42)
    df = pa.table(
        {
            "text": [f"s-{i}" for i in range(512)],
-            "vector": [rng.standard_normal(32).astype(np.float16) for _ in range(512)],
+            "vector": [np.random.randn(32).astype(np.float16) for _ in range(512)],
        }
    )
    table = mem_db.create_table(
@@ -3713,8 +3627,7 @@ def test_stats(mem_db: DBConnection):
    stats = table.stats()
    print(f"{stats=}")
    assert stats == {
-        # Full on-disk size of the data file, footer and metadata included.
-        "total_bytes": 633,
+        "total_bytes": 60,
        "num_rows": 2,
        "num_indices": 0,
        "fragment_stats": {
@@ -3732,13 +3645,6 @@ def test_stats(mem_db: DBConnection):
        },
    }

-    # Index files count toward total_bytes too (only deletion files and
-    # manifests are excluded).
-    table.create_index("id", config=BTree())
-    stats_with_index = table.stats()
-    assert stats_with_index["num_indices"] == 1
-    assert stats_with_index["total_bytes"] > stats["total_bytes"]
-

 def test_create_table_empty_list_with_schema(mem_db: DBConnection):
    """Test creating table with empty list data and schema
@@ -1,15 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-from typing import assert_type
-
-import lancedb
-from lancedb import AsyncConnection, DBConnection
-
-
-def check_connect_type() -> None:
-    assert_type(lancedb.connect("memory://"), DBConnection)
-
-
-async def check_connect_async_type() -> None:
-    assert_type(await lancedb.connect_async("memory://"), AsyncConnection)
@@ -346,23 +346,6 @@ impl Connection {
        })
    }

-    #[pyo3(signature = (name, namespace_path=None))]
-    pub fn drop_table_async(
-        self_: PyRef<'_, Self>,
-        name: String,
-        namespace_path: Option<Vec<String>>,
-    ) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.get_inner()?.clone();
-        let ns_path = namespace_path.unwrap_or_default();
-        future_into_py(self_.py(), async move {
-            inner
-                .drop_table_async(name, &ns_path)
-                .await
-                .infer_error()
-                .map(crate::job::Job::new)
-        })
-    }
-
    #[pyo3(signature = (namespace_path=None,))]
    pub fn drop_all_tables(
        self_: PyRef<'_, Self>,
@@ -289,7 +289,7 @@ struct IvfHnswFlatParams {
    target_partition_size: Option<u32>,
 }

-#[pyclass(module = "lancedb._lancedb", get_all)]
+#[pyclass(get_all)]
 /// A description of an index currently configured on a column
 pub struct IndexConfig {
    /// The type of the index
@@ -11,7 +11,7 @@ use pyo3::{PyResult, pyclass, pymethods};
 /// Sessions allow you to configure cache sizes for index and metadata caches,
 /// which can significantly impact memory use and performance. They can
 /// also be re-used across multiple connections to share the same cache state.
-#[pyclass(module = "lancedb._lancedb", from_py_object)]
+#[pyclass(from_py_object)]
 #[derive(Clone)]
 pub struct Session {
    pub(crate) inner: Arc<LanceSession>,
@@ -28,72 +28,11 @@ use pyo3::{
    Bound, FromPyObject, Py, PyAny, PyRef, PyResult, Python,
    exceptions::{PyRuntimeError, PyValueError},
    pyclass, pyfunction, pymethods,
-    types::{IntoPyDict, PyAnyMethods, PyBytes, PyDict, PyDictMethods, PyList, PyListMethods},
+    types::{IntoPyDict, PyAnyMethods, PyBytes, PyDict, PyDictMethods},
 };

 mod scannable;

-/// Convert `LsmStats` to a Python dict, preserving the per-bucket list.
-///
-/// Deliberately not flattened to a table-level summary: a table is N
-/// buckets on one node, and the per-bucket detail is the reason the
-/// endpoint exists — flattening hides the single hot bucket someone opened
-/// it to find.
-fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult<Py<PyDict>> {
-    let out = PyDict::new(py);
-    let buckets = PyList::empty(py);
-    for b in &stats.buckets {
-        let e = PyDict::new(py);
-        e.set_item("shard_id", &b.shard_id)?;
-        e.set_item("status", &b.status)?;
-        e.set_item("writer_epoch", b.writer_epoch)?;
-        e.set_item("manifest_version", b.manifest_version)?;
-        e.set_item("current_generation", b.current_generation)?;
-        e.set_item(
-            "replay_after_wal_entry_position",
-            b.replay_after_wal_entry_position,
-        )?;
-        e.set_item(
-            "wal_entry_position_last_seen",
-            b.wal_entry_position_last_seen,
-        )?;
-
-        let generations = PyList::empty(py);
-        for g in &b.generations {
-            let ge = PyDict::new(py);
-            ge.set_item("generation", g.generation)?;
-            ge.set_item("bytes", g.bytes)?;
-            ge.set_item("rows", g.rows)?;
-            generations.append(ge)?;
-        }
-        e.set_item("generations", generations)?;
-        e.set_item("compacting", b.compacting)?;
-
-        e.set_item(
-            "memtables",
-            b.memtables
-                .as_ref()
-                .map(|ms| {
-                    let l = PyList::empty(py);
-                    for m in ms {
-                        let d = PyDict::new(py);
-                        d.set_item("generation", m.generation)?;
-                        d.set_item("rows", m.rows)?;
-                        d.set_item("bytes", m.bytes)?;
-                        d.set_item("batches", m.batches)?;
-                        d.set_item("indexes", m.indexes.clone())?;
-                        l.append(d)?;
-                    }
-                    PyResult::Ok(l.unbind())
-                })
-                .transpose()?,
-        )?;
-        buckets.append(e)?;
-    }
-    out.set_item("buckets", buckets)?;
-    Ok(out.unbind())
-}
-
 #[derive(FromPyObject)]
 enum PredicateArg {
    Expr(PyExpr),
@@ -246,22 +185,12 @@ impl From<lancedb::table::MergeResult> for MergeResult {
    }
 }

-/// Render for `__repr__`, so the default reads as Python's `None` rather than
-/// Rust's `Some([..])`.
-fn fmt_maintained(maintained: &Option<Vec<String>>) -> String {
-    match maintained {
-        Some(names) => format!("{:?}", names),
-        None => "None".to_string(),
-    }
-}
-
 /// Specification selecting Lance's MemWAL LSM-style write path for
 /// `merge_insert`.
 ///
 /// Constructed via the `bucket(...)`, `identity(...)`, or `unsharded()`
 /// classmethods, then optionally chain `with_maintained_indexes(...)` and
-/// `with_writer_config_defaults(...)`. A fresh spec maintains every index the
-/// MemWAL supports, resolved on install.
+/// `with_writer_config_defaults(...)`.
 #[pyclass(from_py_object)]
 #[derive(Clone, Debug)]
 pub struct LsmWriteSpec {
@@ -301,11 +230,11 @@ impl LsmWriteSpec {
        }
    }

-    /// Set which indexes the MemWAL maintains. `None` (the default)
-    /// resolves every supported index on install; a list is verbatim,
-    /// and an empty list maintains nothing.
-    #[pyo3(signature = (indexes))]
-    pub fn with_maintained_indexes(&self, indexes: Option<Vec<String>>) -> Self {
+    /// Replace the list of indexes the MemWAL should keep up to date as
+    /// rows are appended. Each name must reference an index that
+    /// already exists on the table at the time `set_lsm_write_spec`
+    /// is called.
+    pub fn with_maintained_indexes(&self, indexes: Vec<String>) -> Self {
        Self {
            inner: self.inner.clone().with_maintained_indexes(indexes),
        }
@@ -327,29 +256,23 @@ impl LsmWriteSpec {
                maintained_indexes,
                writer_config_defaults,
            } => format!(
-                "LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})",
-                column,
-                num_buckets,
-                fmt_maintained(maintained_indexes),
-                writer_config_defaults,
+                "LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={:?}, writer_config_defaults={:?})",
+                column, num_buckets, maintained_indexes, writer_config_defaults,
            ),
            lancedb::table::LsmWriteSpec::Identity {
                column,
                maintained_indexes,
                writer_config_defaults,
            } => format!(
-                "LsmWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})",
-                column,
-                fmt_maintained(maintained_indexes),
-                writer_config_defaults,
+                "LsmWriteSpec.identity(column={:?}, maintained_indexes={:?}, writer_config_defaults={:?})",
+                column, maintained_indexes, writer_config_defaults,
            ),
            lancedb::table::LsmWriteSpec::Unsharded {
                maintained_indexes,
                writer_config_defaults,
            } => format!(
-                "LsmWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})",
-                fmt_maintained(maintained_indexes),
-                writer_config_defaults,
+                "LsmWriteSpec.unsharded(maintained_indexes={:?}, writer_config_defaults={:?})",
+                maintained_indexes, writer_config_defaults,
            ),
        }
    }
@@ -384,10 +307,10 @@ impl LsmWriteSpec {
        }
    }

-    /// Indexes the MemWAL keeps up to date, or `None` for every supported one.
+    /// Names of indexes the MemWAL should keep up to date during writes.
    #[getter]
-    pub fn maintained_indexes(&self) -> Option<Vec<String>> {
-        self.inner.maintained_indexes().map(<[String]>::to_vec)
+    pub fn maintained_indexes(&self) -> Vec<String> {
+        self.inner.maintained_indexes().to_vec()
    }

    /// Default `ShardWriter` configuration recorded by this spec.
@@ -579,7 +502,7 @@ impl PyBlobFile {
    }
 }

-#[pyclass(module = "lancedb._lancedb", get_all, from_py_object)]
+#[pyclass(get_all, from_py_object)]
 #[derive(Clone, Debug)]
 pub struct FtsToken {
    pub text: String,
@@ -1416,51 +1339,6 @@ impl Table {
        })
    }

-    /// Converge the table's LSM write path into its base table.
-    ///
-    /// Best-effort: with writes flowing, new rows may land after the last
-    /// pass. Errors if the table stops making progress.
-    pub fn checkpoint_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.inner_ref()?.clone();
-        future_into_py(self_.py(), async move {
-            inner.checkpoint_lsm().await.infer_error()
-        })
-    }
-
-    /// Seal every bucket's active memtable into L0.
-    pub fn flush_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.inner_ref()?.clone();
-        future_into_py(
-            self_.py(),
-            async move { inner.flush_lsm().await.infer_error() },
-        )
-    }
-
-    /// Trigger a background L0 → base pass per bucket. Returns once the
-    /// passes are dispatched, not once they finish — watch `get_lsm_stats`.
-    pub fn compact_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.inner_ref()?.clone();
-        future_into_py(self_.py(), async move {
-            inner.compact_lsm().await.infer_error()
-        })
-    }
-
-    /// Live LSM state, or `None` when the LSM write path is not enabled.
-    #[pyo3(signature = (include_generation_rows=false))]
-    pub fn get_lsm_stats(
-        self_: PyRef<'_, Self>,
-        include_generation_rows: bool,
-    ) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.inner_ref()?.clone();
-        future_into_py(self_.py(), async move {
-            let stats = inner
-                .get_lsm_stats(include_generation_rows)
-                .await
-                .infer_error()?;
-            Python::attach(|py| stats.map(|s| lsm_stats_to_py(py, &s)).transpose())
-        })
-    }
-
    pub fn close_lsm_writers(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
        let inner = self_.inner_ref()?.clone();
        future_into_py(self_.py(), async move {
@@ -1998,7 +1998,7 @@ requires-dist = [
    { name = "pillow", marker = "extra == 'clip'", specifier = ">=12.1.1" },
    { name = "pillow", marker = "extra == 'embeddings'", specifier = ">=12.1.1" },
    { name = "pillow", marker = "extra == 'siglip'", specifier = ">=12.1.1" },
-    { name = "polars", marker = "extra == 'tests'", specifier = ">=0.19,<=1.32.3" },
+    { name = "polars", marker = "extra == 'tests'", specifier = ">=0.19,<=1.3.0" },
    { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.5.0" },
    { name = "pyarrow", specifier = ">=16" },
    { name = "pyarrow", marker = "extra == 'tests'", specifier = "<25" },
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.38.0-beta.0"
+version = "0.37.1-beta.0"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
@@ -49,6 +49,8 @@ lance-namespace = { workspace = true }
 lance-namespace-impls = { workspace = true }
 metrics = { workspace = true, optional = true }
 metrics-util = { workspace = true, optional = true }
+# Pin the GooseFS SDK to the version required by Lance's OpenDAL dependency.
+goosefs-sdk = { version = "=0.1.9", optional = true }
 moka = { workspace = true }
 pin-project = { workspace = true }
 tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
@@ -98,8 +100,7 @@ anyhow = "1"
 lance-testing = { workspace = true }
 tempfile = "3.5.0"
 random_word = { version = "0.4.3", features = ["en"] }
-roaring = "0.11.4"
-tokio = { version = "1.23", features = ["io-util", "macros", "net", "rt-multi-thread", "sync", "test-util"] }
+tokio = { version = "1.23", features = ["io-util", "macros", "net", "rt-multi-thread", "sync"] }
 uuid = { version = "1.7.0", features = ["v4"] }
 walkdir = "2"
 aws-sdk-dynamodb = { version = "1.55.0" }
@@ -134,6 +135,7 @@ azure = [
 ]
 cos = ["lance/tencent", "lance-io/tencent"]
 goosefs = [
+    "dep:goosefs-sdk",
    "lance/goosefs",
    "lance-io/goosefs",
    "lance-namespace-impls/dir-goosefs",
@@ -188,9 +190,6 @@ required-features = ["bedrock"]
 [[example]]
 name = "bench_streaming_dataloader"

-[[example]]
-name = "bench_open_missing_table"
-
 [[example]]
 name = "simple"

@@ -1,150 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-// Release benchmark for opening a missing table as sibling-table cardinality grows.
-//
-// The fixture uses real `.lance` directories and marker files. Fixture creation is
-// outside the timed section. Defaults intentionally cover 1k, 10k, and 100k siblings
-// with 10 warmups and 100 distinct missing-table opens per scale:
-//
-// ```text
-// cargo run --release -p lancedb --example bench_open_missing_table
-// ```
-//
-// `BENCH_SIBLINGS`, `BENCH_WARMUPS`, and `BENCH_TRIALS` override those defaults.
-// Reduced settings are useful only as a smoke test. Performance comparisons require
-// the same machine, filesystem, fixture sizes, settings, lockfile, and alternating
-// baseline/candidate execution order.
-
-use std::time::{Duration, Instant};
-
-use anyhow::{Context, Result, bail};
-use lancedb::connection::Connection;
-use lancedb::{Error, connect};
-use object_store::ObjectStoreExt as _;
-use object_store::path::Path;
-
-const MAX_SIBLINGS: usize = 1_000_000;
-const MAX_WARMUPS: usize = 10_000;
-const MAX_TRIALS: usize = 100_000;
-
-fn env_usize(key: &str, default: usize, max: usize) -> Result<usize> {
-    let value = match std::env::var(key) {
-        Ok(value) => value
-            .parse()
-            .with_context(|| format!("invalid {key} value: {value}"))?,
-        Err(std::env::VarError::NotPresent) => default,
-        Err(error) => return Err(error).with_context(|| format!("reading {key}")),
-    };
-    if value == 0 || value > max {
-        bail!("{key} must be between 1 and {max}");
-    }
-    Ok(value)
-}
-
-fn sibling_counts() -> Result<Vec<usize>> {
-    let raw = std::env::var("BENCH_SIBLINGS").unwrap_or_else(|_| "1000,10000,100000".into());
-    let mut counts = raw
-        .split(',')
-        .map(|value| {
-            value
-                .trim()
-                .parse::<usize>()
-                .with_context(|| format!("invalid BENCH_SIBLINGS value: {value}"))
-        })
-        .collect::<Result<Vec<_>>>()?;
-    counts.sort_unstable();
-    counts.dedup();
-    if counts.is_empty() || counts[0] == 0 || counts[counts.len() - 1] > MAX_SIBLINGS {
-        bail!("BENCH_SIBLINGS values must be between 1 and {MAX_SIBLINGS}");
-    }
-    Ok(counts)
-}
-
-async fn add_siblings(
-    store: &object_store::local::LocalFileSystem,
-    start: usize,
-    end: usize,
-) -> Result<()> {
-    for index in start..end {
-        let marker = Path::from(format!("sibling_{index:06}.lance/_marker"));
-        store
-            .put(&marker, bytes::Bytes::new().into())
-            .await
-            .with_context(|| format!("creating benchmark marker {marker}"))?;
-    }
-    Ok(())
-}
-
-async fn time_missing_open(db: &Connection, name: &str) -> Result<Duration> {
-    let started = Instant::now();
-    let result = db.open_table(name).execute().await;
-    let elapsed = started.elapsed();
-    match result {
-        Err(Error::TableNotFound { .. }) => Ok(elapsed),
-        Err(error) => bail!("expected TableNotFound for {name}, got {error:?}"),
-        Ok(_) => bail!("benchmark missing-table name unexpectedly exists: {name}"),
-    }
-}
-
-fn percentile(sorted: &[Duration], percentile: usize) -> Duration {
-    let rank = (sorted.len() * percentile).div_ceil(100).saturating_sub(1);
-    sorted[rank]
-}
-
-#[tokio::main]
-async fn main() -> Result<()> {
-    let counts = sibling_counts()?;
-    let warmups = env_usize("BENCH_WARMUPS", 10, MAX_WARMUPS)?;
-    let trials = env_usize("BENCH_TRIALS", 100, MAX_TRIALS)?;
-
-    let fixture = tempfile::tempdir().context("creating benchmark fixture")?;
-    let database_path = fixture.path();
-    let fixture_store = object_store::local::LocalFileSystem::new_with_prefix(database_path)
-        .context("creating benchmark object store")?;
-    let db = connect(database_path.to_str().context("non-UTF-8 fixture path")?)
-        .execute()
-        .await?;
-
-    println!(
-        "config: siblings={counts:?} warmups={warmups} trials={trials} profile={} os={} arch={}",
-        if cfg!(debug_assertions) {
-            "debug"
-        } else {
-            "release"
-        },
-        std::env::consts::OS,
-        std::env::consts::ARCH,
-    );
-    println!("lower is better; fixture setup and teardown are excluded");
-    println!("| siblings | samples | p50 | p95 | max |");
-    println!("| ---: | ---: | ---: | ---: | ---: |");
-
-    let mut created = 0;
-    for sibling_count in counts {
-        add_siblings(&fixture_store, created, sibling_count).await?;
-        created = sibling_count;
-
-        for index in 0..warmups {
-            let name = format!("__missing_warmup_{sibling_count}_{index}");
-            let _ = time_missing_open(&db, &name).await?;
-        }
-
-        let mut samples = Vec::with_capacity(trials);
-        for index in 0..trials {
-            let name = format!("__missing_trial_{sibling_count}_{index}");
-            samples.push(time_missing_open(&db, &name).await?);
-        }
-        samples.sort_unstable();
-
-        println!(
-            "| {sibling_count} | {} | {:?} | {:?} | {:?} |",
-            samples.len(),
-            percentile(&samples, 50),
-            percentile(&samples, 95),
-            samples[samples.len() - 1],
-        );
-    }
-
-    Ok(())
-}
@@ -17,7 +17,7 @@ use arrow_array::builder::LargeBinaryBuilder;
 use arrow_schema::{DataType, Field, Schema};
 use lance::dataset::{BlobRangeRequest as LanceBlobRangeRequest, Dataset, WriteParams};
 use lance_arrow::FieldExt;
-use lance_file::version::{ConcreteFileVersion, LanceFileVersion};
+use lance_file::version::LanceFileVersion;
 use lance_io::object_store::ObjectStore;
 use object_store::path::Path;

@@ -333,10 +333,7 @@ pub(crate) fn ensure_blob_storage_version(schema: &Schema, params: &mut WritePar
        .data_storage_version
        .unwrap_or(LanceFileVersion::Stable)
        .resolve();
-    if matches!(
-        resolved,
-        ConcreteFileVersion::V1 | ConcreteFileVersion::V2_0 | ConcreteFileVersion::V2_1
-    ) {
+    if resolved < LanceFileVersion::V2_2 {
        params.data_storage_version = Some(LanceFileVersion::V2_2);
    }
 }
@@ -502,7 +499,7 @@ mod tests {
        ensure_blob_storage_version(&blob_schema(), &mut params);
        assert_eq!(
            params.data_storage_version.unwrap().resolve(),
-            ConcreteFileVersion::V2_2
+            LanceFileVersion::V2_2
        );
    }

@@ -515,7 +512,7 @@ mod tests {
        ensure_blob_storage_version(&blob_schema(), &mut params);
        assert_eq!(
            params.data_storage_version.unwrap().resolve(),
-            ConcreteFileVersion::V2_2
+            LanceFileVersion::V2_2
        );
    }

@@ -409,11 +409,6 @@ impl Connection {
    ///
    /// The names will be returned in lexicographical order (ascending)
    ///
-    /// Listing databases discover physical `*.lance` entries without opening every
-    /// dataset. The result is a point-in-time discovery snapshot: an entry may still be
-    /// under creation, may contain only uncommitted storage, or may be concurrently
-    /// dropped before it is opened.
-    ///
    /// The parameters `page_token` and `limit` can be used to paginate the results
    pub fn table_names(&self) -> TableNamesBuilder {
        TableNamesBuilder::new(self.internal.clone())
@@ -461,9 +456,10 @@ impl Connection {
    ///
    /// # Returns
    /// Created [`TableRef`], or [`Error::TableNotFound`] if the table does not exist.
-    /// On listing databases, a committed Lance manifest is authoritative for table
-    /// existence. Uncommitted files or a physical `<name>.lance` directory alone do not
-    /// make a table openable.
+    /// If the table's storage is present but holds no readable dataset (for example a
+    /// `<name>.lance` directory left behind by an interrupted drop and re-create, which
+    /// [`Self::table_names`] still lists) this returns [`Error::TableCorrupted`]
+    /// instead.
    pub fn open_table(&self, name: impl Into<String>) -> OpenTableBuilder {
        OpenTableBuilder::new(
            self.internal.clone(),
@@ -565,21 +561,6 @@ impl Connection {
            .await
    }

-    /// Start dropping a table and return a handle to the cleanup job.
-    ///
-    /// The table may become unavailable before its physical data is removed.
-    /// Call [`crate::job::Job::wait`] to wait for cleanup to finish. Local
-    /// backends may complete the drop before returning the handle.
-    pub async fn drop_table_async(
-        &self,
-        name: impl AsRef<str>,
-        namespace_path: &[String],
-    ) -> Result<crate::job::Job> {
-        self.internal
-            .drop_table_async(name.as_ref(), namespace_path)
-            .await
-    }
-
    /// Drop the database
    ///
    /// This is the same as dropping all of the tables
@@ -438,9 +438,10 @@ mod tests {
            .await
            .unwrap()
            .data_storage_format
-            .lance_file_format();
+            .lance_file_version()
+            .unwrap();
        // Compare resolved versions since Stable/Next are aliases that resolve at storage time
-        assert_eq!(storage_format, data_storage_version.resolve());
+        assert_eq!(storage_format.resolve(), data_storage_version.resolve());
    }

    #[tokio::test]
@@ -323,18 +323,6 @@ pub trait Database:
    ) -> Result<()>;
    /// Drop a table in the database
    async fn drop_table(&self, name: &str, namespace_path: &[String]) -> Result<()>;
-    /// Start dropping a table and return a handle to the cleanup job.
-    ///
-    /// Backends without asynchronous cleanup complete the drop before
-    /// returning an already-finished job.
-    async fn drop_table_async(
-        &self,
-        name: &str,
-        namespace_path: &[String],
-    ) -> Result<crate::job::Job> {
-        self.drop_table(name, namespace_path).await?;
-        Ok(crate::job::Job::new_done())
-    }
    /// Drop all tables in the database
    async fn drop_all_tables(&self, namespace_path: &[String]) -> Result<()>;
    fn as_any(&self) -> &dyn std::any::Any;
@@ -1291,21 +1291,14 @@ impl Database for ListingDatabase {
 mod tests {
    use super::*;
    use crate::Table;
-    use crate::arrow::{SendableRecordBatchStream, SimpleRecordBatchStream};
    use crate::connection::ConnectRequest;
    use crate::data::scannable::Scannable;
    use crate::database::{CreateTableMode, CreateTableRequest};
-    use crate::query::QueryRequest;
-    use crate::table::{AnyQuery, WriteOptions};
+    use crate::table::WriteOptions;
    use arrow_array::{Int32Array, RecordBatch, StringArray};
-    use arrow_schema::{DataType, Field, Schema, SchemaRef};
-    use futures::{TryStreamExt, stream::once};
+    use arrow_schema::{DataType, Field, Schema};
    use std::path::PathBuf;
-    use std::sync::Arc;
-    use std::time::Duration;
    use tempfile::tempdir;
-    use tokio::sync::Barrier;
-    use tokio::time::timeout;

    async fn setup_database() -> (tempfile::TempDir, ListingDatabase) {
        let tempdir = tempdir().unwrap();
@@ -1329,114 +1322,6 @@ mod tests {
        (tempdir, db)
    }

-    struct BarrierScannable {
-        batch: RecordBatch,
-        barrier: Arc<Barrier>,
-    }
-
-    impl Scannable for BarrierScannable {
-        fn schema(&self) -> SchemaRef {
-            self.batch.schema()
-        }
-
-        fn scan_as_stream(&mut self) -> SendableRecordBatchStream {
-            let batch = self.batch.clone();
-            let schema = batch.schema();
-            let barrier = self.barrier.clone();
-            Box::pin(SimpleRecordBatchStream {
-                schema,
-                stream: once(async move {
-                    barrier.wait().await;
-                    Ok(batch)
-                }),
-            })
-        }
-    }
-
-    fn create_request(name: &str, data: Box<dyn Scannable>) -> CreateTableRequest {
-        CreateTableRequest {
-            name: name.to_string(),
-            namespace_path: vec![],
-            data,
-            mode: CreateTableMode::Create,
-            write_options: Default::default(),
-            location: None,
-            namespace_client: None,
-        }
-    }
-
-    #[tokio::test]
-    async fn test_create_ignores_uncommitted_storage_without_manifest() {
-        let (tmp_dir, db) = setup_database().await;
-        let data_dir = tmp_dir.path().join("test.lance/data");
-        std::fs::create_dir_all(&data_dir).unwrap();
-        std::fs::write(data_dir.join("orphan.lance"), b"uncommitted").unwrap();
-
-        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
-        let batch =
-            RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1]))]).unwrap();
-
-        let table = db
-            .create_table(create_request("test", Box::new(batch)))
-            .await
-            .unwrap();
-        assert_eq!(table.count_rows(None).await.unwrap(), 1);
-    }
-
-    #[tokio::test]
-    async fn test_concurrent_create_is_arbitrated_by_manifest_commit() {
-        let uri = format!("memory:///concurrent-create-{}", uuid::Uuid::new_v4());
-        let db = crate::connect(&uri).execute().await.unwrap();
-        let store: Arc<dyn object_store::ObjectStore> =
-            Arc::new(object_store::memory::InMemory::new());
-        let table_url = url::Url::parse("memory:///database/test.lance").unwrap();
-        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
-        let batch =
-            RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1]))]).unwrap();
-        let barrier = Arc::new(Barrier::new(2));
-
-        #[allow(deprecated)]
-        let request = |batch, barrier| {
-            let mut request = create_request("test", Box::new(BarrierScannable { batch, barrier }));
-            request.write_options = WriteOptions {
-                lance_write_params: Some(lance::dataset::WriteParams {
-                    store_params: Some(ObjectStoreParams {
-                        object_store: Some((store.clone(), table_url.clone())),
-                        ..Default::default()
-                    }),
-                    commit_handler: Some(Arc::new(
-                        lance_table::io::commit::ConditionalPutCommitHandler,
-                    )),
-                    ..Default::default()
-                }),
-            };
-            request
-        };
-
-        let left = db
-            .database()
-            .create_table(request(batch.clone(), barrier.clone()));
-        let right = db.database().create_table(request(batch, barrier));
-        let (left, right) = timeout(Duration::from_secs(30), async { tokio::join!(left, right) })
-            .await
-            .expect("concurrent creates deadlocked");
-
-        let results = [left, right];
-        assert_eq!(
-            results.iter().filter(|result| result.is_ok()).count(),
-            1,
-            "expected one successful create, got {results:?}"
-        );
-        assert_eq!(
-            results
-                .iter()
-                .filter(|result| matches!(result, Err(Error::TableAlreadyExists { .. })))
-                .count(),
-            1,
-            "expected one manifest conflict, got {results:?}"
-        );
-    }
-
    #[tokio::test]
    async fn test_listing_database_root_ops_do_not_create_manifest() {
        let tempdir = tempdir().unwrap();
@@ -1553,94 +1438,6 @@ mod tests {
        assert!(after_open.hits >= before_open.hits + 3);
    }

-    /// Regression test for https://github.com/lancedb/lancedb/issues/3197.
-    #[cfg(unix)]
-    #[tokio::test]
-    async fn test_open_table_follows_hugging_face_symlinks() {
-        let (tempdir, db) = setup_database().await;
-        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
-        db.create_table(CreateTableRequest {
-            name: "test".to_string(),
-            namespace_path: vec![],
-            data: Box::new(
-                RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))])
-                    .unwrap(),
-            ) as Box<dyn Scannable>,
-            mode: CreateTableMode::Create,
-            write_options: Default::default(),
-            location: None,
-            namespace_client: None,
-        })
-        .await
-        .unwrap();
-
-        let table_dir = tempdir.path().join("test.lance");
-        let versions_dir = table_dir.join("_versions");
-        let manifest_path = std::fs::read_dir(&versions_dir)
-            .unwrap()
-            .map(|entry| entry.unwrap().path())
-            .find(|path| path.extension().is_some_and(|ext| ext == "manifest"))
-            .unwrap();
-        let data_path = std::fs::read_dir(table_dir.join("data"))
-            .unwrap()
-            .map(|entry| entry.unwrap().path())
-            .find(|path| path.extension().is_some_and(|ext| ext == "lance"))
-            .unwrap();
-
-        // Hugging Face snapshots keep dataset objects in a separate blob directory and
-        // expose them through relative symlinks.
-        let blobs_dir = tempdir.path().join("blobs");
-        std::fs::create_dir(&blobs_dir).unwrap();
-        let manifest_blob = "9b603c63d0e692e05d58be25605f2f2064cc781e5ff94fe983a405059547b816";
-        let data_blob = "be64f20e5723bd0a27cfdbdb41cf7d6fad94cd572a71973b717fb8340f4310c5";
-        std::fs::rename(&manifest_path, blobs_dir.join(manifest_blob)).unwrap();
-        std::fs::rename(&data_path, blobs_dir.join(data_blob)).unwrap();
-        std::os::unix::fs::symlink(Path::new("../../blobs").join(manifest_blob), &manifest_path)
-            .unwrap();
-        std::os::unix::fs::symlink(Path::new("../../blobs").join(data_blob), &data_path).unwrap();
-        let symlink_len = std::fs::symlink_metadata(&manifest_path).unwrap().len();
-        let target_len = std::fs::metadata(&manifest_path).unwrap().len();
-        assert_ne!(symlink_len, target_len);
-
-        drop(db);
-        let db = ListingDatabase::connect_with_options(&ConnectRequest {
-            uri: tempdir.path().to_str().unwrap().to_string(),
-            #[cfg(feature = "remote")]
-            client_config: Default::default(),
-            options: Default::default(),
-            namespace_client_properties: Default::default(),
-            manifest_enabled: false,
-            read_consistency_interval: None,
-            session: None,
-        })
-        .await
-        .unwrap();
-
-        let table = db
-            .open_table(OpenTableRequest {
-                name: "test".to_string(),
-                namespace_path: vec![],
-                index_cache_size: None,
-                lance_read_params: None,
-                location: None,
-                namespace_client: None,
-                managed_versioning: None,
-            })
-            .await
-            .unwrap();
-        let batches = table
-            .query(
-                &AnyQuery::Query(QueryRequest::default()),
-                Default::default(),
-            )
-            .await
-            .unwrap()
-            .try_collect::<Vec<_>>()
-            .await
-            .unwrap();
-        assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 3);
-    }
-
    #[tokio::test]
    async fn test_clone_table_basic() {
        let (_tempdir, db) = setup_database().await;
@@ -2545,7 +2342,7 @@ mod tests {

    #[tokio::test]
    async fn test_table_uri() {
-        let (_tempdir, mut db) = setup_database().await;
+        let (_tempdir, db) = setup_database().await;

        let mut pb = PathBuf::new();
        pb.push(db.uri.clone());
@@ -2554,18 +2351,6 @@ mod tests {
        let expected = pb.to_str().unwrap();
        let uri = db.table_uri("test").ok().unwrap();
        assert_eq!(uri, expected);
-
-        // URI paths always use forward slashes, even on Windows. Using
-        // `Path::join` here used to produce `az://container/prefix\\test.lance`,
-        // which Azure treated as a different object from the table returned by
-        // `table_names` (https://github.com/lancedb/lancedb/issues/1072).
-        for base_uri in ["az://container/prefix", "az://container/prefix/"] {
-            db.uri = base_uri.to_string();
-            assert_eq!(
-                db.table_uri("test").unwrap(),
-                "az://container/prefix/test.lance"
-            );
-        }
    }

    /// Regression: connecting via a URL-style URI (which goes through
@@ -169,12 +169,6 @@ impl From<DataFusionError> for Error {

 impl From<lance::Error> for Error {
    fn from(source: lance::Error) -> Self {
-        if has_unsupported_local_filesystem_source(&source) {
-            return Self::NotSupported {
-                message: "the filesystem does not support an operation required for safe Lance commits (such as atomic rename). Object-storage mounts such as Mountpoint for Amazon S3 are not supported; use the native object-store URI (for example, s3://bucket/path) instead".to_string(),
-            };
-        }
-
        // Try to unwrap external errors that were wrapped by lance
        match source {
            lance::Error::Wrapped { error, .. } => Self::from_box_error(error),
@@ -187,27 +181,6 @@ impl From<lance::Error> for Error {
    }
 }

-fn has_unsupported_local_filesystem_source(error: &(dyn std::error::Error + 'static)) -> bool {
-    let mut current = Some(error);
-    let mut is_local_filesystem = false;
-    let mut is_unsupported = false;
-    while let Some(error) = current {
-        is_local_filesystem |= error
-            .downcast_ref::<object_store::Error>()
-            .is_some_and(|error| {
-                matches!(error, object_store::Error::Generic { store, .. } if *store == "LocalFileSystem")
-            });
-        is_unsupported |= error
-            .downcast_ref::<std::io::Error>()
-            .is_some_and(|error| error.kind() == std::io::ErrorKind::Unsupported);
-        if is_local_filesystem && is_unsupported {
-            return true;
-        }
-        current = error.source();
-    }
-    false
-}
-
 impl Error {
    fn from_box_error(mut source: Box<dyn std::error::Error + Send + Sync>) -> Self {
        source = match source.downcast::<Self>() {
@@ -297,46 +270,3 @@ impl From<candle_core::Error> for Error {
        }
    }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn unsupported_filesystem_operations_have_actionable_error() {
-        let object_store_error = object_store::Error::Generic {
-            store: "LocalFileSystem",
-            source: Box::new(std::io::Error::from(std::io::ErrorKind::Unsupported)),
-        };
-        let lance_error = lance::Error::io_source(Box::new(object_store_error));
-
-        let error = Error::from(lance_error);
-
-        assert!(matches!(
-            error,
-            Error::NotSupported { message }
-                if message.contains("Mountpoint for Amazon S3")
-                    && message.contains("s3://bucket/path")
-        ));
-    }
-
-    #[test]
-    fn other_io_errors_remain_lance_errors() {
-        let object_store_error = object_store::Error::Generic {
-            store: "LocalFileSystem",
-            source: Box::new(std::io::Error::from(std::io::ErrorKind::PermissionDenied)),
-        };
-        let lance_error = lance::Error::io_source(Box::new(object_store_error));
-
-        assert!(matches!(Error::from(lance_error), Error::Lance { .. }));
-    }
-
-    #[test]
-    fn unsupported_non_filesystem_errors_remain_lance_errors() {
-        let lance_error = lance::Error::io_source(Box::new(std::io::Error::from(
-            std::io::ErrorKind::Unsupported,
-        )));
-
-        assert!(matches!(Error::from(lance_error), Error::Lance { .. }));
-    }
-}
@@ -19,15 +19,6 @@ const ARROW_FILE_CONTENT_TYPE: &str = "application/vnd.apache.arrow.file";
 #[cfg(test)]
 const JSON_CONTENT_TYPE: &str = "application/json";

-fn extract_job_id(body: &str) -> Option<String> {
-    serde_json::from_str::<serde_json::Value>(body)
-        .ok()?
-        .get("job_id")?
-        .as_str()
-        .filter(|job_id| !job_id.is_empty())
-        .map(str::to_string)
-}
-
 pub use client::{ClientConfig, HeaderProvider, RetryConfig, TimeoutConfig, TlsConfig};
 pub use db::{RemoteDatabaseOptions, RemoteDatabaseOptionsBuilder};
 pub use oauth::{OAuthConfig, OAuthFlow, OAuthHeaderProvider};
@@ -9,7 +9,6 @@ use http::StatusCode;
 use lance_io::object_store::StorageOptions;
 use lance_namespace_impls::{DynamicContextProvider, OperationInfo};
 use moka::future::Cache;
-use reqwest::Response;
 use reqwest::header::CONTENT_TYPE;

 use lance_namespace::models::{
@@ -24,17 +23,15 @@ use crate::database::{
    JobDescription, JobInfo, OpenTableRequest, ReadConsistency, TableNamesRequest,
 };
 use crate::error::Result;
-use crate::job::Job;
-use crate::remote::job::RemoteJob;
 use crate::remote::util::stream_as_body;
 use crate::table::BaseTable;

+use super::ARROW_STREAM_CONTENT_TYPE;
 use super::client::{
    ClientConfig, HeaderProvider, HttpSend, RequestResultExt, RestfulLanceDbClient, Sender,
 };
 use super::table::RemoteTable;
 use super::util::parse_server_version;
-use super::{ARROW_STREAM_CONTENT_TYPE, extract_job_id};

 // Request structure for the remote clone table API
 #[derive(serde::Serialize)]
@@ -329,22 +326,6 @@ impl RemoteDatabase {
    }
 }

-impl<S: HttpSend> RemoteDatabase<S> {
-    async fn submit_drop_table(
-        &self,
-        name: &str,
-        namespace_path: &[String],
-    ) -> Result<(String, Response)> {
-        let identifier = build_table_identifier(name, namespace_path, &self.client.id_delimiter);
-        let cache_key = build_cache_key(name, namespace_path);
-        let req = self.client.post(&format!("/v1/table/{}/drop/", identifier));
-        let (request_id, resp) = self.client.send(req).await?;
-        let resp = self.client.check_response(&request_id, resp).await?;
-        self.table_cache.remove(&cache_key).await;
-        Ok((request_id, resp))
-    }
-}
-
 #[cfg(all(test, feature = "remote"))]
 mod test_utils {
    use super::*;
@@ -913,28 +894,13 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
    }

    async fn drop_table(&self, name: &str, namespace_path: &[String]) -> Result<()> {
-        self.submit_drop_table(name, namespace_path)
-            .await
-            .map(|_| ())
-    }
-
-    async fn drop_table_async(&self, name: &str, namespace_path: &[String]) -> Result<Job> {
-        let (request_id, response) = self.submit_drop_table(name, namespace_path).await?;
-        let status = response.status();
-        let body = response.text().await.err_to_http(request_id.clone())?;
-        let job_id = extract_job_id(&body);
-        Ok(match job_id {
-            Some(job_id) => Job::new(Box::new(RemoteJob::new(self.client.clone(), job_id))),
-            None if status == StatusCode::ACCEPTED => {
-                return Err(Error::Http {
-                    source: "asynchronous drop-table response did not contain a valid job_id"
-                        .into(),
-                    request_id,
-                    status_code: Some(status),
-                });
-            }
-            None => Job::new_done(),
-        })
+        let identifier = build_table_identifier(name, namespace_path, &self.client.id_delimiter);
+        let cache_key = build_cache_key(name, namespace_path);
+        let req = self.client.post(&format!("/v1/table/{}/drop/", identifier));
+        let (request_id, resp) = self.client.send(req).await?;
+        self.client.check_response(&request_id, resp).await?;
+        self.table_cache.remove(&cache_key).await;
+        Ok(())
    }

    async fn drop_all_tables(&self, namespace_path: &[String]) -> Result<()> {
@@ -1526,67 +1492,6 @@ mod tests {
        // NOTE: the API will return 200 even if the table does not exist. So we shouldn't expect 404.
    }

-    #[tokio::test]
-    async fn test_drop_table_does_not_read_response_body() {
-        let conn = Connection::new_with_handler(|_| {
-            http::Response::builder()
-                .status(200)
-                .body(vec![0xff])
-                .unwrap()
-        });
-
-        conn.drop_table("table1", &[]).await.unwrap();
-    }
-
-    #[tokio::test]
-    async fn test_drop_table_async_returns_job() {
-        let conn = Connection::new_with_handler(|request| {
-            assert_eq!(request.method(), &reqwest::Method::POST);
-            assert_eq!(request.url().path(), "/v1/table/table1/drop/");
-            http::Response::builder()
-                .status(202)
-                .body(r#"{"job_id":"drop-job-123"}"#)
-                .unwrap()
-        });
-
-        let job = conn.drop_table_async("table1", &[]).await.unwrap();
-        assert_eq!(job.id(), Some("drop-job-123"));
-    }
-
-    #[tokio::test]
-    async fn test_drop_table_async_old_server_returns_done_job() {
-        let conn = Connection::new_with_handler(|_| {
-            http::Response::builder().status(200).body("").unwrap()
-        });
-
-        let job = conn.drop_table_async("table1", &[]).await.unwrap();
-        assert_eq!(job.id(), None);
-        assert_eq!(job.status().await.unwrap(), "finished");
-    }
-
-    #[tokio::test]
-    async fn test_drop_table_async_rejects_accepted_response_without_job_id() {
-        let conn = Connection::new_with_handler(|_| {
-            http::Response::builder().status(202).body("{}").unwrap()
-        });
-
-        let error = conn.drop_table_async("table1", &[]).await.err().unwrap();
-        assert!(error.to_string().contains("valid job_id"));
-    }
-
-    #[tokio::test]
-    async fn test_drop_table_async_rejects_empty_job_id() {
-        let conn = Connection::new_with_handler(|_| {
-            http::Response::builder()
-                .status(202)
-                .body(r#"{"job_id":""}"#)
-                .unwrap()
-        });
-
-        let error = conn.drop_table_async("table1", &[]).await.err().unwrap();
-        assert!(error.to_string().contains("valid job_id"));
-    }
-
    #[tokio::test]
    async fn test_rename_table() {
        let conn = Connection::new_with_handler(|request| {
@@ -8,7 +8,7 @@ use self::insert::{RemoteWriteExec, WriteOp};
 use super::client::RequestResultExt;
 use super::client::{HttpSend, RestfulLanceDbClient, Sender};
 use super::db::ServerVersion;
-use super::{ARROW_FILE_CONTENT_TYPE, ARROW_STREAM_CONTENT_TYPE, extract_job_id};
+use super::{ARROW_FILE_CONTENT_TYPE, ARROW_STREAM_CONTENT_TYPE};
 use crate::blob::BlobFile;
 use crate::data::scannable::{PeekedScannable, Scannable, estimate_write_partitions};
 use crate::expr::expr_to_sql_string;
@@ -23,13 +23,11 @@ use crate::table::AddResult;
 use crate::table::BranchDiff;
 use crate::table::DeleteResult;
 use crate::table::DropColumnsResult;
-use crate::table::LsmStats;
 use crate::table::LsmWriteSpec;
 use crate::table::MergeBranchResult;
 use crate::table::MergeResult;
 use crate::table::Tags;
 use crate::table::UpdateResult;
-use crate::table::lsm_stats::GetLsmStatsResponse;
 use crate::table::merge::MergeFilter;
 use crate::table::query::create_multi_vector_plan;
 use crate::table::write_progress::FinishOnDrop;
@@ -392,7 +390,13 @@ impl<S: HttpSend> RemoteTable<S> {
            .text()
            .await
            .ok()
-            .and_then(|body| extract_job_id(&body));
+            .and_then(|body| serde_json::from_str::<serde_json::Value>(&body).ok())
+            .and_then(|value| {
+                value
+                    .get("job_id")
+                    .and_then(|id| id.as_str())
+                    .map(str::to_string)
+            });

        if let Some(wait_timeout) = index.wait_timeout {
            let index_name = index.name.unwrap_or_else(|| format!("{}_idx", column));
@@ -987,18 +991,6 @@ impl<S: HttpSend> RemoteTable<S> {
        }
    }

-    /// Send an LSM operator request with the transport retry layer **off**.
-    ///
-    /// Retry policy on these routes belongs to the checkpoint loop, which
-    /// reads the status and can tell contention from a lost claim. Leaving the
-    /// transport layer on would re-ask on its own schedule first, and surface
-    /// an `Error::Retry` whose status the loop would then have to unwrap.
-    async fn send_lsm_route(&self, request: RequestBuilder) -> Result<(String, reqwest::Response)> {
-        let (request_id, response) = self.send(request, false).await?;
-        let response = self.check_table_response(&request_id, response).await?;
-        Ok((request_id, response))
-    }
-
    /// Build a POST request and attach the read-freshness headers
    /// (`x-lancedb-min-version`, `x-lancedb-min-timestamp`).
    fn post_read(&self, uri: &str) -> RequestBuilder {
@@ -2476,47 +2468,13 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
        })
    }

-    async fn flush_lsm(&self) -> Result<()> {
-        let request = self
-            .client
-            .post(&format!("/v1/table/{}/flush_lsm/", self.identifier));
-        self.send_lsm_route(request).await?;
-        Ok(())
-    }
-
-    async fn compact_lsm(&self) -> Result<()> {
-        let request = self
-            .client
-            .post(&format!("/v1/table/{}/compact_lsm/", self.identifier));
-        self.send_lsm_route(request).await?;
-        Ok(())
-    }
-
-    async fn get_lsm_stats(&self, include_generation_rows: bool) -> Result<Option<LsmStats>> {
-        // Read-semantics POST, like `get_lsm_write_spec`.
-        let request = self
-            .post_read(&format!("/v1/table/{}/get_lsm_stats/", self.identifier))
-            .json(&serde_json::json!({
-                "include_generation_rows": include_generation_rows,
-            }));
-        let (request_id, response) = self.send_lsm_route(request).await?;
-        let body = response.text().await.err_to_http(request_id.clone())?;
-        let parsed: GetLsmStatsResponse = serde_json::from_str(&body).map_err(|e| Error::Http {
-            source: format!("Failed to parse get_lsm_stats response: {e}").into(),
-            request_id,
-            status_code: None,
-        })?;
-        // `null` — and only — when the table has no LSM write path.
-        Ok(parsed.lsm_stats)
-    }
-
    async fn set_lsm_write_spec(&self, spec: LsmWriteSpec) -> Result<()> {
        self.check_mutable().await?;

        // Map the spec onto the server's request DTO. `sharding` is internally
-        // tagged on `mode` to mirror sophon's `Sharding` enum. A null
-        // `maintained_indexes` asks the server to resolve every maintainable
-        // index at HEAD; a list is verbatim, an empty one meaning none.
+        // tagged on `mode` to mirror sophon's `Sharding` enum; `maintained_indexes`
+        // and `writer_config_defaults` are sent verbatim (an empty list means "no
+        // maintained indexes", not "default to all").
        let sharding = match &spec {
            LsmWriteSpec::Bucket {
                column,
@@ -6593,7 +6551,7 @@ mod tests {
                .unwrap()
        });
        let spec = crate::table::LsmWriteSpec::unsharded()
-            .with_maintained_indexes(vec!["id_idx".to_string()])
+            .with_maintained_indexes(["id_idx"])
            .with_writer_config_defaults([("max_memtable_rows", "1000")]);
        table.set_lsm_write_spec(spec).await.unwrap();
    }
@@ -6612,29 +6570,11 @@ mod tests {
                body["sharding"],
                serde_json::json!({ "mode": "bucket", "column": "id", "num_buckets": 16 })
            );
-            // An unpinned maintained set sends null: resolve server-side.
-            assert_eq!(body["maintained_indexes"], serde_json::Value::Null);
-            http::Response::builder().status(200).body("{}").unwrap()
-        });
-        table
-            .set_lsm_write_spec(crate::table::LsmWriteSpec::bucket("id", 16))
-            .await
-            .unwrap();
-    }
-
-    /// `[]` (none) must stay distinguishable on the wire from null (all).
-    #[tokio::test]
-    async fn test_set_lsm_write_spec_no_maintained_indexes() {
-        let table = Table::new_with_handler("my_table", |request| {
-            let body = request.body().unwrap().as_bytes().unwrap();
-            let body: serde_json::Value = serde_json::from_slice(body).unwrap();
            assert_eq!(body["maintained_indexes"], serde_json::json!([]));
            http::Response::builder().status(200).body("{}").unwrap()
        });
        table
-            .set_lsm_write_spec(
-                crate::table::LsmWriteSpec::bucket("id", 16).with_maintained_indexes(Vec::new()),
-            )
+            .set_lsm_write_spec(crate::table::LsmWriteSpec::bucket("id", 16))
            .await
            .unwrap();
    }
@@ -6713,7 +6653,7 @@ mod tests {
            } => {
                assert_eq!(column, "id");
                assert_eq!(num_buckets, 4);
-                assert_eq!(maintained_indexes, Some(vec!["id_idx".to_string()]));
+                assert_eq!(maintained_indexes, vec!["id_idx".to_string()]);
                assert_eq!(
                    writer_config_defaults
                        .get("durable_write")
@@ -6742,499 +6682,6 @@ mod tests {
        assert!(table.get_lsm_write_spec().await.unwrap().is_none());
    }

-    /// Build a `get_lsm_stats` body for one bucket holding `generations`.
-    fn stats_body(generations: &[u64], compacting: bool) -> String {
-        serde_json::json!({
-            "lsm_stats": {
-                "buckets": [{
-                    "shard_id": "b0",
-                    "status": "Active",
-                    "writer_epoch": 1,
-                    "manifest_version": 1,
-                    "current_generation": generations.iter().max().copied().unwrap_or(0) + 1,
-                    "replay_after_wal_entry_position": 0,
-                    "wal_entry_position_last_seen": 0,
-                    "generations": generations.iter()
-                        .map(|g| serde_json::json!({ "generation": g, "bytes": 1 }))
-                        .collect::<Vec<_>>(),
-                    "compacting": compacting,
-                    "memtables": [],
-                }],
-            }
-        })
-        .to_string()
-    }
-
-    /// `flush_lsm` / `compact_lsm` answer 202 with no body at all.
-    fn accepted() -> http::Response<String> {
-        http::Response::builder()
-            .status(202)
-            .body(String::new())
-            .unwrap()
-    }
-
-    fn ok_json(body: String) -> http::Response<String> {
-        http::Response::builder().status(200).body(body).unwrap()
-    }
-
-    /// A flush landing in an empty L0 finishes on the opening stats read
-    /// alone. Asserting zero compacts is the point: "it returned Ok" is also
-    /// true of a loop that ran a pointless pass.
-    #[tokio::test(start_paused = true)]
-    async fn test_checkpoint_short_circuits_on_empty_l0() {
-        let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-        let seen = compacts.clone();
-        let table = Table::new_with_handler("my_table", move |request| {
-            let path = request.url().path().to_string();
-            if path.contains("compact_lsm") {
-                seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-                panic!("an already-converged table must issue no compact calls");
-            }
-            if path.contains("flush_lsm") {
-                return accepted();
-            }
-            assert_eq!(path, "/v1/table/my_table/get_lsm_stats/");
-            ok_json(stats_body(&[], false))
-        });
-
-        table.checkpoint_lsm().await.unwrap();
-        assert_eq!(compacts.load(std::sync::atomic::Ordering::SeqCst), 0);
-    }
-
-    /// The loop triggers compaction until every generation that existed at
-    /// the start is gone, one bounded prefix per pass.
-    #[tokio::test(start_paused = true)]
-    async fn test_checkpoint_triggers_until_targets_are_drained() {
-        let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-        let seen = compacts.clone();
-        let table = Table::new_with_handler("my_table", move |request| {
-            let path = request.url().path().to_string();
-            if path.contains("flush_lsm") {
-                return accepted();
-            }
-            if path.contains("compact_lsm") {
-                seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-                return accepted();
-            }
-            // Each pass drains the oldest generation.
-            let drained = seen.load(std::sync::atomic::Ordering::SeqCst);
-            let left: Vec<u64> = [1u64, 2, 3].into_iter().skip(drained).collect();
-            ok_json(stats_body(&left, false))
-        });
-
-        table.checkpoint_lsm().await.unwrap();
-        assert_eq!(
-            compacts.load(std::sync::atomic::Ordering::SeqCst),
-            3,
-            "one trigger per generation prefix, then stop"
-        );
-    }
-
-    /// Generations created *during* the checkpoint are not waited on, which
-    /// is what lets the loop terminate on a table taking writes where "L0 is
-    /// empty" never becomes true.
-    #[tokio::test(start_paused = true)]
-    async fn test_checkpoint_ignores_generations_created_while_it_runs() {
-        let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-        let seen = compacts.clone();
-        let table = Table::new_with_handler("my_table", move |request| {
-            let path = request.url().path().to_string();
-            if path.contains("flush_lsm") {
-                return accepted();
-            }
-            if path.contains("compact_lsm") {
-                seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-                return accepted();
-            }
-            // Target is 5. One pass drains it; a writer keeps adding above.
-            let n = seen.load(std::sync::atomic::Ordering::SeqCst);
-            let body = if n == 0 {
-                stats_body(&[5], false)
-            } else {
-                stats_body(&[6, 7], false)
-            };
-            ok_json(body)
-        });
-
-        table.checkpoint_lsm().await.unwrap();
-        assert_eq!(
-            compacts.load(std::sync::atomic::Ordering::SeqCst),
-            1,
-            "the loop must not chase generations written after it started"
-        );
-    }
-
-    /// Contention is a 429 and must be retried. The server keeps it off 503
-    /// precisely so the client can act on the status alone — reading it as
-    /// terminal stops the checkpoint early on a healthy node.
-    #[tokio::test(start_paused = true)]
-    async fn test_checkpoint_retries_contention() {
-        let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-        let seen = compacts.clone();
-        let table = Table::new_with_handler("my_table", move |request| {
-            let path = request.url().path().to_string();
-            if path.contains("flush_lsm") {
-                return accepted();
-            }
-            if path.contains("compact_lsm") {
-                // First two triggers: every bucket already latched.
-                if seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2 {
-                    return http::Response::builder()
-                        .status(429)
-                        .body(r#"{"code":21,"error":"Too many concurrent writes"}"#.to_string())
-                        .unwrap();
-                }
-                return accepted();
-            }
-            let accepted_triggers = seen
-                .load(std::sync::atomic::Ordering::SeqCst)
-                .saturating_sub(2);
-            let left: Vec<u64> = if accepted_triggers == 0 {
-                vec![1]
-            } else {
-                vec![]
-            };
-            ok_json(stats_body(&left, false))
-        });
-
-        table
-            .checkpoint_lsm()
-            .await
-            .expect("contention must not abort the checkpoint");
-        assert_eq!(
-            compacts.load(std::sync::atomic::Ordering::SeqCst),
-            3,
-            "assert the retry count, not just the outcome"
-        );
-    }
-
-    /// A transient fault on the poll must not abort the checkpoint. This route
-    /// meets the most contention — it runs every `POLL_INTERVAL` for the
-    /// checkpoint's whole life, with the transport retry layer disabled — yet
-    /// was the one call reached with a bare `?`.
-    #[tokio::test(start_paused = true)]
-    async fn test_checkpoint_retries_a_contended_stats_poll() {
-        let polls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-        let seen = polls.clone();
-        let table = Table::new_with_handler("my_table", move |request| {
-            let path = request.url().path().to_string();
-            if path.contains("flush_lsm") || path.contains("compact_lsm") {
-                return accepted();
-            }
-            // The opening read lands; the next two polls are latched out.
-            let n = seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-            if (1..3).contains(&n) {
-                return http::Response::builder()
-                    .status(429)
-                    .body(r#"{"code":21,"error":"Too many concurrent writes"}"#.to_string())
-                    .unwrap();
-            }
-            ok_json(stats_body(if n < 4 { &[1] } else { &[] }, false))
-        });
-
-        table
-            .checkpoint_lsm()
-            .await
-            .expect("a contended poll must be retried, not surfaced");
-        assert_eq!(
-            polls.load(std::sync::atomic::Ordering::SeqCst),
-            5,
-            "the two rejected polls must be re-issued, not skipped"
-        );
-    }
-
-    /// Contention and a lost claim draw on separate budgets: five straight
-    /// 429s on `flush`, more than `MAX_REISSUES`, must still converge. On one
-    /// shared counter this spent the re-issue cap and then reported a lost
-    /// claim nothing had ever reported.
-    #[tokio::test(start_paused = true)]
-    async fn test_contention_does_not_exhaust_the_reissue_budget() {
-        let flushes = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-        let seen = flushes.clone();
-        let table = Table::new_with_handler("my_table", move |request| {
-            let path = request.url().path().to_string();
-            if path.contains("flush_lsm") {
-                if seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 5 {
-                    return http::Response::builder()
-                        .status(429)
-                        .body(r#"{"code":21,"error":"Too many concurrent writes"}"#.to_string())
-                        .unwrap();
-                }
-                return accepted();
-            }
-            if path.contains("compact_lsm") {
-                return accepted();
-            }
-            ok_json(stats_body(&[], false))
-        });
-
-        table
-            .checkpoint_lsm()
-            .await
-            .expect("contention must not be reported as a lost claim");
-        assert_eq!(
-            flushes.load(std::sync::atomic::Ordering::SeqCst),
-            6,
-            "five retries against one seal, then it lands"
-        );
-    }
-
-    /// An exhausted retry budget surfaces the fault that consumed it, not a
-    /// message the loop invented: "429, nine times" points an operator at a
-    /// saturated pool, a generic runtime error points them nowhere.
-    #[tokio::test(start_paused = true)]
-    async fn test_exhausted_retries_surface_the_underlying_fault() {
-        let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-        let seen = calls.clone();
-        let table = Table::new_with_handler("my_table", move |_request| {
-            seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-            http::Response::builder()
-                .status(429)
-                .body(r#"{"code":21,"error":"Too many concurrent writes"}"#.to_string())
-                .unwrap()
-        });
-
-        let err = table.checkpoint_lsm().await.unwrap_err();
-        assert!(
-            matches!(&err, Error::Http { status_code: Some(s), .. } if s.as_u16() == 429),
-            "the fault that spent the budget must be the one reported: {err:?}"
-        );
-        assert_eq!(
-            calls.load(std::sync::atomic::Ordering::SeqCst),
-            9,
-            "one call plus MAX_RETRIES — the re-issue budget is not spent on top"
-        );
-    }
-
-    /// A draining node is terminal, but the client does not know that from the
-    /// status: draining and a proxy blip are both 503, and telling them apart
-    /// takes parsing the body for a namespace code. So it spends the retry
-    /// budget and then reports what the server said — the drain gate never
-    /// releases, so the answer does not change, and the operator still reads
-    /// "WAL node draining" in the error.
-    #[tokio::test(start_paused = true)]
-    async fn test_draining_surfaces_after_the_retry_budget() {
-        let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-        let seen = calls.clone();
-        let table = Table::new_with_handler("my_table", move |_request| {
-            seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-            http::Response::builder()
-                .status(503)
-                .body(r#"{"code":19,"error":"WAL node draining"}"#.to_string())
-                .unwrap()
-        });
-
-        let err = table.checkpoint_lsm().await.unwrap_err();
-        let message = err.to_string();
-        assert!(
-            matches!(&err, Error::Http { status_code: Some(s), .. } if s.as_u16() == 503),
-            "the 503 must surface as itself: {err:?}"
-        );
-        assert!(
-            message.contains("WAL node draining"),
-            "the server's own diagnosis must survive to the caller: {message}"
-        );
-        assert_eq!(
-            calls.load(std::sync::atomic::Ordering::SeqCst),
-            9,
-            "one call plus MAX_RETRIES, then it reports rather than spinning"
-        );
-    }
-
-    /// A long stall with nothing compacting must keep waiting, not fail. The
-    /// client cannot judge this: a checkpoint queued behind unrelated tables
-    /// on the pod-wide compactor pool reports exactly these numbers — flat
-    /// generations, an idle latch — as one whose merges are failing. The
-    /// deadline is the caller's.
-    #[tokio::test(start_paused = true)]
-    async fn test_checkpoint_waits_out_a_long_stall_rather_than_failing() {
-        let polls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-        let seen = polls.clone();
-        let table = Table::new_with_handler("my_table", move |request| {
-            let path = request.url().path().to_string();
-            if path.contains("flush_lsm") || path.contains("compact_lsm") {
-                return accepted();
-            }
-            // Flat for far longer than any bound this loop ever had, with
-            // `compacting: false` throughout — then it drains.
-            let n = seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-            ok_json(stats_body(if n < 40 { &[1, 2] } else { &[] }, false))
-        });
-
-        table
-            .checkpoint_lsm()
-            .await
-            .expect("a stall is the server being slow, not the client's call to make");
-        assert!(
-            polls.load(std::sync::atomic::Ordering::SeqCst) > 40,
-            "the loop must have kept polling well past the old ten-poll bound"
-        );
-    }
-
-    /// A pass already owns the latch on every outstanding bucket, so the loop
-    /// waits rather than piling on triggers it would only refuse. This is the
-    /// sole thing `compacting` is read for.
-    #[tokio::test(start_paused = true)]
-    async fn test_checkpoint_waits_while_a_pass_is_running() {
-        let polls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-        let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-        let seen_polls = polls.clone();
-        let seen_compacts = compacts.clone();
-        let table = Table::new_with_handler("my_table", move |request| {
-            let path = request.url().path().to_string();
-            if path.contains("flush_lsm") {
-                return accepted();
-            }
-            if path.contains("compact_lsm") {
-                seen_compacts.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-                return accepted();
-            }
-            // Latched for many polls, then done.
-            let n = seen_polls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-            ok_json(if n > 15 {
-                stats_body(&[], false)
-            } else {
-                stats_body(&[1], true)
-            })
-        });
-
-        table
-            .checkpoint_lsm()
-            .await
-            .expect("a running pass is progress, not a stall");
-        assert_eq!(
-            compacts.load(std::sync::atomic::Ordering::SeqCst),
-            0,
-            "never trigger against a bucket already compacting"
-        );
-    }
-
-    /// WAL off ⇒ `None`; WAL on ⇒ a fully populated `Some` with no field
-    /// defaulting to a zero it did not measure. `include_generation_rows`
-    /// rides in the body and is off unless asked for.
-    #[tokio::test]
-    async fn test_get_lsm_stats_round_trip() {
-        let table = Table::new_with_handler("my_table", |request| {
-            assert_eq!(request.url().path(), "/v1/table/my_table/get_lsm_stats/");
-            let body = request.body().unwrap().as_bytes().unwrap();
-            let body: serde_json::Value = serde_json::from_slice(body).unwrap();
-            assert_eq!(
-                body["include_generation_rows"], true,
-                "the flag must reach the server, not be silently dropped"
-            );
-            let response = serde_json::json!({
-                "lsm_stats": {
-                    "buckets": [{
-                        "shard_id": "b0",
-                        "status": "Active",
-                        "writer_epoch": 3,
-                        "manifest_version": 11,
-                        "current_generation": 9,
-                        "replay_after_wal_entry_position": 100,
-                        "wal_entry_position_last_seen": 140,
-                        "generations": [{ "generation": 8, "bytes": 4096, "rows": 30 }],
-                        "compacting": false,
-                        "memtables": [
-                            { "generation": 9, "rows": 12, "bytes": 900, "batches": 2,
-                              "indexes": ["vec_idx"] }
-                        ],
-                    }],
-                }
-            });
-            http::Response::builder()
-                .status(200)
-                .body(response.to_string())
-                .unwrap()
-        });
-
-        let stats = table
-            .get_lsm_stats(true)
-            .await
-            .unwrap()
-            .expect("a WAL-backed table reports Some");
-        let bucket = &stats.buckets[0];
-        assert_eq!(bucket.replay_after_wal_entry_position, 100);
-        assert_eq!(bucket.wal_entry_position_last_seen, 140);
-        assert!(!bucket.compacting);
-        assert_eq!(bucket.generations[0].generation, 8);
-        assert_eq!(bucket.generations[0].rows, Some(30));
-        // The line that answers "why is my fresh-tier vector search
-        // brute-force" — an absent index name is the whole explanation.
-        let memtables = bucket.memtables.as_ref().unwrap();
-        assert_eq!(memtables[0].indexes, vec!["vec_idx".to_string()]);
-    }
-
-    /// A 404 arrives as `TableNotFound`, not as a lost claim the loop
-    /// re-issues from flush until its cap. The two are distinguished by
-    /// status: 404 is "no such table", 421 is "this node holds no claim".
-    /// They shared 404 once, and the loop chased a name that never existed.
-    #[tokio::test(start_paused = true)]
-    async fn test_missing_table_is_not_read_as_a_lost_claim() {
-        let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-        let seen = calls.clone();
-        let table = Table::new_with_handler("my_table", move |_request| {
-            seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-            http::Response::builder()
-                .status(404)
-                .body(r#"{"code":4,"error":"Not found: Table not found: my_table"}"#.to_string())
-                .unwrap()
-        });
-
-        let err = table.checkpoint_lsm().await.unwrap_err();
-        assert!(
-            matches!(err, Error::TableNotFound { .. }),
-            "a missing table must say so: {err:?}"
-        );
-        assert_eq!(
-            calls.load(std::sync::atomic::Ordering::SeqCst),
-            1,
-            "no point re-claiming a table that does not exist"
-        );
-    }
-
-    /// A lost claim — 421, not 404 — does re-issue from flush, the call that
-    /// re-claims and replays.
-    #[tokio::test(start_paused = true)]
-    async fn test_registry_miss_reissues_from_flush() {
-        let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-        let seen = calls.clone();
-        let table = Table::new_with_handler("my_table", move |request| {
-            let path = request.url().path().to_string();
-            let n = seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-            if path.contains("flush_lsm") {
-                // First flush lands; the claim is then lost, and the
-                // re-issued flush succeeds.
-                return accepted();
-            }
-            if path.contains("compact_lsm") {
-                if n < 4 {
-                    return http::Response::builder()
-                        .status(421)
-                        .body(r#"{"code":19,"error":"table not claimed"}"#.to_string())
-                        .unwrap();
-                }
-                return accepted();
-            }
-            ok_json(stats_body(if n < 6 { &[1] } else { &[] }, false))
-        });
-
-        table
-            .checkpoint_lsm()
-            .await
-            .expect("a lost claim must be recovered by re-flushing, not surfaced");
-    }
-
-    #[tokio::test]
-    async fn test_get_lsm_stats_absent_when_wal_off() {
-        let table = Table::new_with_handler("my_table", |_request| {
-            http::Response::builder()
-                .status(200)
-                .body(serde_json::json!({ "lsm_stats": null }).to_string())
-                .unwrap()
-        });
-        assert!(table.get_lsm_stats(false).await.unwrap().is_none());
-    }
-
    #[tokio::test]
    async fn test_wait_for_index() {
        let table = _make_table_with_indices(0);
@@ -1,315 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-//! Converging a table's LSM write path into its base table.
-//!
-//! `checkpoint_lsm` seals once, then triggers compaction and watches
-//! generation numbers until the L0 that existed at the start is gone.
-//!
-//! The loop runs in the client, not the server: `compact_lsm` dispatches a
-//! pass and returns, so nothing holds a socket and a client can vanish
-//! mid-operation with nothing to reconcile. Completion is read from
-//! generation numbers in the shard manifest — durable state, unlike a count
-//! in a compact response, which a concurrent write invalidates.
-//!
-//! The target set is fixed at the start, so generations created *during* the
-//! checkpoint are ignored. That is what lets it terminate under write load,
-//! and what makes it best-effort: it converges the fresh tier as of some
-//! instant. Idempotent, abandonable at any point, safe on a cadence.
-//!
-//! No liveness bound — the caller owns the deadline. The compactor pool is
-//! shared pod-wide, so a checkpoint queued behind unrelated tables looks
-//! exactly like one that is merging.
-
-use std::collections::HashMap;
-use std::future::Future;
-use std::time::Duration;
-
-use crate::{Error, Result, Table};
-
-/// The HTTP status a failed request carried, if it carried one.
-///
-/// `None` for anything with no retry story: a `TableNotFound` that
-/// `check_table_response` already translated, or a connection failure that
-/// never reached the server. Both are terminal.
-fn status_of(e: &Error) -> Option<u16> {
-    #[cfg(feature = "remote")]
-    {
-        match e {
-            Error::Http {
-                status_code: Some(status),
-                ..
-            } => Some(status.as_u16()),
-            _ => None,
-        }
-    }
-    #[cfg(not(feature = "remote"))]
-    {
-        let _ = e;
-        None
-    }
-}
-
-/// 429 (latch held, pool saturated, or the pod replaying its WAL) and 503 (a
-/// draining node, or a proxy between here and it).
-///
-/// The status is the whole signal: the server deliberately keeps contention
-/// off 503, so a latch collision is a 429. A draining node *is* terminal, but
-/// it is also a 503 that stays a 503, so retrying spends one budget and then
-/// reports the server's own message — cheaper than parsing the body for the
-/// namespace code it would take to tell the two apart.
-fn is_retryable(e: &Error) -> bool {
-    matches!(status_of(e), Some(429 | 503))
-}
-
-/// 421: the owning node holds no claim. Only `flush` re-claims and replays,
-/// so this cannot be retried in place — the caller has to start over.
-fn is_lost_claim(e: &Error) -> bool {
-    status_of(e) == Some(421)
-}
-
-/// Interval between `get_lsm_stats` polls. One interval is roughly one
-/// compaction pass, the granularity at which the answer can change.
-///
-/// Fixed rather than configurable, matching `wait_for_index`. It costs
-/// nothing on an already-converged table and at most one interval of tail
-/// latency after the final pass lands.
-const POLL_INTERVAL: Duration = Duration::from_secs(5);
-
-/// Cap on re-issues from `flush` after a 421, so a crash-looping node cannot
-/// turn flush → compact → 421 → flush into a spin.
-///
-/// Deliberately not shared with [`MAX_RETRIES`]: a claim that keeps
-/// evaporating is a broken node, while contention is routine and wants a real
-/// budget. One shared counter let a merely contended table exhaust this cap
-/// and then blame a claim it never lost.
-const MAX_REISSUES: usize = 3;
-
-/// Retryable faults tolerated on a *single* request, reset on every success —
-/// scattered contention across a long checkpoint must not accumulate toward a
-/// cap. Roughly 16s of retrying against the backoff below.
-const MAX_RETRIES: usize = 8;
-
-/// Backoff between retries, doubling up to [`RETRY_BACKOFF_MAX`]. Latch
-/// contention clears in about the time one pass takes, so start small; a
-/// saturated pool wants the ceiling.
-const RETRY_BACKOFF_BASE: Duration = Duration::from_millis(100);
-const RETRY_BACKOFF_MAX: Duration = Duration::from_secs(5);
-
-/// Sleep before re-issuing a retryable request.
-async fn backoff(attempt: usize) {
-    let delay = RETRY_BACKOFF_BASE
-        .saturating_mul(1u32 << attempt.min(8) as u32)
-        .min(RETRY_BACKOFF_MAX);
-    tokio::time::sleep(delay).await;
-}
-
-/// Whether the drain loop finished or needs the table re-claimed first.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum CheckpointOutcome {
-    Done,
-    ReissueFromFlush,
-}
-
-/// What one LSM request produced: its value, or word that the owning node
-/// holds no claim and only `flush` can get it back.
-enum Attempt<T> {
-    Ok(T),
-    ReissueFromFlush,
-}
-
-/// Issue one LSM request, retrying in place while the fault is retryable.
-///
-/// The two recoverable faults have separate budgets: contention clears on its
-/// own and retries here against [`MAX_RETRIES`], while a 421 needs `flush` to
-/// re-claim, which only the caller can drive.
-///
-/// An exhausted budget propagates the last error *as itself* rather than a
-/// synthesized one — "429 after nine tries" beats "checkpoint failed", and a
-/// draining node arrives carrying the server's own message.
-async fn issue<T, F, Fut>(mut call: F) -> Result<Attempt<T>>
-where
-    F: FnMut() -> Fut,
-    Fut: Future<Output = Result<T>>,
-{
-    let mut retries = 0;
-    loop {
-        let e = match call().await {
-            Ok(value) => return Ok(Attempt::Ok(value)),
-            Err(e) => e,
-        };
-        if is_lost_claim(&e) {
-            return Ok(Attempt::ReissueFromFlush);
-        }
-        if !is_retryable(&e) || retries >= MAX_RETRIES {
-            return Err(e);
-        }
-        backoff(retries).await;
-        retries += 1;
-    }
-}
-
-/// Drive [`Table::checkpoint_lsm`]: seal once, fix the target watermark
-/// from the resulting L0, then trigger and poll until it drains.
-pub(crate) async fn checkpoint_lsm(table: &Table) -> Result<()> {
-    for reissue in 0..=MAX_REISSUES {
-        // The seal turns everything written before this call into a
-        // generation, so the watermark has to be read after it. Idempotent:
-        // sealing an empty memtable is a no-op, so a re-issue does not churn
-        // empty generations.
-        match issue(|| table.flush_lsm()).await? {
-            Attempt::Ok(()) => {}
-            Attempt::ReissueFromFlush => {
-                backoff(reissue).await;
-                continue;
-            }
-        }
-
-        let stats = match issue(|| table.get_lsm_stats(false)).await? {
-            Attempt::Ok(stats) => stats,
-            Attempt::ReissueFromFlush => {
-                backoff(reissue).await;
-                continue;
-            }
-        };
-        let Some(stats) = stats else {
-            // Not WAL-backed; `flush_lsm` would have errored first but for a race.
-            return Ok(());
-        };
-        let targets: HashMap<String, u64> = stats
-            .buckets
-            .iter()
-            .filter_map(|b| Some((b.shard_id.clone(), b.newest_generation()?)))
-            .collect();
-        if targets.is_empty() {
-            return Ok(());
-        }
-
-        match drain_to_targets(table, &targets).await? {
-            CheckpointOutcome::Done => return Ok(()),
-            CheckpointOutcome::ReissueFromFlush => {
-                backoff(reissue).await;
-                continue;
-            }
-        }
-    }
-    Err(Error::Runtime {
-        message: "checkpoint_lsm: the owning node kept losing its claim; \
-                  re-issued from flush the maximum number of times"
-            .into(),
-    })
-}
-
-/// Trigger and poll until no bucket holds a generation at or below its
-/// target.
-///
-/// No liveness bound, deliberately. The pod-wide compactor pool (a semaphore
-/// of 2 by default, shared across every table on the node) is taken *inside*
-/// the pass, after the bucket latch, so a checkpoint queued behind unrelated
-/// tables is indistinguishable from one that is merging. An idle-poll counter
-/// here could only ever have fired on a table that would have finished.
-async fn drain_to_targets(
-    table: &Table,
-    targets: &HashMap<String, u64>,
-) -> Result<CheckpointOutcome> {
-    loop {
-        let stats = match issue(|| table.get_lsm_stats(false)).await? {
-            Attempt::Ok(stats) => stats,
-            Attempt::ReissueFromFlush => return Ok(CheckpointOutcome::ReissueFromFlush),
-        };
-        let Some(stats) = stats else {
-            return Ok(CheckpointOutcome::Done);
-        };
-        // `compacting` is the bucket's compaction latch, held from dispatch
-        // until the pass ends — including while it waits on the pod-wide
-        // permit. So it answers one question only: do not pile on. Buckets
-        // with nothing outstanding are skipped, not counted as idle.
-        let mut outstanding = 0;
-        let mut all_compacting = true;
-        for b in &stats.buckets {
-            let Some(target) = targets.get(&b.shard_id) else {
-                continue;
-            };
-            let n = b.outstanding_generations(*target);
-            if n > 0 {
-                outstanding += n;
-                all_compacting &= b.compacting;
-            }
-        }
-        if outstanding == 0 {
-            return Ok(CheckpointOutcome::Done);
-        }
-
-        if !all_compacting {
-            match table.compact_lsm().await {
-                Ok(()) => {}
-                Err(e) if is_lost_claim(&e) => return Ok(CheckpointOutcome::ReissueFromFlush),
-                Err(e) if !is_retryable(&e) => return Err(e),
-                // A 429 here means the server could latch no bucket at all,
-                // which the poll above already handles. Not retried in place:
-                // the latch it would contend for is the one doing the work, so
-                // fall through and re-read — `POLL_INTERVAL` is the backoff.
-                Err(_) => {}
-            }
-        }
-        tokio::time::sleep(POLL_INTERVAL).await;
-    }
-}
-
-#[cfg(all(test, feature = "remote"))]
-mod tests {
-    use super::*;
-
-    fn http(status: u16) -> Error {
-        Error::Http {
-            source: "server said no".into(),
-            request_id: "rid".into(),
-            status_code: reqwest::StatusCode::from_u16(status).ok(),
-        }
-    }
-
-    /// Every status the loop acts on. The two predicates are checked together
-    /// because their overlap is what would be wrong: a status must never be
-    /// both, and 421 in particular must not read as retryable — retrying it in
-    /// place re-issues the call that just said the node holds no claim.
-    #[test]
-    fn taxonomy_round_trips() {
-        for status in [429, 503] {
-            assert!(is_retryable(&http(status)), "{status} must retry");
-            assert!(
-                !is_lost_claim(&http(status)),
-                "{status} is not a lost claim"
-            );
-        }
-        assert!(is_lost_claim(&http(421)), "a lost claim must re-claim");
-        assert!(
-            !is_retryable(&http(421)),
-            "retrying a lost claim in place only asks the same node again"
-        );
-        for status in [400, 404, 409, 500] {
-            assert!(!is_retryable(&http(status)), "{status} is terminal");
-            assert!(!is_lost_claim(&http(status)), "{status} is terminal");
-        }
-    }
-
-    /// An error carrying no status has no retry story and must be terminal —
-    /// a connection that never reached the server, or a `TableNotFound` that
-    /// `check_table_response` translated before the loop saw it.
-    #[test]
-    fn errors_without_a_status_are_terminal() {
-        let no_status = Error::Http {
-            source: "connection reset".into(),
-            request_id: "rid".into(),
-            status_code: None,
-        };
-        assert!(!is_retryable(&no_status));
-        assert!(!is_lost_claim(&no_status));
-
-        let translated = Error::TableNotFound {
-            name: "t".into(),
-            source: "gone".into(),
-        };
-        assert!(!is_retryable(&translated));
-        assert!(!is_lost_claim(&translated));
-    }
-}
@@ -1,162 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-//! Live per-bucket LSM state — the shape [`crate::Table::get_lsm_stats`]
-//! returns and [`super::checkpoint`] polls.
-//!
-//! Nothing here is derived: sums and differences (total L0 bytes, WAL lag)
-//! are the caller's to compute. There is no "WAL is off" shape — that case is
-//! `None`, because a struct of zeros would read as measurements.
-
-use serde::Deserialize;
-
-/// One flushed L0 generation.
-#[derive(Debug, Clone, Deserialize)]
-pub struct GenerationStats {
-    pub generation: u64,
-    pub bytes: u64,
-    /// Present only when `include_generation_rows` was requested. Off by
-    /// default because each count opens an uncached Lance dataset, and the
-    /// checkpoint loop polls this route needing only generation numbers.
-    #[serde(default)]
-    pub rows: Option<u64>,
-}
-
-/// One in-memory memtable.
-#[derive(Debug, Clone, Deserialize)]
-pub struct MemtableStats {
-    pub generation: u64,
-    pub rows: u64,
-    pub bytes: u64,
-    pub batches: u64,
-    /// Names of the indexes this memtable carries. An absent name is the whole
-    /// answer to "why is my fresh-tier search on that column brute-force".
-    pub indexes: Vec<String>,
-}
-
-/// Live state of one bucket. A table is N buckets on one node; flattening to
-/// a single number hides the one hot bucket that is usually why someone
-/// opened this endpoint.
-#[derive(Debug, Clone, Deserialize)]
-pub struct BucketStats {
-    pub shard_id: String,
-    /// `Active` | `Sealed` (drop-table 2PC in flight).
-    pub status: String,
-    pub writer_epoch: u64,
-    pub manifest_version: u64,
-    pub current_generation: u64,
-    pub replay_after_wal_entry_position: u64,
-    pub wal_entry_position_last_seen: u64,
-    pub generations: Vec<GenerationStats>,
-    /// Whether a pass owns this bucket's compaction latch right now. Says *a*
-    /// driver is running, not *whose*, and the latch is held from dispatch —
-    /// including while the pass queues for a pod-wide compactor permit. Read
-    /// it as "do not pile on", never as "mine is progressing".
-    pub compacting: bool,
-    /// Oldest first, active last. Absent for a `Sealed` bucket, whose
-    /// in-memory state is torn down.
-    #[serde(default)]
-    pub memtables: Option<Vec<MemtableStats>>,
-}
-
-impl BucketStats {
-    /// The newest flushed generation, or `None` when L0 is empty.
-    pub(crate) fn newest_generation(&self) -> Option<u64> {
-        self.generations.iter().map(|g| g.generation).max()
-    }
-
-    /// How many generations at or below `target` are still in L0.
-    ///
-    /// A count, not a boolean: one pass drains a bounded prefix rather than
-    /// the whole target set, so a boolean would read as "no progress" for
-    /// every pass but the last. Compaction drains oldest-first, so this
-    /// decreases monotonically.
-    pub(crate) fn outstanding_generations(&self, target: u64) -> usize {
-        self.generations
-            .iter()
-            .filter(|g| g.generation <= target)
-            .count()
-    }
-}
-
-/// Live LSM state, one entry per bucket.
-#[derive(Debug, Clone, Deserialize)]
-pub struct LsmStats {
-    pub buckets: Vec<BucketStats>,
-}
-
-/// Server-side JSON envelope for `get_lsm_stats`. `lsm_stats` is null when
-/// the table has no LSM write path.
-#[derive(Debug, Deserialize)]
-pub(crate) struct GetLsmStatsResponse {
-    #[serde(default)]
-    pub lsm_stats: Option<LsmStats>,
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    fn bucket(shard: &str, generations: &[u64], compacting: bool) -> BucketStats {
-        BucketStats {
-            shard_id: shard.into(),
-            status: "Active".into(),
-            writer_epoch: 1,
-            manifest_version: 1,
-            current_generation: generations.iter().max().copied().unwrap_or(0) + 1,
-            replay_after_wal_entry_position: 0,
-            wal_entry_position_last_seen: 0,
-            generations: generations
-                .iter()
-                .map(|g| GenerationStats {
-                    generation: *g,
-                    bytes: 1,
-                    rows: None,
-                })
-                .collect(),
-            compacting,
-            memtables: None,
-        }
-    }
-
-    /// The target watermark is the newest generation at the start, and a
-    /// generation created after it must not hold the loop open — that is why
-    /// the predicate terminates under write load.
-    #[test]
-    fn newer_generations_do_not_extend_the_target() {
-        let start = bucket("b0", &[7, 8], false);
-        let target = start.newest_generation().expect("L0 is non-empty");
-        assert_eq!(target, 8);
-
-        // Compaction drained 7 and 8; 9 and 10 arrived while it ran.
-        let later = bucket("b0", &[9, 10], false);
-        assert_eq!(
-            later.outstanding_generations(target),
-            0,
-            "generations above the target are somebody else's problem"
-        );
-
-        // Still holding 8 means still outstanding.
-        assert_eq!(
-            bucket("b0", &[8, 9], false).outstanding_generations(target),
-            1
-        );
-    }
-
-    /// The metric counts generations, not buckets: a pass drains a bounded
-    /// prefix, so one bucket going 3 → 2 → 1 → 0 is three steps.
-    #[test]
-    fn progress_is_measured_in_generations() {
-        let target = 3;
-        let counts: Vec<usize> = [&[1u64, 2, 3][..], &[2, 3][..], &[3][..], &[][..]]
-            .iter()
-            .map(|gens| bucket("b0", gens, false).outstanding_generations(target))
-            .collect();
-        assert_eq!(counts, vec![3, 2, 1, 0]);
-    }
-
-    #[test]
-    fn empty_l0_has_no_target() {
-        assert!(bucket("b0", &[], false).newest_generation().is_none());
-    }
-}
@@ -1161,7 +1161,7 @@ mod lsm_tests {
            .unwrap();
        let fts_index = table.list_indices().await.unwrap()[0].name.clone();
        table
-            .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![fts_index]))
+            .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes([fts_index]))
            .await
            .unwrap();

@@ -1254,7 +1254,7 @@ mod lsm_tests {
            .unwrap();
        let vec_index = table.list_indices().await.unwrap()[0].name.clone();
        table
-            .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![vec_index]))
+            .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes([vec_index]))
            .await
            .unwrap();

@@ -29,7 +29,6 @@ use arrow_schema::{DataType, Schema as ArrowSchema, SchemaRef};
 use lance::Dataset;
 use lance::dataset::mem_wal::{
    DatasetMemWalExt, ShardWriter, ShardWriterConfig, evaluate_sharding_spec,
-    validate_maintained_indexes,
 };
 use lance::index::DatasetIndexExt;
 use lance_core::datatypes::Schema as LanceSchema;
@@ -38,9 +37,8 @@ use tokio::sync::RwLock;
 use uuid::Uuid;

 use crate::error::{Error, Result};
-use crate::index::IndexConfig;
 use crate::table::merge::{MergeInsertBuilder, MergeResult};
-use crate::table::{BaseTable, LsmWriteSpec, NativeTable};
+use crate::table::{LsmWriteSpec, NativeTable};

 /// Spec id of the sole sharding spec installed by [`set_lsm_write_spec`].
 /// Must match Lance's `InitializeMemWalBuilder` (`SHARDING_SPEC_ID`).
@@ -82,44 +80,32 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
        }
    }

-    // Before the builder borrows the dataset clone. `list_indices` merges an
-    // index's segments into one entry, so the result needs no dedup.
-    let maintained_indexes = {
-        let dataset = table.dataset.get().await?;
-        resolve_maintained_indexes(
-            &dataset,
-            &table.list_indices().await?,
-            spec.maintained_indexes(),
-        )
-        .await?
-    };
-
    let mut dataset = (*table.dataset.get().await?).clone();
    let mut builder = dataset.initialize_mem_wal();
-    let writer_config_defaults = match spec {
+    let (maintained_indexes, writer_config_defaults) = match spec {
        LsmWriteSpec::Bucket {
            column,
            num_buckets,
+            maintained_indexes,
            writer_config_defaults,
-            ..
        } => {
            builder = builder.bucket_sharding(column, num_buckets);
-            writer_config_defaults
+            (maintained_indexes, writer_config_defaults)
        }
        LsmWriteSpec::Identity {
            column,
+            maintained_indexes,
            writer_config_defaults,
-            ..
        } => {
            builder = builder.identity_sharding(column);
-            writer_config_defaults
+            (maintained_indexes, writer_config_defaults)
        }
        LsmWriteSpec::Unsharded {
+            maintained_indexes,
            writer_config_defaults,
-            ..
        } => {
            builder = builder.unsharded();
-            writer_config_defaults
+            (maintained_indexes, writer_config_defaults)
        }
    };
    builder = builder.maintained_indexes(maintained_indexes);
@@ -131,58 +117,6 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
    Ok(())
 }

-/// Resolve a spec's maintained-index selection against `indices`, as reported
-/// by [`Table::list_indices`](crate::Table::list_indices).
-///
-/// `None` means every index on the table, snapshotted now. Lance validates
-/// either selection against its shard-writer rules, so a spec that installs is
-/// one the MemWAL can open.
-///
-/// An unmaintainable index fails an inferred set rather than being dropped from
-/// it — dropping would leave the caller believing it is maintained.
-async fn resolve_maintained_indexes(
-    dataset: &Dataset,
-    indices: &[IndexConfig],
-    requested: Option<&[String]>,
-) -> Result<Vec<String>> {
-    let Some(requested) = requested else {
-        let all: Vec<String> = indices.iter().map(|index| index.name.clone()).collect();
-        validate_maintained_indexes(dataset, &all)
-            .await
-            .map_err(|source| Error::InvalidInput {
-                message: format!(
-                    "cannot maintain every index on this table: {source}. Set \
-                     maintained_indexes explicitly to choose from {}",
-                    index_name_list(indices),
-                ),
-            })?;
-        return Ok(all);
-    };
-    for name in requested {
-        if !indices.iter().any(|index| &index.name == name) {
-            return Err(Error::InvalidInput {
-                message: format!(
-                    "maintained index '{}' does not exist on this table; it has {}",
-                    name,
-                    index_name_list(indices),
-                ),
-            });
-        }
-    }
-    validate_maintained_indexes(dataset, requested).await?;
-    Ok(requested.to_vec())
-}
-
-/// Index names for an error message.
-fn index_name_list(indices: &[IndexConfig]) -> String {
-    if indices.is_empty() {
-        return "no indexes".to_string();
-    }
-    let mut names: Vec<&str> = indices.iter().map(|index| index.name.as_str()).collect();
-    names.sort_unstable();
-    format!("[{}]", names.join(", "))
-}
-
 // =============================================================================
 // unset_lsm_write_spec
 // =============================================================================
@@ -84,8 +84,9 @@ pub(super) async fn create_lsm_plan(
    let pk_columns = pk_columns(&ds_ref)?;
    // The base index an indexed arm relies on may lag compaction; resolve it so the
    // snapshot retains SSTables the index has not yet caught up to.
-    let arm_indexes = arm_maintained_index_names(&ds_ref, &query, &details).await?;
-    let (snapshots, in_memory) = build_read_context(table, &ds_ref, &details, &arm_indexes).await?;
+    let arm_index = arm_maintained_index_name(&ds_ref, &query, &details).await?;
+    let (snapshots, in_memory) =
+        build_read_context(table, &ds_ref, &details, arm_index.as_deref()).await?;

    let limit = query.base.limit;
    let offset = query.base.offset;
@@ -231,36 +232,28 @@ fn pk_columns(dataset: &Dataset) -> Result<Vec<String>> {
    Ok(pk)
 }

-/// Per-shard SSTable exclusion watermark: the generation at or below which
-/// SSTables are safe to drop for this query.
-///
-/// A generation is droppable only once it is compacted into the base table AND
-/// covered by the catch-up of every index the query relies on, so the watermark
-/// is the minimum across `index_names`. Gating on fewer than all of them would
-/// drop SSTables holding rows an uncounted index has not yet indexed, and that
-/// arm would silently return fewer rows.
-///
-/// See [`arm_maintained_index_names`] for which indexes are collected today: a
-/// vector search with a scalar prefilter is not yet among them.
-///
-/// An empty `index_names` (a plain scan) uses the compaction watermark alone.
-/// First occurrence per shard mirrors Lance's `compacted_generation_for_shard`.
+/// Per-shard SSTable exclusion watermark: the generation at or below which SSTables
+/// are safe to drop for this arm. A generation is droppable only once it is
+/// compacted into the base table AND covered by `index_name`'s catch-up (for an
+/// indexed arm); a plain scan (`index_name == None`) uses the compaction watermark
+/// alone. Capping at the index catch-up keeps rows the base index has not yet
+/// indexed visible through their SSTable. First occurrence per shard mirrors Lance's
+/// `compacted_generation_for_shard`.
 fn exclusion_watermarks(
    details: &MemWalIndexDetails,
-    index_names: &[String],
+    index_name: Option<&str>,
 ) -> HashMap<Uuid, u64> {
    let mut exclude: HashMap<Uuid, u64> = HashMap::new();
    for entry in &details.compacted_sstables {
        let mut watermark = entry.generation;
-        for name in index_names {
-            if let Some(caught_up) = details
+        if let Some(name) = index_name
+            && let Some(caught_up) = details
                .index_catchup
                .iter()
-                .find(|icp| icp.index_name == *name)
+                .find(|icp| icp.index_name == name)
                .and_then(|icp| icp.caught_up_generation_for_shard(&entry.shard_id))
-            {
-                watermark = watermark.min(caught_up);
-            }
+        {
+            watermark = watermark.min(caught_up);
        }
        exclude.entry(entry.shard_id).or_insert(watermark);
    }
@@ -278,9 +271,9 @@ async fn build_read_context(
    table: &NativeTable,
    dataset: &Dataset,
    details: &MemWalIndexDetails,
-    index_names: &[String],
+    index_name: Option<&str>,
 ) -> Result<(Vec<ShardSnapshot>, HashMap<Uuid, InMemoryMemTables>)> {
-    let exclude = exclusion_watermarks(details, index_names);
+    let exclude = exclusion_watermarks(details, index_name);

    let shard_ids = dataset.list_mem_wal_latest_shard_ids().await?;
    // Use the dataset's own object store (not `ObjectStore::from_uri`, which
@@ -494,33 +487,19 @@ async fn index_maintained(
    }))
 }

-/// Every maintained base index this query relies on, used to gate SSTable
-/// exclusion by index catch-up.
-///
-/// Returns a list because the watermark must be the lowest across every index a
-/// query relies on. Today it never holds more than one: `reject_unsupported`
-/// refuses hybrid search, so the vector and full-text arms are mutually
-/// exclusive.
-///
-/// The case that is genuinely multi-index -- a vector search with a scalar or
-/// bitmap prefilter -- is **not collected yet**. Identifying those needs the
-/// planner's chosen indexes, not the columns the filter names, and no Lance API
-/// exposes them. Until it does, such a query is gated on its vector index alone.
-///
-/// Empty for a plain scan, or when no maintained index covers the searched
-/// column.
-async fn arm_maintained_index_names(
+/// The maintained base index the query's arm relies on (vector index for ANN, FTS
+/// index for full-text), used to gate SSTable compaction exclusion by index catch-up.
+/// `None` for a plain scan or when no maintained index covers the searched column.
+async fn arm_maintained_index_name(
    dataset: &Dataset,
    query: &VectorQueryRequest,
    details: &MemWalIndexDetails,
-) -> Result<Vec<String>> {
+) -> Result<Option<String>> {
    use lance::index::DatasetIndexExt;
-
-    // Each arm's searched column, the index-detail type it relies on, and a
+    // Resolve the arm's searched column, the index-detail type it relies on, and a
    // label for diagnostics — catch-up is taken from the vector/FTS index
    // specifically, not a BTree on the same column.
-    let mut arms: Vec<(String, &str, &str)> = Vec::new();
-    if !query.query_vector.is_empty() {
+    let (column, type_url_suffix, arm) = if !query.query_vector.is_empty() {
        let arrow_schema = ArrowSchema::from(dataset.schema());
        let column = match &query.column {
            Some(column) => column.clone(),
@@ -529,43 +508,31 @@ async fn arm_maintained_index_names(
                default_vector_column(&arrow_schema, dim)?
            }
        };
-        arms.push((column, "VectorIndexDetails", "vector"));
-    }
-    if let Some(fts) = &query.base.full_text_search
-        && let Some(column) = fts.columns().into_iter().next()
-    {
-        arms.push((column, "InvertedIndexDetails", "full-text"));
-    }
-    if arms.is_empty() {
-        return Ok(Vec::new());
-    }
-
-    let indices = dataset.load_indices().await?;
-    let mut names = Vec::with_capacity(arms.len());
-    for (column, type_url_suffix, arm) in arms {
-        let Some(field) = dataset.schema().field(&column) else {
-            continue;
-        };
-        let segment_names: Vec<String> = indices
-            .iter()
-            .filter(|idx| {
-                idx.fields.contains(&field.id)
-                    && idx
-                        .index_details
-                        .as_ref()
-                        .is_some_and(|d| d.type_url.ends_with(type_url_suffix))
-            })
-            .map(|idx| idx.name.clone())
-            .collect();
-        if let Some(name) =
-            resolve_single_index(segment_names, &details.maintained_indexes, arm, &column)?
-        {
-            names.push(name);
+        (column, "VectorIndexDetails", "vector")
+    } else if let Some(fts) = &query.base.full_text_search {
+        match fts.columns().into_iter().next() {
+            Some(column) => (column, "InvertedIndexDetails", "full-text"),
+            None => return Ok(None),
        }
-    }
-    names.sort();
-    names.dedup();
-    Ok(names)
+    } else {
+        return Ok(None);
+    };
+    let Some(field) = dataset.schema().field(&column) else {
+        return Ok(None);
+    };
+    let indices = dataset.load_indices().await?;
+    let segment_names: Vec<String> = indices
+        .iter()
+        .filter(|idx| {
+            idx.fields.contains(&field.id)
+                && idx
+                    .index_details
+                    .as_ref()
+                    .is_some_and(|d| d.type_url.ends_with(type_url_suffix))
+        })
+        .map(|idx| idx.name.clone())
+        .collect();
+    resolve_single_index(segment_names, &details.maintained_indexes, arm, &column)
 }

 /// Resolve the single logical index from the names of its matching physical
@@ -767,85 +734,24 @@ mod tests {
        };

        // Plain scan: drop every compacted generation (through 5).
-        assert_eq!(exclusion_watermarks(&details, &[]).get(&shard), Some(&5));
+        assert_eq!(exclusion_watermarks(&details, None).get(&shard), Some(&5));

        // FTS arm with a lagging index: exclusion is capped at the index catch-up
        // (2), so SSTable generations 3..=5 are retained until the index covers
        // them — otherwise those documents would silently vanish from FTS results.
        assert_eq!(
-            exclusion_watermarks(&details, &["fts_idx".to_string()]).get(&shard),
+            exclusion_watermarks(&details, Some("fts_idx")).get(&shard),
            Some(&2)
        );

        // A caught-up index — or one untracked in index_catchup — falls back to the
        // compaction watermark.
        assert_eq!(
-            exclusion_watermarks(&details, &["caught_up_idx".to_string()]).get(&shard),
+            exclusion_watermarks(&details, Some("caught_up_idx")).get(&shard),
            Some(&5)
        );
    }

-    /// A hybrid search reads a vector and a full-text index, and either may lag.
-    /// Retaining to the lower of the two is what keeps both arms complete;
-    /// gating on one alone would drop SSTables the other has not indexed.
-    #[test]
-    fn exclusion_watermark_takes_the_minimum_across_every_index_used() {
-        let shard = Uuid::from_u128(1);
-        let details = MemWalIndexDetails {
-            compacted_sstables: vec![CompactedSsTable::new(shard, 9)],
-            index_catchup: vec![
-                IndexCatchupProgress::new(
-                    "vec_idx".to_string(),
-                    vec![CompactedSsTable::new(shard, 7)],
-                ),
-                IndexCatchupProgress::new(
-                    "fts_idx".to_string(),
-                    vec![CompactedSsTable::new(shard, 4)],
-                ),
-            ],
-            maintained_indexes: vec!["vec_idx".to_string(), "fts_idx".to_string()],
-            ..Default::default()
-        };
-
-        // Each index alone stops at its own catch-up.
-        assert_eq!(
-            exclusion_watermarks(&details, &["vec_idx".to_string()]).get(&shard),
-            Some(&7)
-        );
-        assert_eq!(
-            exclusion_watermarks(&details, &["fts_idx".to_string()]).get(&shard),
-            Some(&4)
-        );
-
-        // Used together, the lower one governs regardless of order.
-        let both = ["vec_idx".to_string(), "fts_idx".to_string()];
-        assert_eq!(exclusion_watermarks(&details, &both).get(&shard), Some(&4));
-        let reversed = ["fts_idx".to_string(), "vec_idx".to_string()];
-        assert_eq!(
-            exclusion_watermarks(&details, &reversed).get(&shard),
-            Some(&4)
-        );
-    }
-
-    /// An index with no catch-up entry contributes no cap today, so a lagging
-    /// sibling must still govern rather than being widened by the untracked one.
-    #[test]
-    fn an_untracked_index_does_not_widen_a_lagging_sibling() {
-        let shard = Uuid::from_u128(1);
-        let details = MemWalIndexDetails {
-            compacted_sstables: vec![CompactedSsTable::new(shard, 9)],
-            index_catchup: vec![IndexCatchupProgress::new(
-                "fts_idx".to_string(),
-                vec![CompactedSsTable::new(shard, 4)],
-            )],
-            maintained_indexes: vec!["fts_idx".to_string(), "untracked_idx".to_string()],
-            ..Default::default()
-        };
-
-        let both = ["fts_idx".to_string(), "untracked_idx".to_string()];
-        assert_eq!(exclusion_watermarks(&details, &both).get(&shard), Some(&4));
-    }
-
    #[test]
    fn resolve_single_index_dedupes_segments() {
        let maintained = vec!["fts_idx".to_string()];
@@ -10,7 +10,7 @@ use arrow_array::{
 use arrow_schema::{DataType, Field, Fields, Schema};
 use futures::TryStreamExt;
 use lance::Dataset;
-use lance_file::version::{ConcreteFileVersion, LanceFileVersion};
+use lance_file::version::LanceFileVersion;
 use lancedb::{
    Connection, Error, Result, Table,
    blob::{BlobRangeRequest, blob},
@@ -61,7 +61,7 @@ async fn create_inline_blob_table(
    Ok(table)
 }

-async fn storage_format_version(table: &Table) -> ConcreteFileVersion {
+async fn storage_format_version(table: &Table) -> LanceFileVersion {
    table
        .as_native()
        .unwrap()
@@ -69,14 +69,9 @@ async fn storage_format_version(table: &Table) -> ConcreteFileVersion {
        .await
        .unwrap()
        .data_storage_format
-        .lance_file_format()
-}
-
-fn supports_blob_v2(version: ConcreteFileVersion) -> bool {
-    matches!(
-        version,
-        ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3
-    )
+        .lance_file_version()
+        .unwrap()
+        .resolve()
 }

 async fn uses_stable_row_ids(table: &Table) -> bool {
@@ -117,7 +112,7 @@ async fn declaring_blob_column_bumps_format_and_enables_stable_row_ids() -> Resu
        .execute()
        .await?;

-    assert!(supports_blob_v2(storage_format_version(&table).await));
+    assert!(storage_format_version(&table).await >= LanceFileVersion::V2_2);
    assert!(uses_stable_row_ids(&table).await);
    Ok(())
 }
@@ -132,7 +127,7 @@ async fn explicit_stable_row_id_setting_wins_over_blob_default() -> Result<()> {
        .execute()
        .await?;

-    assert!(supports_blob_v2(storage_format_version(&table).await));
+    assert!(storage_format_version(&table).await >= LanceFileVersion::V2_2);
    assert!(!uses_stable_row_ids(&table).await);
    Ok(())
 }
@@ -144,7 +139,7 @@ async fn non_blob_table_keeps_default_format_and_row_id_setting() -> Result<()>
    let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
    let table = db.create_empty_table("t", schema).execute().await?;

-    assert!(!supports_blob_v2(storage_format_version(&table).await));
+    assert!(storage_format_version(&table).await < LanceFileVersion::V2_2);
    assert!(!uses_stable_row_ids(&table).await);
    Ok(())
 }
@@ -176,7 +171,7 @@ async fn creating_with_blob_data_bumps_format() -> Result<()> {
    .unwrap();
    let table = db.create_table("t", batch).execute().await?;

-    assert!(supports_blob_v2(storage_format_version(&table).await));
+    assert!(storage_format_version(&table).await >= LanceFileVersion::V2_2);
    assert!(uses_stable_row_ids(&table).await);
    assert_eq!(table.count_rows(None).await?, 1);
    Ok(())
@@ -286,7 +281,7 @@ async fn connection_level_stable_row_id_setting_wins_over_blob_default() -> Resu
        .execute()
        .await?;

-    assert!(supports_blob_v2(storage_format_version(&table).await));
+    assert!(storage_format_version(&table).await >= LanceFileVersion::V2_2);
    assert!(!uses_stable_row_ids(&table).await);
    Ok(())
 }
@@ -302,7 +297,7 @@ async fn namespace_create_applies_blob_defaults() -> Result<()> {
        .execute()
        .await?;

-    assert!(supports_blob_v2(storage_format_version(&table).await));
+    assert!(storage_format_version(&table).await >= LanceFileVersion::V2_2);
    assert!(uses_stable_row_ids(&table).await);
    Ok(())
 }
@@ -479,7 +474,7 @@ async fn fetch_blobs_round_trips_nested_blob_column() -> Result<()> {
    let batch = RecordBatch::try_new(schema, vec![Arc::new(info_array) as ArrayRef]).unwrap();
    let table = db.create_table("t", batch).execute().await?;

-    assert!(supports_blob_v2(storage_format_version(&table).await));
+    assert!(storage_format_version(&table).await >= LanceFileVersion::V2_2);
    assert!(uses_stable_row_ids(&table).await);

    let ids = collect_row_ids(&table).await?;
@@ -1310,7 +1305,7 @@ async fn optimize_preserves_blob_v2_null_and_empty_distinction() -> Result<()> {
        .await?;
    table.add(null_empty_input_batch()).execute().await?;
    assert!(
-        supports_blob_v2(storage_format_version(&table).await),
+        storage_format_version(&table).await >= LanceFileVersion::V2_2,
        "blob v2 columns require storage >= 2.2"
    );