feat: refresh materialized views (#4010 )

A declared view holds no rows; refresh computes them. It pins one source version, brings the view to exactly the definition's result at that version, and records the version as a watermark in the view's schema metadata. It is incremental when it can reconcile what changed: appended rows are computed and appended, and rows the source deleted or updated are found by the lance delta and evicted by their __source_row_id provenance, the updated ones recomputed in the same commit. Compaction rearranges rows without changing them, so its outputs cost nothing -- which is what keeps routine background compaction from rebuilding the view. A vacuumed watermark, a delta the transaction-log walk cannot classify, a Legacy-storage source, or more staged ids than a fixed cap all fall back to a rebuild; rebuilding an indexed view swaps every fragment in one Update, so readers never see it unindexed or empty. Concurrent refreshes serialize at commit -- each carries the same sentinel row id in its inserted-rows filter, so the loser lands nothing. On the append path the watermark moves in a follow-up commit, so a crash between the two re-appends those rows. Bumps lance to v11.0.0-beta.19 for the delta reader.
chore: update lance dependency to v11.0.0-beta.19 (#4025 )
2026-08-26 07:58:31 +00:00 · 2026-08-21 22:39:47 -07:00 · 2026-08-21 21:43:45 -07:00 · 2026-08-21 16:44:21 -07:00 · 2026-08-21 16:39:13 -07:00 · 2026-08-21 16:31:34 -07:00
144 changed files with 11692 additions and 2365 deletions
@@ -5,7 +5,3 @@ This directory contains repo-scoped code agent skills for the LanceDB project.
 Each skill is a folder that contains a required `SKILL.md` and optional bundled resources.

 Codex discovers skills from `.agents/skills` in the current working directory and parent directories.
-
-The `lancedb` skill lives in the `plugins/lancedb` plugin (see `plugins/lancedb/skills/lancedb`)
-so it can be installed via the plugin marketplaces (`.claude-plugin/marketplace.json` and
-`.agents/plugins/marketplace.json`); the `lancedb` entry here is a symlink into that plugin.
@@ -1 +0,0 @@
-../../plugins/lancedb/skills/lancedb
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.38.0-beta.2"
+current_version = "0.38.0-beta.3"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
@@ -9,6 +9,18 @@ debug = true
 codegen-units = 16
 lto = "thin"

+[profile.release-no-lto]
+inherits = "release"
+debug = true
+lto = false
+# Prioritize compile time when LTO is not relevant to the measurement.
+codegen-units = 16
+
+[profile.bench]
+inherits = "release"
+lto = "thin"
+codegen-units = 16
+
 [target.'cfg(all())']
 rustflags = [
    "-Wclippy::all",
@@ -17,6 +17,18 @@ updates:
    # newer minimum versions.
    versioning-strategy: lockfile-only
    groups:
+      # The arrow-rs and datafusion crates are released in lockstep and have to
+      # move together, so keep them in one PR instead of one per sub-crate.
+      # Listed first: a dependency joins the first group it matches.
+      arrow-datafusion:
+        patterns:
+          - arrow
+          - arrow-*
+          - parquet
+          - parquet-*
+          - datafusion
+          - datafusion-*
+          - object_store
      rust-minor-patch:
        update-types:
          - minor
@@ -0,0 +1,30 @@
+name: CI scripts
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - ci/set_lance_version.py
+      - ci/tests/**
+      - .github/workflows/ci-scripts.yml
+  pull_request:
+    paths:
+      - ci/set_lance_version.py
+      - ci/tests/**
+      - .github/workflows/ci-scripts.yml
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Test CI scripts
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+      - name: Run tests
+        run: python -m unittest discover -s ci/tests -v
@@ -129,6 +129,12 @@ jobs:
      # link.exe is single-threaded and the long pole on Windows builds. Use
      # rustc's bundled lld-link instead.
      CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld
+      # Fat LTO of the cdylib is single-threaded and the peak-memory step of the
+      # build. ThinLTO parallelizes it across the runner's cores, at some cost
+      # to runtime performance on our least performance-sensitive platform.
+      # Matches what the nodejs Windows builds already do in npm-publish.yml.
+      CARGO_PROFILE_RELEASE_LTO: thin
+      CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16
    steps:
      - uses: actions/checkout@v6
        with:
@@ -229,7 +229,8 @@ jobs:
      # Make sure wheels are not included in the Rust cache
      - name: Delete wheels
        run: rm -rf target/wheels
-  pydantic1x:
+  min-deps:
+    name: "Minimum dependencies"
    timeout-minutes: 60
    runs-on: "ubuntu-24.04"
    defaults:
@@ -259,8 +260,7 @@ jobs:
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: Install lancedb
        run: |
-          pip install "pydantic<2"
-          pip install pyarrow==16
+          pip install "pydantic==2.7.4" "pyarrow==16"
          pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
      - name: Run tests
        run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests
@@ -121,7 +121,6 @@ jobs:
      # Need up-to-date compilers for kernels
      CC: clang-18
      CXX: clang++-18
-      GH_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
    steps:
      - uses: actions/checkout@v6
        with:
@@ -165,11 +164,40 @@ jobs:
      - name: Run feature tests
        run: CARGO_ARGS="--profile ci" make -C ./lancedb feature-tests
      - name: Run examples
-        run: cargo run  --profile ci --example simple --locked
+        run: cargo run --profile ci --all-features --example simple --locked
+
+  remote:
+    timeout-minutes: 30
+    # Running this requires access to secrets, so skip if this is a PR from a
+    # fork. Keep it separate from the all-features build so Cargo does not
+    # retain both dependency graphs in one target directory.
+    if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
+    runs-on: ubuntu-2404-4x-x64
+    defaults:
+      run:
+        shell: bash
+        working-directory: rust
+    env:
+      CC: clang-18
+      CXX: clang++-18
+      GH_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+          lfs: true
+      - uses: Swatinem/rust-cache@v2
+        with:
+          # Remote tests use a different feature graph from the main Linux
+          # job. Cache downloads, but build into a fresh target directory.
+          cache-targets: false
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - uses: rui314/setup-mold@v1
      - name: Run remote tests
-        # Running this requires access to secrets, so skip if this is
-        # a PR from a fork.
-        if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
        run: CARGO_ARGS="--profile ci" make -C ./lancedb remote-tests

  macos:
@@ -18,6 +18,9 @@ Common commands:
 * Run specific test: `cargo test --quiet --features remote -p <package_name> --test <test_name>`
 * Lint: `cargo clippy --quiet --features remote --tests --examples`
 * Format Rust: `cargo fmt --all`
+* Use repository-defined Cargo profiles instead of ad hoc LTO overrides.
+* Use `release-with-debug` for benchmarks and profiling so optimized builds keep debug symbols without a rebuild.
+* Use `release-no-lto` only for local debugging, IO-bound benchmarks, or compile-time-sensitive performance investigation where LTO would not affect the measured bottleneck.
 * Format Python: `ruff format .`
 * Lint Python: `ruff check .`
 * Bootstrap Python dev env: `cd python && uv run --extra tests --extra dev maturin develop --extras tests,dev`
@@ -1740,9 +1740,9 @@ dependencies = [

 [[package]]
 name = "cmov"
-version = "0.5.3"
+version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746"
+checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"

 [[package]]
 name = "colorchoice"
@@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"

 [[package]]
 name = "fsst"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arrow-array",
 "rand 0.9.5",
@@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"

 [[package]]
 name = "lance"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -4888,8 +4888,8 @@ dependencies = [

 [[package]]
 name = "lance-arrow"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4911,7 +4911,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4925,7 +4925,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -4934,8 +4934,8 @@ dependencies = [

 [[package]]
 name = "lance-bitpacking"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arrayref",
 "crunchy",
@@ -4945,8 +4945,8 @@ dependencies = [

 [[package]]
 name = "lance-core"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4983,8 +4983,8 @@ dependencies = [

 [[package]]
 name = "lance-datafusion"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5013,8 +5013,8 @@ dependencies = [

 [[package]]
 name = "lance-datagen"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5031,8 +5031,8 @@ dependencies = [

 [[package]]
 name = "lance-derive"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -5041,8 +5041,8 @@ dependencies = [

 [[package]]
 name = "lance-encoding"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5075,8 +5075,8 @@ dependencies = [

 [[package]]
 name = "lance-file"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5107,8 +5107,8 @@ dependencies = [

 [[package]]
 name = "lance-index"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -5172,8 +5172,8 @@ dependencies = [

 [[package]]
 name = "lance-index-core"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5195,8 +5195,8 @@ dependencies = [

 [[package]]
 name = "lance-io"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5232,8 +5232,8 @@ dependencies = [

 [[package]]
 name = "lance-linalg"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5247,8 +5247,8 @@ dependencies = [

 [[package]]
 name = "lance-namespace"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arrow",
 "async-trait",
@@ -5260,8 +5260,8 @@ dependencies = [

 [[package]]
 name = "lance-namespace-impls"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arrow",
 "arrow-ipc",
@@ -5314,8 +5314,8 @@ dependencies = [

 [[package]]
 name = "lance-select"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5329,8 +5329,8 @@ dependencies = [

 [[package]]
 name = "lance-table"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5370,8 +5370,8 @@ dependencies = [

 [[package]]
 name = "lance-testing"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5384,8 +5384,8 @@ dependencies = [

 [[package]]
 name = "lance-tokenizer"
-version = "11.0.0-beta.15"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.15#8064b3a27dc4e05a6ab6ceb439fa1be9950e00eb"
+version = "11.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
 dependencies = [
 "frostem",
 "icu_segmenter",
@@ -5398,7 +5398,7 @@ dependencies = [

 [[package]]
 name = "lancedb"
-version = "0.38.0-beta.2"
+version = "0.38.0-beta.3"
 dependencies = [
 "ahash",
 "anyhow",
@@ -5486,7 +5486,7 @@ dependencies = [

 [[package]]
 name = "lancedb-nodejs"
-version = "0.38.0-beta.2"
+version = "0.38.0-beta.3"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5511,7 +5511,7 @@ dependencies = [

 [[package]]
 name = "lancedb-python"
-version = "0.38.0-beta.2"
+version = "0.38.0-beta.3"
 dependencies = [
 "arrow",
 "async-trait",
@@ -13,20 +13,21 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"

 [workspace.dependencies]
-lance = { "version" = "=11.0.0-beta.15", default-features = false, "tag" = "v11.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=11.0.0-beta.15", default-features = false, "tag" = "v11.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=11.0.0-beta.15", default-features = false, "tag" = "v11.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lancedb = { path = "rust/lancedb", default-features = false }
 ahash = "0.8"
 # Note that this one does not include pyarrow
 arrow = { version = "58.0.0", optional = false }
@@ -39,6 +40,7 @@ arrow-schema = "58.0.0"
 arrow-select = "58.0.0"
 arrow-cast = "58.0.0"
 async-trait = "0"
+bytes = "1"
 datafusion = { version = "54.0.0", default-features = false }
 datafusion-catalog = "54.0.0"
 datafusion-common = { version = "54.0.0", default-features = false }
@@ -65,7 +67,12 @@ url = "2"
 num-traits = "0.2"
 regex = "1.10"
 semver = "1.0.25"
-chrono = "0.4"
+serde = "1"
+serde_json = "1"
+tempfile = "3.5.0"
+tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
+uuid = { version = "1.7.0", features = ["v4"] }
+chrono = { version = "0.4", default-features = false, features = ["clock"] }

 [profile.ci]
 debug = "line-tables-only"
@@ -2,6 +2,7 @@
 Check whether there are any breaking changes in the PRs between the base and head commits.
 If there are, assert that we have incremented the minor version.
 """
+
 import argparse
 import os
 from packaging.version import parse
@@ -27,7 +28,7 @@ if __name__ == "__main__":
    else:
        print("No breaking changes found.")
        exit(0)
-    
+
    last_stable_version = parse(args.last_stable_version)
    current_version = parse(args.current_version)
    if current_version.minor <= last_stable_version.minor:
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 """Determine whether a newer Lance tag exists and expose results for CI."""
+
 from __future__ import annotations

 import argparse
@@ -36,8 +37,16 @@ class SemVer:
    prerelease: Tuple[Union[int, str], ...]

    def __lt__(self, other: "SemVer") -> bool:  # pragma: no cover - simple comparison
-        if (self.major, self.minor, self.patch) != (other.major, other.minor, other.patch):
-            return (self.major, self.minor, self.patch) < (other.major, other.minor, other.patch)
+        if (self.major, self.minor, self.patch) != (
+            other.major,
+            other.minor,
+            other.patch,
+        ):
+            return (self.major, self.minor, self.patch) < (
+                other.major,
+                other.minor,
+                other.patch,
+            )
        if self.prerelease == other.prerelease:
            return False
        if not self.prerelease:
@@ -142,7 +151,9 @@ def read_current_version(repo_root: Path) -> str:
        deps = data["workspace"]["dependencies"]
        entry = deps["lance"]
    except KeyError as exc:  # pragma: no cover - configuration guard
-        raise RuntimeError("Failed to locate workspace.dependencies.lance in Cargo.toml") from exc
+        raise RuntimeError(
+            "Failed to locate workspace.dependencies.lance in Cargo.toml"
+        ) from exc

    if isinstance(entry, str):
        raw_version = entry
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 """A zero-dependency mock OpenAI embeddings API endpoint for testing purposes."""
+
 import argparse
 import json
 import http.server
@@ -22,11 +23,13 @@ class MockOpenAIRequestHandler(http.server.BaseHTTPRequestHandler):

        data = []
        for i in range(num_inputs):
-            data.append({
-                "object": "embedding",
-                "embedding": [0.1] * 1536,
-                "index": i,
-            })
+            data.append(
+                {
+                    "object": "embedding",
+                    "embedding": [0.1] * 1536,
+                    "index": i,
+                }
+            )

        response = {
            "object": "list",
@@ -35,7 +38,7 @@ class MockOpenAIRequestHandler(http.server.BaseHTTPRequestHandler):
            "usage": {
                "prompt_tokens": 0,
                "total_tokens": 0,
-            }
+            },
        }

        self.send_response(200)
@@ -7,6 +7,7 @@ from packaging.version import parse, InvalidVersion

 if __name__ == "__main__":
    import argparse
+
    parser = argparse.ArgumentParser()
    parser.add_argument("prefix", default="v")
    args = parser.parse_args()
@@ -22,7 +22,7 @@ def run_command(command: str) -> str:
 def get_latest_stable_version() -> str:
    version_line = run_command("cargo info lance | grep '^version:'")
    # Example output: "version: 0.35.0 (latest 0.37.0)"
-    match = re.search(r'\(latest ([0-9.]+)\)', version_line)
+    match = re.search(r"\(latest ([0-9.]+)\)", version_line)
    if match:
        return match.group(1)
    # Fallback: use the first version after 'version:'
@@ -69,7 +69,7 @@ def extract_default_features(line: str) -> bool:
    """
    import re

-    match = re.search(r'default-features\s*=\s*false', line)
+    match = re.search(r"default-features\s*=\s*false", line)
    return match is not None


@@ -104,7 +104,7 @@ def dict_to_toml_line(package_name: str, config: dict) -> str:
            # This shouldn't happen with our current usage
            parts.append(f'"{key}" = {json.dumps(value)}')

-    return f'{package_name} = {{ {", ".join(parts)} }}\n'
+    return f"{package_name} = {{ {', '.join(parts)} }}\n"


 def update_cargo_toml(line_updater):
@@ -119,7 +119,7 @@ def update_cargo_toml(line_updater):
    lance_line = ""
    is_parsing_lance_line = False
    for line in lines:
-        if line.startswith("lance"):
+        if re.match(r"^lance(?:\s|[-_])", line):
            # Check if this is a single-line or multi-line entry
            # Single-line entries either:
            # 1. End with } (complete inline table)
@@ -0,0 +1,185 @@
+import os
+import stat
+import subprocess
+import sys
+import tempfile
+import textwrap
+import unittest
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+SCRIPT = REPO_ROOT / "ci" / "set_lance_version.py"
+LANCE_GIT_URL = "https://github.com/lance-format/lance.git"
+
+CARGO_TOML = """\
+[workspace.dependencies]
+lance = { "version" = "=1.0.0", default-features = false, "features" = ["dynamodb"] }
+lance-core = "1.0.0"
+lance_datafusion = {
+    "version" = "=1.0.0",
+    "features" = ["substrait"]
+}
+lancedb = { path = "rust/lancedb", default-features = false }
+lancedb-common = { path = "rust/lancedb-common" }
+lancewood = "1.0.0"
+my-lance = "1.0.0"
+"""
+
+UNTOUCHED_DEPENDENCIES = """\
+lancedb = { path = "rust/lancedb", default-features = false }
+lancedb-common = { path = "rust/lancedb-common" }
+lancewood = "1.0.0"
+my-lance = "1.0.0"
+"""
+
+
+class SetLanceVersionTest(unittest.TestCase):
+    def test_supported_update_modes_only_rewrite_lance_dependencies(self):
+        cases = {
+            "stable": (
+                """\
+lance = { "version" = "=9.9.9", default-features = false, "features" = ["dynamodb"] }
+lance-core = "=9.9.9"
+lance_datafusion = { "version" = "=9.9.9", "features" = ["substrait"] }
+""",
+                ["cargo info lance", "cargo metadata"],
+            ),
+            "preview": (
+                f"""\
+lance = {{ "version" = "=10.0.0-beta.3", default-features = false, "features" = ["dynamodb"], "tag" = "v10.0.0-beta.3", "git" = "{LANCE_GIT_URL}" }}
+lance-core = {{ "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "{LANCE_GIT_URL}" }}
+lance_datafusion = {{ "version" = "=10.0.0-beta.3", "features" = ["substrait"], "tag" = "v10.0.0-beta.3", "git" = "{LANCE_GIT_URL}" }}
+""",
+                ["git ls-remote --tags", "cargo metadata"],
+            ),
+            "local": (
+                """\
+lance = { "path" = "../lance/rust/lance", default-features = false, "features" = ["dynamodb"] }
+lance-core = { "path" = "../lance/rust/lance-core" }
+lance_datafusion = { "path" = "../lance/rust/lance_datafusion", "features" = ["substrait"] }
+""",
+                ["cargo metadata"],
+            ),
+            "v8.1.2": (
+                """\
+lance = { "version" = "=8.1.2", default-features = false, "features" = ["dynamodb"] }
+lance-core = "=8.1.2"
+lance_datafusion = { "version" = "=8.1.2", "features" = ["substrait"] }
+""",
+                ["cargo metadata"],
+            ),
+            "v8.2.0-beta.4": (
+                f"""\
+lance = {{ "version" = "=8.2.0-beta.4", default-features = false, "features" = ["dynamodb"], "tag" = "v8.2.0-beta.4", "git" = "{LANCE_GIT_URL}" }}
+lance-core = {{ "version" = "=8.2.0-beta.4", "tag" = "v8.2.0-beta.4", "git" = "{LANCE_GIT_URL}" }}
+lance_datafusion = {{ "version" = "=8.2.0-beta.4", "features" = ["substrait"], "tag" = "v8.2.0-beta.4", "git" = "{LANCE_GIT_URL}" }}
+""",
+                ["cargo metadata"],
+            ),
+        }
+
+        for version, (updated_dependencies, expected_commands) in cases.items():
+            with self.subTest(version=version), tempfile.TemporaryDirectory() as tmp:
+                workdir = Path(tmp)
+                (workdir / "Cargo.toml").write_text(CARGO_TOML)
+                command_log = workdir / "commands.log"
+                fake_bin = workdir / "bin"
+                fake_bin.mkdir()
+                self._write_fake_executables(fake_bin)
+                self._write_fake_python_dependencies(workdir)
+
+                env = os.environ.copy()
+                env["PATH"] = os.pathsep.join([str(fake_bin), env["PATH"]])
+                env["FAKE_COMMAND_LOG"] = str(command_log)
+                env["PYTHONPATH"] = os.pathsep.join(
+                    filter(None, [str(workdir), env.get("PYTHONPATH")])
+                )
+                result = subprocess.run(
+                    [sys.executable, str(SCRIPT), version],
+                    cwd=workdir,
+                    env=env,
+                    capture_output=True,
+                    text=True,
+                    timeout=10,
+                )
+
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertEqual(
+                    (workdir / "Cargo.toml").read_text(),
+                    "[workspace.dependencies]\n"
+                    + updated_dependencies
+                    + UNTOUCHED_DEPENDENCIES,
+                )
+                commands = command_log.read_text().splitlines()
+                for command in expected_commands:
+                    self.assertTrue(
+                        any(line.startswith(command) for line in commands),
+                        f"{command!r} not found in {commands!r}",
+                    )
+
+    def _write_fake_executables(self, fake_bin: Path) -> None:
+        cargo = fake_bin / "cargo"
+        cargo.write_text(
+            textwrap.dedent(
+                """\
+                #!/bin/sh
+                printf 'cargo %s\\n' "$*" >> "$FAKE_COMMAND_LOG"
+                case "$1" in
+                  info)
+                    printf '%s\\n' 'version: 8.8.8 (latest 9.9.9)'
+                    ;;
+                  metadata)
+                    ;;
+                  *)
+                    exit 2
+                    ;;
+                esac
+                """
+            )
+        )
+        cargo.chmod(cargo.stat().st_mode | stat.S_IXUSR)
+
+        git = fake_bin / "git"
+        git.write_text(
+            textwrap.dedent(
+                """\
+                #!/bin/sh
+                printf 'git %s\\n' "$*" >> "$FAKE_COMMAND_LOG"
+                if [ "$1" != "ls-remote" ]; then
+                  exit 2
+                fi
+                printf '%s\\n' \\
+                  '111111 refs/tags/v9.9.9' \\
+                  '222222 refs/tags/v10.0.0-beta.1' \\
+                  '333333 refs/tags/v10.0.0-beta.3'
+                """
+            )
+        )
+        git.chmod(git.stat().st_mode | stat.S_IXUSR)
+
+    def _write_fake_python_dependencies(self, workdir: Path) -> None:
+        packaging = workdir / "packaging"
+        packaging.mkdir()
+        (packaging / "__init__.py").write_text("")
+        (packaging / "version.py").write_text(
+            textwrap.dedent(
+                """\
+                class Version:
+                    def __init__(self, value):
+                        release, _, prerelease = value.partition("-beta.")
+                        self._key = (
+                            tuple(int(part) for part in release.split(".")),
+                            not prerelease,
+                            int(prerelease or 0),
+                        )
+
+                    def __lt__(self, other):
+                        return self._key < other._key
+                """
+            )
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -12,7 +12,7 @@ with open("Cargo.toml", "rb") as f:
            elif isinstance(dep, dict):
                # Version doesn't have the beta tag in it, so we instead look
                # at the git tag.
-                version = dep.get('tag', dep.get('version'))
+                version = dep.get("tag", dep.get("version"))
            else:
                raise ValueError("Unexpected type for dependency: " + str(dep))

@@ -177,6 +177,11 @@ multiple-versions = "warn"
 # Wildcard version requirements (`foo = "*"`) are a footgun — they let any
 # future release in without review. Ban them outright.
 wildcards = "deny"
+# Lint every dependency declared by a workspace member against the shared
+# `[workspace.dependencies]` table: any crate used by more than one member must
+# go through `workspace = true`, and entries nothing uses are an error. This
+# keeps versions from drifting between the core crate and the bindings.
+workspace-dependencies = { duplicates = "deny", unused = "deny" }
 # Internal workspace crates reference each other via `path = "..."`, which
 # cargo-deny sees as a wildcard version. That's fine for private workspace
 # members (not published to crates.io), so allow it specifically for paths.
@@ -5,5 +5,5 @@ mkdocs-autorefs>=0.5,<=1.0
 mkdocstrings[python]>=0.24,<1.0
 griffe>=0.40,<1.0
 mkdocs-render-swagger-plugin>=0.1.0
-pydantic>=2.0,<3.0
-mkdocs-redirects>=1.2.0
+pydantic>=2.7.4,<3
+mkdocs-redirects>=1.2.0
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-core</artifactId>
-    <version>0.38.0-beta.2</version>
+    <version>0.38.0-beta.3</version>
 </dependency>
 ```

@@ -25,9 +25,12 @@
 ### Type Aliases

 - [CreateReturnType](type-aliases/CreateReturnType.md)
+- [EmbeddingMetadataEntry](type-aliases/EmbeddingMetadataEntry.md)
+- [ResolvedEmbeddingFunctionConfig](type-aliases/ResolvedEmbeddingFunctionConfig.md)

 ### Functions

 - [LanceSchema](functions/LanceSchema.md)
 - [getRegistry](functions/getRegistry.md)
+- [parseEmbeddingMetadata](functions/parseEmbeddingMetadata.md)
 - [register](functions/register.md)
@@ -0,0 +1,22 @@
+[**@lancedb/lancedb**](../../../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / parseEmbeddingMetadata
+
+# Function: parseEmbeddingMetadata()
+
+```ts
+function parseEmbeddingMetadata(json): EmbeddingMetadataEntry[]
+```
+
+The single parser for `embedding_functions` schema metadata: every reader
+goes through here, so the wire contract cannot fork between them.
+
+## Parameters
+
+* **json**: `string`
+
+## Returns
+
+[`EmbeddingMetadataEntry`](../type-aliases/EmbeddingMetadataEntry.md)[]
@@ -0,0 +1,40 @@
+[**@lancedb/lancedb**](../../../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / EmbeddingMetadataEntry
+
+# Type Alias: EmbeddingMetadataEntry
+
+```ts
+type EmbeddingMetadataEntry: object;
+```
+
+One entry of the `embedding_functions` schema metadata, with the column
+keys normalized across the bindings' spellings.
+
+## Type declaration
+
+### model
+
+```ts
+model: EmbeddingFunction["TOptions"];
+```
+
+### name
+
+```ts
+name: string;
+```
+
+### sourceColumn
+
+```ts
+sourceColumn: string;
+```
+
+### vectorColumn
+
+```ts
+vectorColumn: string;
+```
@@ -0,0 +1,22 @@
+[**@lancedb/lancedb**](../../../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / ResolvedEmbeddingFunctionConfig
+
+# Type Alias: ResolvedEmbeddingFunctionConfig
+
+```ts
+type ResolvedEmbeddingFunctionConfig: EmbeddingFunctionConfig & object;
+```
+
+An [EmbeddingFunctionConfig] read back from table metadata, where the
+vector column is always recorded.
+
+## Type declaration
+
+### vectorColumn
+
+```ts
+vectorColumn: string;
+```
@@ -54,6 +54,54 @@ listing a storage directory.

 ::: lancedb.LsmWriteSpec

+## Functions and Jobs
+
+::: lancedb.functions.FunctionArtifact
+
+::: lancedb.functions.FunctionParameter
+
+::: lancedb.functions.FunctionResultField
+
+::: lancedb.functions.FunctionOutput
+
+::: lancedb.functions.FunctionSignature
+
+::: lancedb.functions.PythonEnvironmentSpec
+
+::: lancedb.functions.udf
+
+::: lancedb.functions.UdfDefinition
+
+::: lancedb.functions.FunctionRegistrationRequest
+
+::: lancedb.functions.FunctionArtifactRequest
+
+::: lancedb.functions.FunctionArtifactContent
+
+::: lancedb.functions.PythonAdapterSpec
+
+::: lancedb.functions.FunctionVersion
+
+::: lancedb.functions.PythonRuntimeSpec
+
+::: lancedb.functions.FunctionVersionRef
+
+::: lancedb.functions.ApplicationInput
+
+::: lancedb.functions.FunctionApplication
+
+::: lancedb.functions.InputBinding
+
+::: lancedb.functions.OutputMapping
+
+::: lancedb.functions.FunctionBinding
+
+::: lancedb.functions.RefreshColumnResult
+
+::: lancedb.job.Job
+
+::: lancedb.job.AsyncJob
+
 ## Expressions

 Type-safe expression builder for filters and projections. Use these instead
@@ -153,8 +201,9 @@ The same option is available on `lancedb.tokenize(...)` and the deprecated
 ```python
 import lancedb

-tokens = list(lancedb.tokenize("acme makes searchable data",
-                               custom_stop_words=["acme"]))
+tokens = list(
+    lancedb.tokenize("acme makes searchable data", custom_stop_words=["acme"])
+)
 ```

 ::: lancedb.tokenize
@@ -8,7 +8,7 @@
    <parent>
      <groupId>com.lancedb</groupId>
      <artifactId>lancedb-parent</artifactId>
-      <version>0.38.0-beta.2</version>
+      <version>0.38.0-beta.3</version>
      <relativePath>../pom.xml</relativePath>
    </parent>

@@ -6,7 +6,7 @@

    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-parent</artifactId>
-    <version>0.38.0-beta.2</version>
+    <version>0.38.0-beta.3</version>
    <packaging>pom</packaging>
    <name>${project.artifactId}</name>
    <description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>11.0.0-beta.15</lance-core.version>
+        <lance-core.version>11.0.0-beta.19</lance-core.version>
        <spotless.skip>false</spotless.skip>
        <spotless.version>2.30.0</spotless.version>
        <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.38.0-beta.2"
+version = "0.38.0-beta.3"
 publish = false
 license.workspace = true
 description.workspace = true
@@ -16,12 +16,12 @@ crate-type = ["cdylib"]
 async-trait.workspace = true
 arrow-ipc.workspace = true
 arrow-array.workspace = true
-arrow-buffer = "58.0.0"
+arrow-buffer.workspace = true
 half.workspace = true
 arrow-schema.workspace = true
 env_logger.workspace = true
 futures.workspace = true
-lancedb = { path = "../rust/lancedb", default-features = false }
+lancedb.workspace = true
 lance-namespace.workspace = true
 napi = { version = "3.8.3", default-features = false, features = [
    "napi9",
@@ -29,8 +29,8 @@ napi = { version = "3.8.3", default-features = false, features = [
    "chrono_date",
    "serde-json",
 ] }
-chrono = { version = "0.4", default-features = false, features = ["clock"] }
-serde_json = "1"
+chrono.workspace = true
+serde_json.workspace = true
 napi-derive = "3.5.2"
 # Prevent dynamic linking of lzma, which comes from datafusion
 lzma-sys = { version = "0.1", features = ["static"] }
@@ -173,6 +173,36 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
    }

    describe("The function makeArrowTable", function () {
+      it("accepts snake_case embedding metadata like camelCase", function () {
+        const spellings = [
+          // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+          { source_column: "text", vector_column: "vector" },
+          { sourceColumn: "text", vectorColumn: "vector" },
+        ];
+        for (const columns of spellings) {
+          const schema = new Schema(
+            [
+              new Field("text", new Utf8(), false),
+              new Field(
+                "vector",
+                new FixedSizeList(3, new Field("item", new Float32(), true)),
+                false,
+              ),
+            ],
+            new Map([
+              [
+                "embedding_functions",
+                JSON.stringify([{ name: "mock", model: {}, ...columns }]),
+              ],
+            ]),
+          );
+          // The vector field is non-nullable and absent from the data; only a
+          // recognized embedding config makes that acceptable.
+          const table = makeArrowTable([{ text: "hello" }], { schema });
+          expect(table.numRows).toBe(1);
+        }
+      });
+
      it("will use data types from a provided schema instead of inference", async function () {
        const schema = new Schema([
          new Field("a", new Int32(), false),
@@ -106,6 +106,77 @@ describe.each([arrow15, arrow16, arrow17, arrow18])("Registry", (arrow) => {
      'Embedding function with alias "mock-embedding" already exists',
    );
  });
+  test("parseFunctions keeps entries sharing a function name", async () => {
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      ndims() {
+        return 3;
+      }
+      embeddingDataType() {
+        return new arrow.Float32() as apiArrow.Float;
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        return data.map(() => [1, 2, 3]);
+      }
+    }
+    register("mock-embedding")(MockEmbeddingFunction);
+    const parsed = await getRegistry().parseFunctions(
+      new Map([
+        [
+          "embedding_functions",
+          JSON.stringify([
+            {
+              name: "mock-embedding",
+              sourceColumn: "text",
+              vectorColumn: "vector_a",
+              model: {},
+            },
+            {
+              name: "mock-embedding",
+              sourceColumn: "text",
+              vectorColumn: "vector_b",
+              model: {},
+            },
+          ]),
+        ],
+      ]),
+    );
+    expect([...parsed.values()].map((f) => f.vectorColumn)).toEqual([
+      "vector_a",
+      "vector_b",
+    ]);
+
+    // The Python bindings write snake_case keys.
+    const snake = await getRegistry().parseFunctions(
+      new Map([
+        [
+          "embedding_functions",
+          JSON.stringify([
+            {
+              name: "mock-embedding",
+              // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+              source_column: "text",
+              // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+              vector_column: "vector_a",
+              model: {},
+            },
+            {
+              name: "mock-embedding",
+              // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+              source_column: "text",
+              // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+              vector_column: "vector_b",
+              model: {},
+            },
+          ]),
+        ],
+      ]),
+    );
+    expect([...snake.keys()]).toEqual(["vector_a", "vector_b"]);
+    expect([...snake.values()].map((f) => f.sourceColumn)).toEqual([
+      "text",
+      "text",
+    ]);
+  });
  test("schema should contain correct metadata", async () => {
    class MockEmbeddingFunction extends EmbeddingFunction<string> {
      constructor(args: FunctionOptions = {}) {
@@ -48,7 +48,11 @@ import {
 } from "apache-arrow";
 import { Buffers } from "apache-arrow/data";
 import { type EmbeddingFunction } from "./embedding/embedding_function";
-import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
+import {
+  EmbeddingFunctionConfig,
+  getRegistry,
+  parseEmbeddingMetadata,
+} from "./embedding/registry";
 import {
  sanitizeField,
  sanitizeSchema,
@@ -933,7 +937,7 @@ async function applyEmbeddingsFromMetadata(

  for (const functionEntry of functions.values()) {
    const sourceColumn = columns[functionEntry.sourceColumn];
-    const destColumn = functionEntry.vectorColumn ?? "vector";
+    const destColumn = functionEntry.vectorColumn;
    if (sourceColumn === undefined) {
      throw new Error(
        `Cannot apply embedding function because the source column '${functionEntry.sourceColumn}' was not present in the data`,
@@ -1385,11 +1389,10 @@ function validateSchemaEmbeddings(

        // Check schema metadata for embedding functions
        if (schema.metadata.has("embedding_functions")) {
-          const embeddings = JSON.parse(
+          const entries = parseEmbeddingMetadata(
            schema.metadata.get("embedding_functions")!,
          );
-          // biome-ignore lint/suspicious/noExplicitAny: we don't know the type of `f`
-          if (embeddings.find((f: any) => f["vectorColumn"] === field.name)) {
+          if (entries.some((f) => f.vectorColumn === field.name)) {
            hasEmbeddingFunction = true;
          }
        }
@@ -104,41 +104,29 @@ export class EmbeddingFunctionRegistry {
  async parseFunctions(
    this: EmbeddingFunctionRegistry,
    metadata: Map<string, string>,
-  ): Promise<Map<string, EmbeddingFunctionConfig>> {
+  ): Promise<Map<string, ResolvedEmbeddingFunctionConfig>> {
    if (!metadata.has("embedding_functions")) {
      return new Map();
-    } else {
-      type FunctionConfig = {
-        name: string;
-        sourceColumn: string;
-        vectorColumn: string;
-        model: EmbeddingFunction["TOptions"];
-      };
-
-      const functions = <FunctionConfig[]>(
-        JSON.parse(metadata.get("embedding_functions")!)
-      );
-
-      const items: [string, EmbeddingFunctionConfig][] = await Promise.all(
-        functions.map(async (f) => {
-          const fn = this.get(f.name);
-          if (!fn) {
-            throw new Error(`Function "${f.name}" not found in registry`);
-          }
-          const func = await this.get(f.name)!.create(f.model);
-          return [
-            f.name,
-            {
-              sourceColumn: f.sourceColumn,
-              vectorColumn: f.vectorColumn,
-              function: func,
-            },
-          ];
-        }),
-      );
-
-      return new Map(items);
    }
+    const entries = parseEmbeddingMetadata(
+      metadata.get("embedding_functions")!,
+    );
+    const items = await Promise.all(
+      entries.map(async (f): Promise<ResolvedEmbeddingFunctionConfig> => {
+        const fn = this.get(f.name);
+        if (!fn) {
+          throw new Error(`Function "${f.name}" not found in registry`);
+        }
+        const func = await fn.create(f.model);
+        return {
+          sourceColumn: f.sourceColumn,
+          vectorColumn: f.vectorColumn,
+          function: func,
+        };
+      }),
+    );
+    // Keyed by output column: one function may serve several columns.
+    return new Map(items.map((config) => [config.vectorColumn, config]));
  }
  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
  functionToMetadata(conf: EmbeddingFunctionConfig): Record<string, any> {
@@ -218,3 +206,52 @@ export interface EmbeddingFunctionConfig {
  vectorColumn?: string;
  function: EmbeddingFunction;
 }
+
+/** An [EmbeddingFunctionConfig] read back from table metadata, where the
+ * vector column is always recorded. */
+export type ResolvedEmbeddingFunctionConfig = EmbeddingFunctionConfig & {
+  vectorColumn: string;
+};
+
+/** One entry of the `embedding_functions` schema metadata, with the column
+ * keys normalized across the bindings' spellings. */
+export type EmbeddingMetadataEntry = {
+  name: string;
+  sourceColumn: string;
+  vectorColumn: string;
+  model: EmbeddingFunction["TOptions"];
+};
+
+/** The single parser for `embedding_functions` schema metadata: every reader
+ * goes through here, so the wire contract cannot fork between them. */
+export function parseEmbeddingMetadata(json: string): EmbeddingMetadataEntry[] {
+  // The wire format, honestly: the Python bindings write snake_case keys.
+  type Raw = {
+    name: string;
+    sourceColumn?: string;
+    // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+    source_column?: string;
+    vectorColumn?: string;
+    // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+    vector_column?: string;
+    model: EmbeddingFunction["TOptions"];
+  };
+  const entries = <Raw[]>JSON.parse(json);
+  const seen = new Set<string>();
+  return entries.map((f) => {
+    const sourceColumn = f.sourceColumn ?? f.source_column;
+    const vectorColumn = f.vectorColumn ?? f.vector_column;
+    if (sourceColumn === undefined || vectorColumn === undefined) {
+      throw new Error(
+        `Embedding function "${f.name}" metadata names no source or vector column`,
+      );
+    }
+    if (seen.has(vectorColumn)) {
+      throw new Error(
+        `Multiple embedding configs claim vector column "${vectorColumn}"`,
+      );
+    }
+    seen.add(vectorColumn);
+    return { name: f.name, sourceColumn, vectorColumn, model: f.model };
+  });
+}
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.38.0-beta.2",
+	"version": "0.38.0-beta.3",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.38.0-beta.2",
+	"version": "0.38.0-beta.3",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.38.0-beta.2",
+	"version": "0.38.0-beta.3",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.38.0-beta.2",
+	"version": "0.38.0-beta.3",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.38.0-beta.2",
+	"version": "0.38.0-beta.3",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",
@@ -1,6 +1,6 @@
 {
  "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.38.0-beta.2",
+  "version": "0.38.0-beta.3",
  "os": [
    "win32"
  ],
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.38.0-beta.2",
+	"version": "0.38.0-beta.3",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",
@@ -1,12 +1,12 @@
 {
  "name": "@lancedb/lancedb",
-  "version": "0.38.0-beta.2",
+  "version": "0.38.0-beta.3",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "@lancedb/lancedb",
-      "version": "0.38.0-beta.2",
+      "version": "0.38.0-beta.3",
      "cpu": [
        "x64",
        "arm64"
@@ -11,7 +11,7 @@
    "ann"
  ],
  "private": false,
-  "version": "0.38.0-beta.2",
+  "version": "0.38.0-beta.3",
  "main": "dist/index.js",
  "exports": {
    ".": "./dist/index.js",
@@ -1,21 +0,0 @@
-{
-  "name": "lancedb",
-  "description": "Write, review, debug, and document LanceDB pipelines in Python and TypeScript that work across local LanceDB OSS tables and remote LanceDB Enterprise/Cloud tables, with idiomatic query/search patterns and performance defaults for ingestion, indexing, filtering, and diagnostics.",
-  "version": "0.1.0",
-  "author": {
-    "name": "LanceDB"
-  },
-  "homepage": "https://www.lancedb.com",
-  "keywords": [
-    "lancedb",
-    "vector-search",
-    "full-text-search",
-    "hybrid-search",
-    "python",
-    "typescript",
-    "pipelines",
-    "ingestion",
-    "indexing",
-    "performance"
-  ]
-}
@@ -1,33 +0,0 @@
-{
-  "name": "lancedb",
-  "version": "0.1.0",
-  "description": "Codex plugin for building LanceDB pipelines in Python and TypeScript.",
-  "author": {
-    "name": "LanceDB"
-  },
-  "keywords": [
-    "lancedb",
-    "vector-search",
-    "full-text-search",
-    "hybrid-search",
-    "python",
-    "typescript",
-    "pipelines"
-  ],
-  "skills": "./skills/",
-  "interface": {
-    "displayName": "LanceDB",
-    "shortDescription": "Build LanceDB pipelines in Python and TypeScript.",
-    "longDescription": "Write, review, debug, and document LanceDB pipelines in Python and TypeScript that work across local LanceDB OSS tables and remote LanceDB Enterprise/Cloud tables, with idiomatic query/search patterns and performance defaults for ingestion, indexing, filtering, and diagnostics.",
-    "developerName": "LanceDB",
-    "websiteURL": "https://www.lancedb.com",
-    "category": "Developer Tools",
-    "capabilities": [
-      "Developer Tools"
-    ],
-    "defaultPrompt": "Create a LanceDB table, embed sample text, and run a vector search.",
-    "composerIcon": "./assets/logo.png",
-    "logo": "./assets/logo.png",
-    "logoDark": "./assets/logo-dark.png"
-  }
-}
@@ -1,87 +0,0 @@
---
-name: lancedb
-description: Use when writing, reviewing, debugging, or documenting LanceDB pipelines in Python or TypeScript, especially code that should work across local LanceDB OSS tables and remote LanceDB Enterprise/Cloud tables. Helps avoid non-portable full-table materialization, choose idiomatic query/search patterns, apply LanceDB performance defaults for ingestion, indexing, filtering, and diagnostics, and resolve connections to the remote server for Enterprise-only operations such as jobs.
---
-
-# Building LanceDB Pipelines
-
-Use this skill to produce LanceDB pipelines that are portable between local and remote tables (for LanceDB Enterprise/Cloud) and idiomatic for the selected SDK.
-
-## LanceDB Table Modes
-
-LanceDB has two common execution modes:
-
- **Local table**: embedded, open source, in-process LanceDB. The client opens data from a local path or object storage URI and executes queries in the application process.
- **Remote table**: LanceDB Enterprise/Cloud table opened through a `db://...` URI. The data may be very large, commonly backed by object storage, and queried through a remote service.
-
-Do NOT assume local-only table helpers exist on remote tables. If the user asks for LanceDB Enterprise, Cloud, `db://...`, production remote access, or a remote table, focus on the remote table path: use `search()` / `query()`, keep reads bounded with `select()` and `limit()`, and avoid table-level full materialization APIs.
-
-## Workflow
-
-1. Identify the SDK: Python, TypeScript, or both.
-2. Identify the table mode: local/embedded OSS, remote Enterprise/Cloud, or portable across both. If the user says "LanceDB Enterprise", choose the remote table path. If the task involves jobs in any way (listing, inspecting, creating, or canceling jobs), it is always the remote path and requires a remote server connection — see "Connecting to the LanceDB remote server" below before doing anything else.
-3. Read the matching language branch before writing or changing code:
-   - Python patterns: `references/python/patterns.md`
-   - Python API quick reference: `references/python/api_reference.md`
-   - Python performance guidance: `references/python/performance.md`
-   - TypeScript patterns: `references/typescript/patterns.md`
-   - TypeScript API quick reference: `references/typescript/api_reference.md`
-   - TypeScript performance guidance: `references/typescript/performance.md`
-   - Column metadata authoring (both SDKs): `references/column_metadata.md`
-   - Branch operations (both SDKs): `references/branch_ops.md`
-   - Remote server connection resolution (jobs, raw REST): `references/remote_connect.md`
-   - Job operations REST API (list/describe/cancel/query_events): `references/remote_jobs.md`
-4. Start with `patterns.md` for the selected SDK. Read `api_reference.md` when choosing method names or return collectors. Read `performance.md` when the task involves ingestion, indexing, filtering, query tuning, diagnostics, or large datasets. Read `column_metadata.md` when the task is documenting, tagging, classifying, or grouping table columns (field descriptions, `lancedb:tag:*` tags, logical column families). Read `branch_ops.md` when the task involves branch lifecycle (list/create/delete), writing to a non-main branch, or verifying a change stayed off main. Read `remote_connect.md` when the task involves jobs or direct REST access to an Enterprise deployment, and `remote_jobs.md` for the job REST methods themselves (list, describe, cancel, query_events).
-5. For Python schemas, favor Pydantic models and validate records before writing. Use PyArrow schemas when Arrow-native, streaming, or highly dynamic data makes them materially better suited.
-6. Prefer `search()` or `query()` builders with explicit `select()` and `limit()` for reads.
-7. Avoid table-level full materialization in remote or portable code. This is the main local-vs-remote read pitfall.
-8. After a successful embedded OSS ingestion, call `table.optimize()`. Do not call it for Enterprise/Cloud; remote maintenance is automatic.
-9. For remote Enterprise/Cloud writes, never drop-then-reuse or `mode="overwrite"` the same table name — see "Enterprise: never drop-then-reuse the same table name" below. This is the main local-vs-remote write pitfall.
-10. If reviewing an existing file or repo, run `scripts/check_materialization.py` on the relevant paths and inspect each finding before editing.
-11. Cross-check unfamiliar or non-trivial API claims against the source tree instead of relying on memory.
-
-## Core Portability Rule
-
-Do not write code that assumes a local table API will exist on a remote table. Remote tables can be very large, so whole-table materialization helpers are intentionally unavailable or unsafe.
-
-This does **not** mean result conversion is forbidden. Bounded query/search result collection is normal:
-
- Python: `table.search(...).select([...]).limit(10).to_pandas()`
- TypeScript: `await table.search(...).select([...]).limit(10).toArray()`
-
-The unsafe pattern is table-level or unbounded collection, plus local-only dataset escape hatches in remote code:
-
- Python: `table.to_pandas()`, `table.to_arrow()`, `table.to_polars()`; `table.to_lance()` is local/OSS-only dataset access, not materialization
- TypeScript: `await table.toArrow()`, `await table.query().toArray()` without `limit()`
-
-## Enterprise: never drop-then-reuse the same table name
-
-LanceDB Enterprise/Cloud splits a **control plane** (DDL: create/drop/rename) from a **data plane** (query nodes that serve reads). Query nodes cache the resolved dataset for a table name for up to `table_cache_ttl` — **default 300 seconds (5 minutes)**. After you drop or overwrite a table, the control plane updates immediately but the data plane keeps serving the *old* dataset until that cache entry expires. During the window the two planes disagree.
-
-The failure this causes: you `drop_table("t")` then immediately `create_table("t", ...)` (or `create_table("t", ..., mode="overwrite")`). The DDL returns success, but every query against `t` returns **`500 Internal Server Error`** (the query node resolves the stale/deleted dataset), and a fresh `describe` may still show the *old* schema/version. It looks like your write silently failed; it didn't — the name is cached.
-
-**`mode="overwrite"` has the same problem** — it is a drop+create of the same name under the hood.
-
-Rules for portable Enterprise ingestion:
-
-1. **Never reuse a table name you just dropped/overwrote within the cache TTL.** Do not use `mode="overwrite"` to replace an existing Enterprise table in place.
-2. To (re)load data, **write to a fresh table name** (e.g. `<table>_v2`, or a run-stamped suffix). A brand-new name has no cached data-plane entry, so writes and reads work immediately.
-3. Before creating, `list_tables()` and **fail loudly if the name already exists** rather than overwriting — prompt for a new name.
-4. To land on a specific final name that is currently occupied by an old table: drop the old table, **wait out the TTL (~5 min), then `rename_table(fresh_name, final_name)`**. Renaming onto a name whose old dataset is still cached hits the same race, so the wait is mandatory. `rename_table` is a supported control-plane op.
-5. When you hand a table name back to a human, tell them which step still needs the propagation wait (usually: "the old `t` was dropped; run the rename in ~5 minutes").
-
-This is Enterprise/Cloud-specific. Local/OSS tables have no separate data plane, so `mode="overwrite"` and immediate same-name reuse are fine there.
-
-## Connecting to the LanceDB remote server
-
-LanceDB Enterprise/Cloud deployments are served by a server implementing the lance-namespace OpenAPI spec (<https://github.com/lance-format/lance-namespace/blob/main/docs/src/spec.yaml>). Every remote (`db://...`) connection talks to such a server, and some operations exist only there. In particular, **all operations around jobs (listing, inspecting, creating, or canceling jobs) run server-side** — there is no local/OSS equivalent. Before any job work, or any direct REST call to an Enterprise deployment, read `references/remote_connect.md` to resolve the base URL, credentials, and database header and to validate the connection. Then use the four job REST methods documented in `references/remote_jobs.md` (list, describe, cancel, query_events).
-
-## Script
-
-Run the scanner when reviewing or modifying an existing codebase:
-
-```bash
-python skills/lancedb/scripts/check_materialization.py path/to/file_or_dir
-```
-
-The script reports likely unsafe full-table materialization in Python and TypeScript. Treat results as review prompts, not automatic proof of a bug.
@@ -1,6 +0,0 @@
-interface:
-  display_name: "LanceDB"
-  short_description: "Build LanceDB pipelines in Python and TypeScript"
-  default_prompt: "Use $lancedb to create a table, embed sample text, and run a vector search."
-  icon_small: "./assets/icon.png"
-  icon_large: "./assets/icon.png"
@@ -1,182 +0,0 @@
-# Branch Operations
-
-Manage branches on a LanceDB table: list what exists, create new ones, delete stale ones, and direct read/write operations at a specific branch without touching main. Use for branch lifecycle tasks, experimental/isolated table versions, targeting an operation at a non-main branch, or confirming a mutation did not affect main.
-
-Works on local/OSS and remote Enterprise/Cloud tables, except merging a branch into main, which is Enterprise-only.
-
-## The branch model (important)
-
-Branches are isolated, writable lines of history forked from another branch (or a specific version). Writes on a branch never affect `main`.
-
-There is **no global "switch branch" state** — you never repoint the whole table at a branch. Instead, **operations are scoped by which table handle you use**:
-
- The handle you got from `open_table(name)` / `openTable(name)` targets `main`.
- `branches.create(...)` and `branches.checkout(...)` return a **new table handle scoped to that branch**. Every read/write on that handle (add, update, `update_field_metadata`, `create_index`, search, …) lands on the branch.
- The original main handle is unaffected — keep it around to verify isolation.
-
-`branches.list()` returns only non-main branches. Main always exists and is not listed.
-
-## Python
-
-`table.branches` is a property returning the branch manager; `table.current_branch()` tells you what a handle is scoped to (`None` = main).
-
-```python
-table = db.open_table("products")            # scoped to main
-
-# list — dict of name -> metadata (parent_branch, parent_version, ...); {} = only main
-table.branches.list()
-
-# create: forks from main by default and returns a handle scoped to the new branch
-exp = table.branches.create("experiment-reindex")
-exp = table.branches.create("exp2", from_ref="main", from_version=None)  # optional fork point
-
-# checkout an existing branch -> branch-scoped handle
-wip = table.branches.checkout("wip-branch")
-# with version= it pins to that version (read-only detached view); omit to track latest, writable
-
-# operate on the branch simply by using its handle
-wip.update_field_metadata(
-    {"path": "category", "metadata": {"lancedb:description": "Product category label."}}
-)
-wip.create_scalar_index("category")
-
-# delete: removes only the branch pointer; main and row data remain intact
-table.branches.delete("stale-2024")
-
-# alternatively, open a branch handle directly from the connection
-wip = db.open_table("products", branch="wip-branch")
-
-exp.current_branch()    # "experiment-reindex"
-table.current_branch()  # None (main)
-```
-
-Async: same shape — `table.branches` returns `AsyncBranches`; `await table.branches.create(...)` etc.
-
-## TypeScript
-
-`table.branches()` is an **async method** returning the `Branches` manager; `table.currentBranch()` returns the scoped branch or `null` for main.
-
-```typescript
-const table = await db.openTable("products"); // scoped to main
-const branches = await table.branches();
-
-// list — Record<string, BranchContents>; {} = only main
-await branches.list();
-
-// create: forks from main by default, returns a Table scoped to the new branch
-const exp = await branches.create("experiment-reindex");
-const exp2 = await branches.create("exp2", "main" /* fromRef */, undefined /* fromVersion */);
-
-// checkout an existing branch -> branch-scoped Table
-const wip = await branches.checkout("wip-branch");
-// with a version arg it pins (read-only detached view); omit to track latest, writable
-
-// operate on the branch simply by using its handle
-await wip.updateFieldMetadata([
-  { path: "category", metadata: { "lancedb:description": "Product category label." } },
-]);
-await wip.createIndex("category");
-
-// delete: removes only the branch pointer; main and row data remain intact
-await branches.delete("stale-2024");
-
-// alternatively, open a branch handle directly from the connection
-const wip2 = await db.openTable("products", { branch: "wip-branch" });
-
-exp.currentBranch();   // "experiment-reindex"
-table.currentBranch(); // null (main)
-```
-
-## Verifying isolation
-
-After writing to a branch, confirm the change did NOT land on main by reading through both handles:
-
-```python
-wip = table.branches.checkout("wip-branch")
-wip.update_field_metadata({"path": "category", "metadata": {"lancedb:description": "..."}})
-
-assert b"lancedb:description" in (wip.schema.field("category").metadata or {})
-assert b"lancedb:description" not in (table.schema.field("category").metadata or {})  # main untouched
-```
-
-Two handles on the same branch see each other's writes (e.g. `table.branches.create("exp")` and `db.open_table(name, branch="exp")`); main stays isolated.
-
-## Merging a branch into main (Enterprise only)
-
-Merge is available through the SDKs (`table.branches.merge(...)`) on **Enterprise tables only** — it is not supported on Cloud or local/OSS tables, which raise `NotSupported`.
-
-`merge` takes the branch to merge **from** and a `dry_run` flag. Both the SDK method and the underlying REST endpoint **actually merge by default** (`dry_run=False`); pass `dry_run=True` to only preview. A rejected merge is **not an exception** — it returns a result with `status="rejected"` rather than raising, so inspect the return value. Use `branches.diff(from_branch)` to inspect a branch's pending diff without attempting a merge.
-
-```python
-exp = "experiment-reindex"
-
-# preview only — returns status="ready" if it would merge cleanly
-preview = table.branches.merge(exp, dry_run=True)
-
-# actually merge (default)
-result = table.branches.merge(exp)
-if result["status"] == "merged":
-    print("landed at", result["mainVersionAfter"])
-elif result["status"] == "rejected":
-    print(result["diff"]["mergeBlockers"])   # why it was refused
-
-# inspect a branch's pending diff without merging
-diff = table.branches.diff(exp)
-```
-
-Async: `await table.branches.merge(exp)`, `await table.branches.diff(exp)`.
-
-```typescript
-const branches = await table.branches();
-const exp = "experiment-reindex";
-
-// preview only (second arg is dryRun)
-const preview = await branches.merge(exp, true);
-
-// actually merge (default)
-const result = await branches.merge(exp);
-if (result.status === "merged") {
-  console.log("landed at", result.mainVersionAfter);
-} else if (result.status === "rejected") {
-  console.log(result.diff.mergeBlockers);
-}
-
-const diff = await branches.diff(exp);
-```
-
-The result is the wire JSON, containing `status` (`ready` on a passing dry run, `merged` on success, `rejected` when refused — also `notImplemented`/`unknown`), the branch `diff` (including `mergeBlockers` explaining any rejection), a `preview` of the columns that would be promoted, and — after a real merge — `mainVersionAfter`.
-
-### Merge preconditions
-
-Merge only **promotes newly added columns** onto main; it does not replay arbitrary commits. Practically, a branch is mergeable only if it has **exactly one commit since it was created, and that commit added a column**. The merge is rejected (`status: "rejected"`, with `mergeBlockers` set) if:
-
- the branch was forked from another branch rather than directly from main
- main has advanced since the branch was forked
- the branch's rows changed since the fork (row counts must match main exactly)
- the branch removed columns or changed a column's type/nullability
- the branch added no columns (index-only changes are not merged)
-
-### Adding a column in a single commit
-
-Because the branch must contain just one column-adding commit, add the column with its values in one operation rather than add-then-backfill:
-
-1. **SQL transformation** — `add_columns` with a SQL expression computed from existing columns, so the column lands populated in one commit.
-2. **Precompute the values** — compute the column's values externally, then add the fully-populated column in a single operation (e.g. via `merge_insert`/`add_columns` with the data ready).
-3. **Lance-format-level data evolution (pylance)** — use Lance's data evolution with backfill, documented at <https://lance.org/guide/data_evolution/#with-data-backfill>.
-
-## Quick reference
-
-| Goal | Python | TypeScript |
-|------|--------|------------|
-| List branches (non-main) | `table.branches.list()` | `await (await table.branches()).list()` |
-| Create branch (off main) | `table.branches.create(name)` → branch handle | `await branches.create(name)` → branch `Table` |
-| Create from a fork point | `table.branches.create(name, from_ref=..., from_version=...)` | `await branches.create(name, fromRef, fromVersion)` |
-| Get a branch handle | `table.branches.checkout(name)` or `db.open_table(t, branch=name)` | `await branches.checkout(name)` or `await db.openTable(t, { branch: name })` |
-| Pin to a branch version (read-only) | `table.branches.checkout(name, version=v)` | `await branches.checkout(name, v)` |
-| Delete branch | `table.branches.delete(name)` | `await branches.delete(name)` |
-| Which branch is this handle on? | `table.current_branch()` (`None` = main) | `table.currentBranch()` (`null` = main) |
-| Target main | use the original (non-branch) handle | use the original (non-branch) handle |
-| Merge branch into main (Enterprise only) | `table.branches.merge(from_branch, dry_run=False)` | `await branches.merge(fromBranch, dryRun)` |
-| Preview a branch's pending diff (Enterprise only) | `table.branches.diff(from_branch)` | `await branches.diff(fromBranch)` |
-
-Branch names must be non-empty; empty names raise a validation error.
@@ -1,183 +0,0 @@
-# Column Metadata Authoring
-
-Write column-level descriptions, tags, and logical groupings onto a LanceDB table's schema. Use this when the user wants to document, annotate, tag, or classify what their table columns ARE (embeddings vs labels vs eval metrics, model provenance, version families, etc.).
-
-Works on local/OSS and remote Enterprise/Cloud tables alike — read the schema through the table handle, write through `update_field_metadata` (Python) / `updateFieldMetadata` (TypeScript).
-
-## Metadata key conventions
-
-All metadata uses namespaced keys:
-
-| Key | Purpose | Example value |
-|-----|---------|---------------|
-| `lancedb:description` | Human-readable explanation of what the column contains | `"CLIP ViT-L/14 image embedding, L2-normalized (768-dim)"` |
-| `lancedb:tag:<name>` | Flexible key-value tag; the suffix names the tag category | `lancedb:tag:field_type: "embedding"`, `lancedb:tag:model: "clip"`, `lancedb:tag:project_id: "foo"` |
-| `lancedb:logical-column` | Logical group/family this column belongs to | `"clip_features"` |
-
-Tags are open-ended — use whatever key suffix and value make sense given the user's intent. The tag suffix should describe *what is being classified* (e.g., `field_type`, `model`, `project_id`) and the value describes *how*. Multiple tags on the same column are fine — each is a separate key. All values are strings.
-
-## Step 1: Read the schema and existing metadata
-
-Read existing metadata before writing, to avoid redundant updates.
-
-Python — `table.schema` (sync property; async: `await table.schema()`) returns a `pyarrow.Schema`. **Arrow field metadata is bytes-keyed in Python**:
-
-```python
-schema = table.schema
-for field in schema:
-    meta = field.metadata or {}  # dict[bytes, bytes], e.g. {b"lancedb:description": b"..."}
-    print(field.name, field.type, field.nullable, meta)
-```
-
-TypeScript — `await table.schema()` returns an Arrow `Schema`; field metadata is a `Map<string, string>`:
-
-```typescript
-const schema = await table.schema();
-for (const field of schema.fields) {
-  console.log(field.name, field.type, field.nullable, field.metadata); // Map
-  // field.metadata.get("lancedb:description")
-}
-```
-
-For struct/nested fields, recurse into the field's children and address them as dot-paths (e.g., `parent.child`).
-
-If the user hasn't specified which columns to update, work with all columns.
-
-## Step 2: Generate metadata
-
-Decide what to generate based on the user's request.
-
-### Descriptions (`lancedb:description`)
-
-Base descriptions on:
- The column name and Arrow type (e.g., `FixedSizeList` of floats → likely an embedding)
- User-supplied context (upstream pipeline, sample values, domain knowledge)
- Name patterns: `_embedding`/`_vec`/`_embed` → vector; `_label`/`_class` → label; `_score`/`_eval`/`_metric` → evaluation metric
-
-Be specific and concise. Good: `"Sentence-BERT embedding of the query text (768-dim)."` Not: `"An embedding column."`
-
-### Tags (`lancedb:tag:<name>`)
-
-Choose tag key names that match what the user asked to annotate. Common patterns:
-
- Semantic field type → `lancedb:tag:field_type: "embedding"` / `"text"` / `"image"` / `"label"` / `"eval"` / `"id"` / `"metadata"`
- Model or source → `lancedb:tag:model: "clip"` / `"bert"` / `"vit"`
- Project affiliation → `lancedb:tag:project_id: "<name>"`
- Version → `lancedb:tag:version: "v3"` (and `lancedb:tag:latest: "true"` for the newest)
-
-Use Arrow type as a hint: `FixedSizeList` + float → embedding; `Utf8`/`LargeUtf8` → text; `Binary` → image or blob.
-
-### Logical groupings (`lancedb:logical-column`)
-
-Look for naming patterns across columns:
- `clip_v1`, `clip_v2`, `clip_v3` → logical column `"clip"`, latest is `v3`
- `text_embed_20240101`, `text_embed_20240601` → logical column `"text_embed"`, latest is the most recent date suffix
-
-Write `lancedb:logical-column` on all members of a group. Mark the newest with `lancedb:tag:latest: "true"` (in addition to its version tag).
-
-## Step 3: Write the metadata
-
-Each update names a field by dot-path and carries a metadata map. Semantics (identical in both SDKs):
-
- **Merge by default** (`replace` omitted/false) — preserves existing metadata the user didn't ask to change
- `replace: true` swaps the field's entire metadata map — only if the user explicitly asks to overwrite
- A value of `None`/`null` deletes that specific key
- Batch all field updates into a single call when possible
- Returns the new table version
-
-Python (sync and async take one dict per field, as varargs):
-
-```python
-res = table.update_field_metadata(
-    {
-        "path": "clip_v3",
-        "metadata": {
-            "lancedb:description": "CLIP ViT-L/14 image embedding, L2-normalized (1024-dim).",
-            "lancedb:tag:field_type": "embedding",
-            "lancedb:tag:model": "clip",
-            "lancedb:tag:version": "v3",
-            "lancedb:tag:latest": "true",
-            "lancedb:logical-column": "clip",
-        },
-    },
-    {
-        "path": "clip_v2",
-        "metadata": {
-            "lancedb:description": "CLIP ViT-B/32 image embedding (768-dim), superseded by v3.",
-            "lancedb:tag:field_type": "embedding",
-            "lancedb:tag:model": "clip",
-            "lancedb:tag:version": "v2",
-            "lancedb:logical-column": "clip",
-        },
-    },
-)
-print(res.version)  # new table version
-
-# merge semantics: add a key, delete one via None, keep the rest
-table.update_field_metadata(
-    {"path": "clip_v2", "metadata": {"lancedb:tag:archived": "true", "lancedb:tag:latest": None}}
-)
-```
-
-(`replace_field_metadata` is deprecated — use `update_field_metadata`.)
-
-TypeScript (takes an array of `FieldMetadataUpdate`):
-
-```typescript
-const res = await table.updateFieldMetadata([
-  {
-    path: "clip_v3",
-    metadata: {
-      "lancedb:description": "CLIP ViT-L/14 image embedding, L2-normalized (1024-dim).",
-      "lancedb:tag:field_type": "embedding",
-      "lancedb:tag:model": "clip",
-      "lancedb:tag:version": "v3",
-      "lancedb:tag:latest": "true",
-      "lancedb:logical-column": "clip",
-    },
-  },
-  {
-    path: "clip_v2",
-    metadata: {
-      "lancedb:description": "CLIP ViT-B/32 image embedding (768-dim), superseded by v3.",
-      "lancedb:tag:field_type": "embedding",
-      "lancedb:tag:model": "clip",
-      "lancedb:tag:version": "v2",
-      "lancedb:logical-column": "clip",
-    },
-  },
-]);
-console.log(res.version); // new table version
-
-// merge semantics: add a key, delete one via null, keep the rest
-await table.updateFieldMetadata([
-  { path: "clip_v2", metadata: { "lancedb:tag:archived": "true", "lancedb:tag:latest": null } },
-]);
-```
-
-## Step 4: Confirm
-
-Report back:
- Which columns were updated and what was written
- The new table version number (from the result)
- Any columns skipped (e.g., already had up-to-date metadata)
-
-## Quick examples
-
-**"Write descriptions for all columns in the `product_embeddings` table"**
-1. Read `table.schema` → all fields + existing metadata
-2. Generate a `lancedb:description` for each column based on name + type
-3. One `update_field_metadata` call with all descriptions
-4. Report
-
-**"Tag the columns in `model_outputs` with their field type and model"**
-1. Read the schema
-2. For each field, classify by name + Arrow type → set `lancedb:tag:field_type` and `lancedb:tag:model` where applicable
-3. Write in one batched call
-4. Report
-
-**"Group the feature columns in `training_features` into logical families and mark the latest version"**
-1. Read the schema
-2. Find version patterns → assign `lancedb:logical-column` and `lancedb:tag:version`; mark newest with `lancedb:tag:latest: "true"`
-3. Write in one batched call
-4. Show the grouping
@@ -1,138 +0,0 @@
-# Python API Reference
-
-Quick method reference for Python LanceDB code. Cross-check source for non-trivial claims.
-
-## Connect
-
-If you're connecting to a remote database, use this:
-```python
-import lancedb
-
-db = lancedb.connect("db://my-db", api_key=api_key, host_override=host_override) # remote
-```
-(values may be found in LANCEDB_API_KEY and LANCEDB_HOST_OVERRIDE, either in env vars or a .env file)
-
-If you're connecting to a local table using OSS LanceDB, use this:
-```python
-db = lancedb.connect("./camelot-db") # local/OSS
-```
-If you're not sure which, or if you can't find the api_key or host_override params, ask the user.
-
-**Place the local database directory next to the script/entrypoint that opens it** (i.e. resolve the path relative to the script, `Path(__file__).parent / "camelot-db"`), not buried under a shared `data/` folder. The Lance dataset is the database, not a data file — keeping it beside its code makes ownership obvious and paths stable regardless of the working directory the script is launched from.
-
-**Do not name the directory `lancedb`** (e.g. `./lancedb`, `./data/lancedb`). It collides with the imported `lancedb` package name, which is confusing to read and easy to shadow in scripts. Give it a name derived from the repo or dataset with a clear prefix/suffix — for example `./<dataset>-db`, `./<repo>_lancedb`, or `./vectordb`.
-
-Async:
-
-```python
-db = await lancedb.connect_async("./camelot-db")
-```
-
-## Table Reads
-
-| Task | Preferred API |
-| --- | --- |
-| Vector search | `table.search(query_vector).limit(k)` |
-| Full scan with filters/projection (sync) | `table.search().where(...).select(...).limit(...)` |
-| Full scan with filters/projection (async) | `table.query().where(...).select(...).limit(...)` |
-| Filter | `.where("col > 10")` |
-| Projection | `.select(["id", "text"])` |
-| Bound result count | `.limit(20)` |
-| Collect bounded result as Python objects (default, no extra deps) | `.to_list()` on query/search result |
-| Collect bounded result as Arrow (default, `pyarrow` always available) | `.to_arrow()` on query/search result |
-| Collect bounded result as pandas (only if project uses pandas) | `.to_pandas()` on query/search result |
-| Collect bounded result as Polars (only if project uses polars) | `.to_polars()` on query/search result |
-
-## Sync vs Async Scan API
-
-The plain-scan entry point differs between the sync and async clients. **Verified against `lancedb` 0.34.0** — re-check if the pinned version changes:
-
- **Sync** (`lancedb.connect(...)`): the table has **no `.query()` method**. Use `.search()` with no argument for a plain scan; it returns a query builder that supports `.where()`, `.select()`, `.limit()`, and the `.to_list()` / `.to_arrow()` / `.to_pandas()` / `.to_polars()` collectors.
-  ```python
-  rows = table.search().where("status = 'ready'").select(["id", "text"]).limit(20).to_list()
-  ```
- **Async** (`lancedb.connect_async(...)`): the table has **both** `.query()` and `.search()`. Use `.query()` for a plain scan.
-  ```python
-  rows = await async_table.query().where("status = 'ready'").select(["id", "text"]).limit(20).to_list()
-  ```
-
-Do not call `table.query()` on a sync table — it raises `AttributeError`.
-
-## Local vs Remote Table Methods
-
-| API | Local table | Remote table | Agent guidance |
-| --- | --- | --- | --- |
-| `table.search(...)` | Yes | Yes | Preferred read path (sync + async) |
-| `table.query()` | Async only | Async only | Sync scan path is `table.search()`; `.query()` is the async scan builder |
-| `table.to_pandas()` | Yes | No / unsafe for portability | Avoid in portable code |
-| `table.to_arrow()` | Yes | No / unsafe for portability | Avoid in portable code |
-| `table.to_polars()` | Yes | No / unsafe for portability | Avoid in portable code |
-| `table.to_lance()` | Yes | No | Local/OSS escape hatch only |
-
-## Indexes
-
-Use `create_index(...)` for vector indexes and modern index configs. Use scalar indexes for filtered or merge keys.
-
-Common calls:
-
-```python
-table.create_index("vector")
-table.create_scalar_index("status")
-table.create_fts_index("text")
-```
-
-Check source docs before specifying advanced index config names or parameters.
-
-## Filtering And Recall Knobs
-
-```python
-table.search(query_vector).where("status = 'ready'")  # pre-filter by default
-table.search(query_vector).where("status = 'ready'", prefilter=False)
-table.search(query_vector).limit(10).refine_factor(20)
-table.search(query_vector).limit(10).nprobes(50)
-```
-
-Use post-filtering only when fewer than `limit` results are acceptable.
-
-## Diagnostics
-
-```python
-print(table.search(query_vector).where("year > 2000").limit(10).analyze_plan())
-print(table.index_stats("vector_idx"))
-```
-
-Use these before changing indexes or search tuning.
-
-## Column (Field) Metadata
-
-```python
-schema = table.schema                       # sync property; async: await table.schema()
-meta = schema.field("category").metadata    # dict[bytes, bytes] — Arrow metadata is bytes-keyed
-res = table.update_field_metadata(          # varargs: one dict per field; works local + remote
-    {"path": "category", "metadata": {"lancedb:description": "...", "lancedb:tag:field_type": "label"}}
-)
-res.version  # new table version
-```
-
-Merges by default; a `None` value deletes that key; `"replace": True` swaps the whole map. Nested fields use dot-paths (`"a.b.c"`). `replace_field_metadata` is deprecated. See `references/column_metadata.md` for key conventions (`lancedb:description`, `lancedb:tag:<name>`, `lancedb:logical-column`) and the authoring workflow.
-
-## Branches
-
-```python
-table.branches.list()                          # non-main branches; {} = only main
-exp = table.branches.create("exp")             # fork off main -> handle scoped to the branch
-wip = table.branches.checkout("wip")           # existing branch -> scoped handle (version= pins read-only)
-wip = db.open_table("t", branch="wip")         # or open scoped directly
-table.branches.delete("stale")                 # removes only the branch pointer
-table.current_branch()                         # None = main
-```
-
-There is no global switch — scoping is per table handle: any read/write on a branch handle lands on that branch; the original handle keeps targeting main. See `references/branch_ops.md` for the model and isolation checks.
-
-## Maintenance
-
-```python
-table.optimize()
-```
-
-Call this after every successful local/OSS ingestion. It handles compaction, cleanup of old versions according to retention, and index optimization. Do not add this for LanceDB Enterprise/Cloud remote tables; Enterprise handles compaction and cleanup automatically from cluster configuration.
@@ -1,173 +0,0 @@
-# Python Patterns
-
-Use these patterns when writing Python code with `lancedb`.
-
-## Before Writing Code
-
-Choose the output type from what the project actually depends on. **Do not assume `pandas` or `polars` is installed** — they are heavy dependencies that many LanceDB projects do not use. `pyarrow`, by contrast, ships as a LanceDB dependency and is always available, so it is a safe default to lean on.
-
-Default output (after applying `select()` and `limit()`):
-
- **Python objects**: `.to_list()` — a list of dicts, no extra dependencies. Prefer this for scripts, examples, and agent-generated code unless there is a reason to do otherwise.
- **PyArrow**: `.to_arrow()` — a `pyarrow.Table`, when the surrounding code is Arrow-native or you need columnar/zero-copy handoff.
-
-Only reach for a DataFrame when the project *already* declares that dependency:
-
- Pandas projects (pandas in `pyproject.toml`/requirements): `.to_pandas()`.
- Polars projects (polars declared): `.to_polars()`.
-
-If unsure, check the dependency manifest or the imports in surrounding files. When in doubt, use `.to_list()` or `.to_arrow()`.
-
-## Schema Design and Validation
-
-Favor `LanceModel` and Pydantic validation for Python schemas. They keep field
-types readable, validate source records before a write, and map directly to a
-LanceDB schema. Use `Vector(dimension)` for fixed-size vectors:
-
-```python
-from lancedb.pydantic import LanceModel, Vector
-
-class Document(LanceModel):
-    id: int
-    text: str
-    vector: Vector(384, nullable=False)
-
-rows = [Document.model_validate(row) for row in source_rows]
-table = db.create_table("documents", schema=Document)
-table.add(rows)
-```
-
-Use PyArrow schemas instead when the pipeline is already Arrow-native, needs
-record-batch streaming, or has runtime schema requirements that would make a
-Pydantic model harder to understand. Declare Pydantic as a direct project
-dependency when application code imports it, even if LanceDB also depends on it.
-
-## Recommended Patterns
-
-### Bounded search or query
-
-Use this for application reads, examples, notebooks, and agent-generated scripts:
-
-```python
-results = (
-    table.search(query_vector)
-    .where("status = 'ready'")
-    .select(["id", "text"])
-    .limit(20)
-    .to_list()  # or .to_arrow(); .to_pandas()/.to_polars() only if the project uses them
-)
-```
-
-Why: `search()` works across local and remote tables and on both the sync and async clients. `select()` avoids fetching unused columns. `limit()` prevents accidental full-table reads. `.to_list()` and `.to_arrow()` avoid assuming pandas/polars is installed (see "Before Writing Code").
-
-For a **plain scan** (no query vector), the entry point differs by client:
-
-```python
-# Sync client: no .query() method — use .search() with no argument.
-rows = table.search().where("status = 'ready'").select(["id", "text"]).limit(20).to_list()
-
-# Async client: use .query().
-rows = await async_table.query().where("status = 'ready'").select(["id", "text"]).limit(20).to_list()
-```
-
-`table.query()` on a sync table raises `AttributeError` (verified on `lancedb` 0.34.0). See the "Sync vs Async Scan API" section in `api_reference.md`.
-
-### Bounded query result conversion
-
-It is fine to collect bounded query/search results:
-
-```python
-arrow_table = table.search().select(["id"]).limit(100).to_arrow()  # sync plain scan
-rows = table.search(query_vector).limit(10).to_list()
-df = table.search(query_vector).limit(10).to_pandas()  # only if pandas is a project dep
-```
-
-### Local-only Lance dataset API
-
-`table.to_lance()` does not itself materialize the full dataset. It returns the underlying `lance.LanceDataset`, making the table accessible through the PyLance dataset API. Use it when the task is explicitly local/OSS and needs Lance dataset methods not exposed by LanceDB:
-
-```python
-# Local/OSS only: RemoteTable does not expose table.to_lance().
-ds = table.to_lance()
-for batch in ds.to_batches(columns=["id", "text"], batch_size=10_000):
-    process(batch)
-```
-
-### Async Python
-
-Keep the same shape and bound the result before collecting:
-
-```python
-results = await (
-    async_table.query()
-    .where("status = 'ready'")
-    .select(["id", "text"])
-    .limit(20)
-    .to_list()  # or .to_arrow()
-)
-```
-
-## Anti-Patterns
-
-**Avoid the following anti-patterns in your code.**
-
-### Table-level full materialization
-
-Avoid whole-table collectors in portable or large-table code:
-
-```python
-df = table.to_pandas()
-arrow_table = table.to_arrow()
-polars_df = table.to_polars()
-```
-
-Why: local tables expose these whole-table collectors, but remote tables intentionally do not — a remote production table can be far larger than a local development table, so it is easy to accidentally pull the entire table into memory.
-
-`table.to_lance()` is different: it is not a full materialization call, but it is still local/OSS-only and should not appear in code meant to run against remote Enterprise tables.
-
-### Unbounded result collection
-
-Avoid query/search collection without a meaningful limit:
-
-```python
-rows = table.search().to_list()               # unbounded plain scan
-rows = table.search(query_vector).to_list()   # unbounded vector search
-```
-
-Prefer `select(...).limit(...)` before collecting; for large reads, stream in batches instead.
-
-### Per-row writes
-
-Avoid loops that write one row per call:
-
-```python
-for row in rows:
-    table.add([row])  # one commit + fragment per row
-```
-
-Each `add()` creates a new version and fragment. Pass the whole batch in a single call, or chunk very large inputs:
-
-```python
-table.add(rows)  # single commit
-# for very large inputs, add batches of several thousand rows
-```
-
-After the final successful write to an embedded OSS table, call
-`table.optimize()`. Skip this for Enterprise/Cloud tables because their
-maintenance is automatic.
-
-### Drop-then-reuse the same table name (Enterprise/Cloud)
-
-Avoid dropping or overwriting a remote table and then reusing that name right away:
-
-```python
-db.drop_table("my_table")
-table = db.create_table("my_table", data=rows)          # reads 500 for ~5 min
-table = db.create_table("my_table", data=rows, mode="overwrite")  # same problem
-```
-
-Why: Enterprise/Cloud splits DDL (control plane) from query serving (data plane). The data plane caches the dataset behind a table name for up to `table_cache_ttl` (default 300s / 5 min), so after a drop/overwrite the DDL succeeds but queries against the reused name return `500 Internal Server Error` until the cache expires — and a fresh `describe` may still show the old schema. Instead, write to a **fresh name**, use `list_tables()` and fail if it already exists, then `rename_table(fresh, final)` onto the final name only after the old table's drop has propagated (~5 min). See the "Enterprise: never drop-then-reuse the same table name" section in `SKILL.md`. Local/OSS tables have no separate data plane — overwrite freely there.
-
-### Guessing performance fixes
-
-Avoid changing `nprobes`, `refine_factor`, or index types before checking the query plan and index stats. Diagnose first, then tune one knob at a time.
@@ -1,131 +0,0 @@
-# Python Performance Guidance
-
-Use this when writing Python code that ingests data, queries large tables, builds indexes, or investigates latency.
-
-## Ingestion
-
-### Recommended: validate schemas and records with Pydantic
-
-Favor `LanceModel` for readable Python schema definitions and validate source
-records before writing. Use PyArrow directly for Arrow-native or streaming
-pipelines where it is the clearer representation.
-
-```python
-from lancedb.pydantic import LanceModel, Vector
-
-class Document(LanceModel):
-    id: int
-    text: str
-    vector: Vector(384, nullable=False)
-
-rows = [Document.model_validate(row) for row in source_rows]
-table = db.create_table("documents", schema=Document)
-table.add(rows)
-```
-
-### Recommended: bulk ingestion for materialized data
-
-```python
-table.add(arrow_table)
-table.add(df)
-table.add(pa.dataset("data/", format="parquet"))
-```
-
-For very large initial loads, create the table empty first, then call `add(...)`. Passing data directly to `create_table(name, data)` can skip the auto-parallel write path.
-
-### Recommended: iterator ingestion for generated or streamed data
-
-```python
-def batches():
-    for raw in source:
-        vectors = model.encode(raw["text"])
-        yield pa.RecordBatch.from_pydict({**raw, "vector": vectors})
-
-table.add(batches())
-```
-
-Use chunks of several thousand rows or more when practical. Tiny batches and per-row writes create many small fragments.
-
-### Anti-pattern: per-row `add()`
-
-```python
-for row in rows:
-    table.add([row])
-```
-
-Each call creates a version and fragment. This slows ingestion and later queries.
-
-## Indexing
-
- Build a vector index once brute-force vector search becomes too slow. As a rule of thumb, local brute force is fine below roughly 100K vectors; beyond that, build an index.
- Use `IVF_PQ` as the general-purpose default. Enterprise builds this automatically.
- Use scalar indexes for filtered columns and merge/upsert keys.
- Use `BTREE` for mostly distinct numeric/string/temporal columns, `BITMAP` for booleans and low-cardinality columns, and `LABEL_LIST` for list membership queries.
- Keep full-text defaults unless phrase queries require position data.
-
-## Querying
-
-Always be explicit:
-
-```python
-table.search(query_vector).select(["id", "title"]).limit(20)
-```
-
- `select()` reduces bytes read and transferred.
- `limit()` prevents accidental full-table materialization.
- Pre-filtering is the default and guarantees returned rows satisfy the predicate.
- Use post-filtering only when fewer than `limit` results are acceptable.
-
-## Recall Tuning
-
-Tune one knob at a time:
-
- Quantized indexes: raise `refine_factor` to rescore more candidates on full vectors.
- HNSW-backed indexes: raise `ef`; start around `1.5 * k`, increase toward `10 * k` if recall is short.
- IVF candidate breadth: `nprobes` is auto-tuned; override only when a selective pre-filter leaves too few neighbors.
-
-## Maintenance
-
-After every successful embedded OSS/local ingestion, call `table.optimize()`.
-Do not add this to LanceDB Enterprise/Cloud remote table code; remote compaction
-and cleanup are handled automatically based on the Enterprise cluster
-configuration.
-
-Why local maintenance is needed:
-
- Frequent writes can create many small fragments. Queries then need to scan across more files, which can increase latency.
- Updates, deletes, and appends create new table versions. Old versions are retained for time travel and rollback, which can grow disk usage.
- Indexes may have newly added rows that are not yet fully optimized into the index structure.
-
-For local/OSS tables, run `optimize()` after the final successful ingestion
-write. Also run it after later batches of update/delete operations or on a
-regular maintenance schedule:
-
-```python
-table.optimize()
-```
-
-If the user wants more aggressive local disk cleanup, pass a shorter cleanup retention window:
-
-```python
-from datetime import timedelta
-
-table.optimize(cleanup_older_than=timedelta(days=1))
-```
-
-Do not use very short cleanup windows when the application depends on time travel, rollback, or old versions.
-
-## Diagnostics
-
-Before changing code or indexes, inspect:
-
-```python
-print(table.search(query_vector).where("year > 2000").limit(10).analyze_plan())
-print(table.index_stats("vector_idx"))
-```
-
-Look for high scan bytes, missing indexes, fragmented data, and unindexed rows.
-
-## Python Multiprocessing
-
-When using multiprocessing, use `spawn` rather than `fork`. LanceDB is multi-threaded internally, and `fork` plus a multi-threaded process is unsafe.
@@ -1,45 +0,0 @@
-# Connecting to a LanceDB remote server
-
-LanceDB Enterprise/Cloud deployments are served by a server implementing the
-lance-namespace OpenAPI spec
-(<https://github.com/lance-format/lance-namespace/blob/main/docs/src/spec.yaml>).
-Every remote (`db://...`) connection talks to such a server, and some operations
-exist only there. In particular, all operations around jobs (listing, inspecting,
-creating, or canceling jobs) run server-side — there is no local/OSS equivalent, so
-resolve a server connection before attempting any job work. The job REST methods
-themselves are documented in `references/remote_jobs.md`.
-
-Every request needs two things:
-
-1. **Base URL** — the server endpoint
-2. **Credentials** — an API key (`x-api-key` header over REST), and usually a database name (`x-lancedb-database` header)
-
-## Resolution steps
-
-1. If the user already gave a URL and API key (or said which environment they're working against), use that.
-2. Otherwise, look for credentials already available in the environment:
-   - Env vars like `LANCEDB_URI` / `LANCEDB_HOST` / `LANCEDB_API_KEY`
-   - A server endpoint already running or port-forwarded locally (the REST default port is 2333, i.e. `http://localhost:2333`)
-3. If you didn't find both pieces, ask the user directly: **"What's your LanceDB endpoint's URL, and what's your API key?"** Also ask which database to use if it isn't obvious. Don't guess or probe further — the user knows their deployment.
-
-## Validating the connection
-
-Make a cheap authenticated request and check the status before starting real work:
-
-```bash
-curl -s -w "\n%{http_code}" "{base_url}/v1/table/?limit=1" \
-  -H "x-api-key: <key>" \
-  -H "x-lancedb-database: <database>"
-```
-
- `200` — connection, key, and database header all good
- `401` — API key missing or wrong
- `400` mentioning a database header — this deployment expects `x-lancedb-database`
-
-## Non-REST equivalents
-
-The same credentials work through the SDKs and CLI:
-
- Python SDK: `lancedb.connect("db://<database>", api_key="<key>", host_override="<base_url>")`
- TypeScript SDK: `await lancedb.connect("db://<database>", { apiKey: "<key>", hostOverride: "<base_url>" })`
- `lancedb` CLI: a `[profiles.<name>]` entry in `~/.lancedb/config.toml` with `http_server_url`, `api_key`, `database`
@@ -1,151 +0,0 @@
-# Job operations over the LanceDB remote server REST API
-
-Jobs are server-side background operations on LanceDB Enterprise/Cloud — index builds,
-column backfills, materialized view refreshes, and similar async work. Endpoints that
-trigger async work (e.g. the column backfill or materialized view refresh endpoints)
-return a `job_id`; these four methods are how you track and manage those jobs.
-
-Resolve the connection first — see `references/remote_connect.md`. All four methods
-are **POST** requests under `{base_url}/v1/jobs/` with JSON bodies, and take the usual
-`x-api-key` / `x-lancedb-database` headers. If every job call returns `501`, job APIs
-are disabled on that deployment (the server has no job registry configured) — report
-that rather than retrying.
-
-## 1. List jobs — `POST /v1/jobs/list`
-
-The body is optional; an empty body lists everything. All fields are filters:
-
-```json
-{
-  "limit": 100,
-  "table_name": "my_table",
-  "job_type": "...",
-  "job_subtype": "...",
-  "state": "...",
-  "page_token": "..."
-}
-```
-
-```bash
-curl -s -X POST "{base_url}/v1/jobs/list" \
-  -H "x-api-key: <key>" -H "x-lancedb-database: <database>" \
-  -H "content-type: application/json" \
-  -d '{"table_name": "my_table"}'
-```
-
-Response:
-
-```json
-{
-  "jobs": [
-    {
-      "job_id": "...",
-      "table": "my_table",
-      "job_type": "...",
-      "job_subtype": "...",
-      "state": "done",
-      "created_at_millis": 1720000000000
-    }
-  ],
-  "page_token": "..."
-}
-```
-
-A `page_token` in the response means there are more results — pass it back in the next
-request to continue. Note list rows use a lowercase `state` string, while describe uses
-an uppercase `job_state`.
-
-## 2. Describe a job — `POST /v1/jobs/describe`
-
-Body: `{"job_id": "<id>"}`. Returns full detail for one job:
-
-```json
-{
-  "job_id": "...",
-  "job_type": "...",
-  "job_subtype": "...",
-  "job_state": "IN_PROGRESS",
-  "creation_ms": 1720000000000,
-  "spec": {},
-  "status": {}
-}
-```
-
-`job_state` is one of `IN_PROGRESS`, `CANCELLED`, `FAILED`, `DONE`. `spec` and `status`
-are job-type-specific JSON objects (the job's input specification and its current
-progress/status). Returns `404` for an unknown job id.
-
-## 3. Cancel a job — `POST /v1/jobs/cancel`
-
-Body: `{"job_id": "<id>"}`; response echoes `{"job_id": "<id>"}`. Cancellation is a
-service-level operation requiring the same administrative authorization as the
-`/admin` routes — a database-scoped API key that can list and describe jobs may still
-get a permission error here. Other errors: `404` unknown job, `409` state conflict
-(e.g. already in a terminal state), `429` too much write contention (safe to retry).
-
-## 4. Query job event history — `POST /v1/jobs/query_events`
-
-Returns the event history (state transitions, progress updates) for one or more jobs.
-Body: `{"job_id": "<id>"}` for one job, or `{"job_ids": ["<id>", ...]}` for a batch.
-Optional fields: `limit` (max event rows), `limit_per_job` (per job in a batch query),
-and `filter` — a SQL-like expression over the columns `state`, `updated_by`,
-`owner_component`, and `claim_entity`. (`full_text_search` is reserved and currently
-rejected as not implemented.)
-
-The response is **not JSON** — it is an Arrow IPC stream
-(`content-type: application/vnd.apache.arrow.stream`). Decode it, e.g. in Python:
-
-```python
-import pyarrow.ipc
-import requests
-
-resp = requests.post(
-    f"{base_url}/v1/jobs/query_events",
-    headers={"x-api-key": key, "x-lancedb-database": database},
-    json={"job_id": job_id},
-)
-resp.raise_for_status()
-events = pyarrow.ipc.open_stream(resp.content).read_all()
-```
-
-## Feature engineering (Geneva) jobs
-
-Feature engineering jobs — UDF column backfills and materialized view refreshes run
-through Geneva — are tracked **separately** from the `/v1/jobs` registry above. Their
-records live in a `geneva_jobs` table inside the database itself (in the `__system`
-namespace), and you access them through a Python `geneva` connection rather than the
-REST endpoints above:
-
-```python
-import geneva
-from geneva.jobs import JobStateManager
-
-# Same credentials as lancedb.connect / the REST API
-conn = geneva.connect("db://<database>", api_key="<key>", host_override="<base_url>")
-jsm = JobStateManager(conn)
-
-# List jobs. NOTE: status defaults to "RUNNING"; pass status=None for all jobs.
-# Statuses: PENDING | RUNNING | DONE | FAILED | CANCELLED
-jobs = jsm.list_jobs(table_name="my_table", status=None)
-
-# Fetch one job by id (returns a list of JobRecord)
-records = jsm.get("<job_id>")
-```
-
-Each `JobRecord` has `table_name`, `column_name`, `job_id`, `job_type`, `status`,
-`launched_at`, `completed_at`, `config`, `launched_by`, `manifest_id`, `cluster_name`,
-`metrics` (progress counters), `events` (human-readable history), and `updated_at`.
-For filters `list_jobs` doesn't support (e.g. time ranges), query the underlying table
-directly: `jsm.get_table(True).search().where("launched_at >= TIMESTAMP '...'")` —
-pass `True` to check out the latest version, since other processes update job state.
-
-Stale-status caveat: nothing reaps dead Geneva jobs, so a job can sit in
-`RUNNING`/`PENDING` forever if its worker died. Treat a job as effectively `FAILED`
-when it has been running longer than ~36 hours, or its `updated_at` is more than ~2
-hours old (this matches the heuristic the Geneva console UI applies on read).
-
-## Workflow tips
-
- To wait for async work (a backfill, an index build), poll `describe` until
-  `job_state` leaves `IN_PROGRESS`; on `FAILED`, pull `status` and `query_events` for
-  the failure detail.
@@ -1,105 +0,0 @@
-# TypeScript API Reference
-
-Quick method reference for TypeScript LanceDB code. Cross-check source for non-trivial claims.
-
-## Connect
-
-```typescript
-import * as lancedb from "@lancedb/lancedb";
-
-const db = await lancedb.connect("./camelot-db");
-```
-
-**Place the local database directory next to the script/entrypoint that opens it** (resolve the path relative to the module, e.g. via `import.meta.dirname` / `__dirname`), not buried under a shared `data/` folder. The Lance dataset is the database, not a data file — keeping it beside its code makes ownership obvious and paths stable regardless of the working directory the script is launched from.
-
-**Do not name the directory `lancedb`** (e.g. `./lancedb`, `./data/lancedb`). It collides with the imported `lancedb` package/namespace, which is confusing to read. Give it a name derived from the repo or dataset with a clear prefix/suffix — for example `./<dataset>-db`, `./<repo>_lancedb`, or `./vectordb`.
-
-Remote connections use `db://...` plus Enterprise/Cloud credentials and deployment settings. Check current source/docs for exact connection options.
-
-## Table Reads
-
-| Task | Preferred API |
-| --- | --- |
-| Vector search | `table.search(queryVector).limit(k)` |
-| Full scan with filters/projection | `table.query().where(...).select(...).limit(...)` |
-| Filter | `.where("col > 10")` |
-| Projection | `.select(["id", "text"])` |
-| Bound result count | `.limit(20)` |
-| Collect bounded result as objects | `.toArray()` on query/search result |
-| Collect bounded result as Arrow | `.toArrow()` on query/search result |
-| Stream result batches | `for await (const batch of table.query()...)` |
-
-## Local vs Remote Safety
-
-| API | Agent guidance |
-| --- | --- |
-| `table.search(...)` | Preferred read path |
-| `table.query()` | Preferred scan/filter path |
-| `await table.toArrow()` | Avoid in portable or large-table code |
-| `await table.query().toArray()` with no `limit()` | Avoid; unbounded collection |
-| `await table.query().toArrow()` with no `limit()` | Avoid; unbounded collection |
-
-## Indexes
-
-```typescript
-await table.createIndex("vector");
-await table.createIndex("status");
-```
-
-Use vector indexes for large vector search workloads and scalar indexes for filtered columns or merge/upsert keys. Check source/docs before specifying advanced index options.
-
-## Filtering And Recall Knobs
-
-```typescript
-await table.search(queryVector).where("status = 'ready'").limit(10).toArray();
-await table.search(queryVector).limit(10).refineFactor(20).toArray();
-await table.search(queryVector).limit(10).nprobes(50).toArray();
-await table.search(queryVector).limit(10).ef(100).toArray();
-await table.search(queryVector).where("status = 'ready'").postfilter().limit(10).toArray();
-```
-
-Use `postfilter()` only when fewer than `limit` results are acceptable.
-
-## Diagnostics
-
-```typescript
-console.log(await table.search(queryVector).where("year > 2000").limit(10).analyzePlan());
-console.log(await table.indexStats("vector_idx"));
-```
-
-Use these before changing indexes or search tuning.
-
-## Column (Field) Metadata
-
-```typescript
-const schema = await table.schema();
-const meta = schema.fields.find((f) => f.name === "category")?.metadata; // Map<string, string>
-const res = await table.updateFieldMetadata([
-  { path: "category", metadata: { "lancedb:description": "...", "lancedb:tag:field_type": "label" } },
-]);
-res.version; // new table version
-```
-
-Merges by default; a `null` value deletes that key; `replace: true` swaps the whole map. Nested fields use dot-paths (`"a.b.c"`). See `references/column_metadata.md` for key conventions (`lancedb:description`, `lancedb:tag:<name>`, `lancedb:logical-column`) and the authoring workflow.
-
-## Branches
-
-```typescript
-const branches = await table.branches();       // async manager
-await branches.list();                         // non-main branches; {} = only main
-const exp = await branches.create("exp");      // fork off main -> Table scoped to the branch
-const wip = await branches.checkout("wip");    // existing branch -> scoped Table (version arg pins read-only)
-const wip2 = await db.openTable("t", { branch: "wip" }); // or open scoped directly
-await branches.delete("stale");                // removes only the branch pointer
-table.currentBranch();                         // null = main
-```
-
-There is no global switch — scoping is per table handle: any read/write on a branch handle lands on that branch; the original handle keeps targeting main. See `references/branch_ops.md` for the model and isolation checks.
-
-## Maintenance
-
-```typescript
-await table.optimize();
-```
-
-Call this after every successful local/OSS ingestion. It handles compaction, cleanup of old versions according to retention, and index optimization. Do not add this for LanceDB Enterprise/Cloud remote tables; Enterprise handles compaction and cleanup automatically from cluster configuration.
@@ -1,100 +0,0 @@
-# TypeScript Patterns
-
-Use these patterns when writing TypeScript code with `@lancedb/lancedb`.
-
-## Recommended Patterns
-
-### Bounded query
-
-Use this for application reads, scripts, and examples:
-
-```typescript
-const rows = await table
-  .query()
-  .where("status = 'ready'")
-  .select(["id", "text"])
-  .limit(20)
-  .toArray();
-```
-
-### Bounded vector search
-
-```typescript
-const rows = await table
-  .search(queryVector)
-  .select(["id", "text"])
-  .limit(20)
-  .toArray();
-```
-
-### Batch streaming for larger reads
-
-When the task needs many rows, avoid collecting everything at once:
-
-```typescript
-for await (const batch of table
-  .query()
-  .where("status = 'ready'")
-  .select(["id", "text"])
-  .limit(10_000)) {
-  process(batch);
-}
-```
-
-## Anti-Patterns
-
-**Avoid the following anti-patterns in your code.**
-
-### Table-level full materialization
-
-Avoid whole-table collectors in portable or large-table code:
-
-```typescript
-const tableArrow = await table.toArrow();
-```
-
-Why: local tables expose these whole-table collectors, but remote tables intentionally do not — a remote production table can be far larger than a local development table, so it is easy to accidentally pull the entire table into memory.
-
-### Unbounded result collection
-
-Avoid query/search collection without a meaningful limit:
-
-```typescript
-const rows = await table.query().toArray();              // unbounded plain scan
-const rows = await table.search(queryVector).toArray();  // unbounded vector search
-```
-
-Prefer `select(...).limit(...)` before collecting; for large reads, stream in batches instead.
-
-### Per-row writes
-
-Avoid loops that write one row per call:
-
-```typescript
-for (const row of rows) {
-  await table.add([row]); // one commit + fragment per row
-}
-```
-
-Each `add()` creates a new version and fragment. Pass the whole batch in a single call, or chunk very large inputs:
-
-```typescript
-await table.add(rows); // single commit
-// for very large inputs, add in chunks of several thousand rows
-```
-
-### Drop-then-reuse the same table name (Enterprise/Cloud)
-
-Avoid dropping or overwriting a remote table and then reusing that name right away:
-
-```typescript
-await db.dropTable("my_table");
-const table = await db.createTable("my_table", rows);                    // reads 500 for ~5 min
-const table = await db.createTable("my_table", rows, { mode: "overwrite" }); // same problem
-```
-
-Why: Enterprise/Cloud splits DDL (control plane) from query serving (data plane). The data plane caches the dataset behind a table name for up to `table_cache_ttl` (default 300s / 5 min), so after a drop/overwrite the DDL succeeds but queries against the reused name return `500 Internal Server Error` until the cache expires — and a fresh `describe` may still show the old schema. Instead, write to a **fresh name**, use `tableNames()` and fail if it already exists, then `renameTable(fresh, final)` onto the final name only after the old table's drop has propagated (~5 min). See the "Enterprise: never drop-then-reuse the same table name" section in `SKILL.md`. Local/OSS tables have no separate data plane — overwrite freely there.
-
-### Guessing performance fixes
-
-Avoid changing `nprobes`, `refineFactor`, `ef`, or index settings before checking `analyzePlan()` and `indexStats(...)`. Diagnose first, then tune one knob at a time.
@@ -1,78 +0,0 @@
-# TypeScript Performance Guidance
-
-Use this when writing TypeScript code that ingests data, queries large tables, builds indexes, or investigates latency.
-
-## Ingestion
-
- Prefer bulk or batched writes.
- Avoid per-row write loops; they create many small commits/fragments.
- For generated data, accumulate reasonable batches before adding.
- For file-backed data, prefer APIs that stream from Arrow/Parquet-style inputs when available.
-
-## Indexing
-
- Build a vector index once brute-force vector search becomes too slow. As a rule of thumb, local brute force is fine below roughly 100K vectors; beyond that, build an index.
- Use the general-purpose vector index defaults unless the task has explicit recall/latency requirements.
- Build scalar indexes for filtered columns and merge/upsert keys.
- Use full-text index phrase options only when phrase queries require them.
-
-## Querying
-
-Always be explicit:
-
-```typescript
-await table.search(queryVector).select(["id", "title"]).limit(20).toArray();
-```
-
- `select()` reduces bytes read and transferred.
- `limit()` prevents accidental full-table collection.
- Pre-filtering is the default behavior. Use `postfilter()` only when fewer than `limit` results are acceptable.
-
-## Recall Tuning
-
-Tune one knob at a time:
-
- Quantized indexes: raise `refineFactor(...)` to rescore more candidates on full vectors.
- HNSW-backed indexes: raise `ef(...)`; start around `1.5 * k`, increase toward `10 * k` if recall is short.
- IVF candidate breadth: `nprobes(...)` is usually auto-tuned; override only when a selective pre-filter leaves too few neighbors.
-
-## Maintenance
-
-After every successful embedded OSS/local ingestion, call `table.optimize()`.
-Do not add this to LanceDB Enterprise/Cloud remote table code; remote compaction
-and cleanup are handled automatically based on the Enterprise cluster
-configuration.
-
-Why local maintenance is needed:
-
- Frequent writes can create many small fragments. Queries then need to scan across more files, which can increase latency.
- Updates, deletes, and appends create new table versions. Old versions are retained for time travel and rollback, which can grow disk usage.
- Indexes may have newly added rows that are not yet fully optimized into the index structure.
-
-For local/OSS tables, run `optimize()` after the final successful ingestion
-write. Also run it after later batches of update/delete operations or on a
-regular maintenance schedule:
-
-```typescript
-await table.optimize();
-```
-
-If the user wants more aggressive local disk cleanup, pass a shorter cleanup retention window:
-
-```typescript
-const olderThan = new Date(Date.now() - 24 * 60 * 60 * 1000);
-await table.optimize({ cleanupOlderThan: olderThan });
-```
-
-Do not use very short cleanup windows when the application depends on time travel, rollback, or old versions.
-
-## Diagnostics
-
-Before changing code or indexes, inspect:
-
-```typescript
-console.log(await table.search(queryVector).where("year > 2000").limit(10).analyzePlan());
-console.log(await table.indexStats("vector_idx"));
-```
-
-Look for high scan cost, missing indexes, fragmented data, and unindexed rows.
@@ -1,135 +0,0 @@
-#!/usr/bin/env python3
-"""Scan Python and TypeScript for likely unsafe LanceDB materialization."""
-
-from __future__ import annotations
-
-import argparse
-import re
-import sys
-from dataclasses import dataclass
-from pathlib import Path
-
-
-PY_FULL_TABLE = re.compile(r"\b\w+\.(to_pandas|to_arrow|to_polars)\s*\(")
-TS_TABLE_TO_ARROW = re.compile(r"\b\w+\.toArrow\s*\(")
-TS_QUERY_COLLECTOR = re.compile(r"\.query\s*\(\s*\)[\s\S]*?\.to(Array|Arrow)\s*\(")
-
-
-@dataclass
-class Finding:
-    path: Path
-    line: int
-    message: str
-    text: str
-
-
-def iter_files(paths: list[Path]) -> list[Path]:
-    files: list[Path] = []
-    for path in paths:
-        if path.is_dir():
-            files.extend(
-                p
-                for p in path.rglob("*")
-                if p.suffix in {".py", ".ts", ".tsx"} and "node_modules" not in p.parts
-            )
-        elif path.suffix in {".py", ".ts", ".tsx"}:
-            files.append(path)
-    return sorted(set(files))
-
-
-def line_number(text: str, offset: int) -> int:
-    return text.count("\n", 0, offset) + 1
-
-
-def scan_python(path: Path, text: str) -> list[Finding]:
-    findings: list[Finding] = []
-    for match in PY_FULL_TABLE.finditer(text):
-        line_start = text.rfind("\n", 0, match.start()) + 1
-        line_end = text.find("\n", match.start())
-        if line_end == -1:
-            line_end = len(text)
-        line = text[line_start:line_end].strip()
-        if ".search(" in line or ".query(" in line:
-            continue
-        findings.append(
-            Finding(
-                path,
-                line_number(text, match.start()),
-                f"Review Python `{match.group(1)}()` call; table-level materialization is not portable to remote tables.",
-                line,
-            )
-        )
-    return findings
-
-
-def statement_around(text: str, start: int, end: int) -> str:
-    before = max(text.rfind(";", 0, start), text.rfind("\n\n", 0, start))
-    after_candidates = [pos for pos in (text.find(";", end), text.find("\n\n", end)) if pos != -1]
-    after = min(after_candidates) if after_candidates else len(text)
-    return text[before + 1 : after].strip()
-
-
-def scan_typescript(path: Path, text: str) -> list[Finding]:
-    findings: list[Finding] = []
-    for match in TS_TABLE_TO_ARROW.finditer(text):
-        stmt = statement_around(text, match.start(), match.end())
-        if ".query(" in stmt or ".search(" in stmt:
-            continue
-        findings.append(
-            Finding(
-                path,
-                line_number(text, match.start()),
-                "Review TypeScript `table.toArrow()`-style call; table-level materialization is not portable for large/remote tables.",
-                stmt.splitlines()[0].strip(),
-            )
-        )
-
-    for match in TS_QUERY_COLLECTOR.finditer(text):
-        stmt = statement_around(text, match.start(), match.end())
-        if ".limit(" in stmt:
-            continue
-        findings.append(
-            Finding(
-                path,
-                line_number(text, match.start()),
-                "Review unbounded TypeScript query collection; add `limit()` or stream batches.",
-                stmt.splitlines()[0].strip(),
-            )
-        )
-    return findings
-
-
-def scan_file(path: Path) -> list[Finding]:
-    text = path.read_text(encoding="utf-8", errors="replace")
-    if path.suffix == ".py":
-        return scan_python(path, text)
-    if path.suffix in {".ts", ".tsx"}:
-        return scan_typescript(path, text)
-    return []
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("paths", nargs="+", type=Path)
-    parser.add_argument(
-        "--no-fail", action="store_true", help="Always exit 0 after reporting findings."
-    )
-    args = parser.parse_args()
-
-    findings: list[Finding] = []
-    for path in iter_files(args.paths):
-        findings.extend(scan_file(path))
-
-    for finding in findings:
-        print(f"{finding.path}:{finding.line}: {finding.message}")
-        print(f"  {finding.text}")
-
-    if findings:
-        print(
-            f"\n{len(findings)} finding(s). Review manually; bounded query result conversion may be OK."
-        )
-    return 0 if args.no_fail or not findings else 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.38.0-beta.2"
+version = "0.38.0-beta.3"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"
@@ -15,10 +15,10 @@ name = "_lancedb"
 crate-type = ["cdylib"]

 [dependencies]
-arrow = { version = "58.0.0", features = ["pyarrow"] }
-async-trait = "0.1"
-bytes = "1"
-lancedb = { path = "../rust/lancedb", default-features = false }
+arrow = { workspace = true, features = ["pyarrow"] }
+async-trait.workspace = true
+bytes.workspace = true
+lancedb.workspace = true
 datafusion-common.workspace = true
 lance-core.workspace = true
 lance-namespace.workspace = true
@@ -26,25 +26,24 @@ lance-namespace-impls.workspace = true
 lance-io.workspace = true
 env_logger.workspace = true
 log.workspace = true
-pyo3 = { version = "0.28", features = ["extension-module", "abi3-py310", "chrono"] }
-chrono = { version = "0.4", default-features = false, features = ["clock"] }
+# Maturin enables extension-module mode for Python builds. Keeping it out of
+# Cargo features lets Rust unit tests link against libpython.
+pyo3 = { version = "0.28", features = ["abi3-py310", "chrono"] }
+chrono.workspace = true
 pyo3-async-runtimes = { version = "0.28", features = [
    "attributes",
    "tokio-runtime",
 ] }
-pin-project = "1.1.5"
+pin-project.workspace = true
 futures.workspace = true
-serde = "1"
-serde_json = "1"
+serde.workspace = true
+serde_json.workspace = true
 snafu.workspace = true
-tokio = { version = "1.40", features = ["sync", "rt-multi-thread"] }
+tokio.workspace = true
 libc = "0.2"

 [build-dependencies]
-pyo3-build-config = { version = "0.28", features = [
-    "extension-module",
-    "abi3-py310",
-] }
+pyo3-build-config = { version = "0.28", features = ["abi3-py310"] }

 [features]
 default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/cos", "lancedb/goosefs", "lancedb/metrics-otel"]
@@ -38,6 +38,25 @@ Stable releases are created about every 2 weeks. For the latest features and bug
 pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb
 ```

+### Threading in CPU-limited containers
+
+LanceDB uses separate pools for compute work and storage I/O. On a container with
+two visible CPUs, current releases intentionally use one compute worker by default;
+no manual configuration is needed. If every query logs an I/O core reservation
+warning on a two-CPU container, upgrade from LanceDB 0.21.1 or earlier.
+
+The two commonly tuned environment variables control different resources:
+
+- `LANCE_CPU_THREADS` overrides the number of compute workers. One worker is the
+  appropriate setting for a two-CPU container when an explicit override is needed.
+- `LANCE_IO_THREADS` controls concurrent storage operations, not reserved CPU
+  cores. Its default can be greater than the number of CPUs because I/O workers
+  spend much of their time waiting for storage.
+
+Keep the defaults unless measurements show that the workload benefits from an
+override. See the [Lance threading model](https://lance.org/guide/performance/#threading-model)
+for the current defaults and tuning guidance.
+
 ## Usage

 ### Basic Example
@@ -8,7 +8,7 @@ dependencies = [
    "overrides>=0.7; python_version<'3.12'",
    "packaging>=23.0",
    "pyarrow>=16",
-    "pydantic>=1.10",
+    "pydantic>=2.7.4,<3",
    "tqdm>=4.27.0",
    "lance-namespace>=0.3.2"
 ]
@@ -103,7 +103,7 @@ python-source = "python"
 module-name = "lancedb._lancedb"

 [build-system]
-requires = ["maturin>=1.4"]
+requires = ["maturin>=1.9.4"]
 build-backend = "maturin"

 [tool.ruff.lint]
@@ -22,6 +22,16 @@ from .remote.db import RemoteDBConnection
 from .expr import Expr, col, lit, func
 from .schema import blob, vector, BlobType
 from .job import AsyncJob, Job
+from .functions import (
+    FunctionArtifactRequest as FunctionArtifactRequest,
+    FunctionApplication as FunctionApplication,
+    FunctionBinding as FunctionBinding,
+    FunctionRegistrationRequest as FunctionRegistrationRequest,
+    FunctionVersion as FunctionVersion,
+    PythonRuntimeSpec as PythonRuntimeSpec,
+    UdfDefinition as UdfDefinition,
+    udf as udf,
+)
 from .table import AsyncTable, Table
 from .types import BaseTokenizerType
 from ._lancedb import Session
@@ -147,6 +147,8 @@ class Connection(object):
        limit: Optional[int],
    ) -> list[str]: ...  # Deprecated: Use list_tables instead
    def job(self, job_id: str) -> Job: ...
+    async def create_function_async(self, request_json: str) -> FunctionJob: ...
+    async def get_function(self, name: str, version: str) -> str: ...
    async def list_jobs(self) -> List[JobInfo]: ...
    async def get_job(self, job_id: str) -> Optional[JobDescription]: ...
    async def cancel_job(self, job_id: str) -> bool: ...
@@ -226,6 +228,13 @@ class Job:
    async def wait(self) -> None: ...
    async def cancel(self) -> None: ...

+class FunctionJob:
+    @property
+    def id(self) -> Optional[str]: ...
+    async def status(self) -> str: ...
+    async def wait(self) -> str: ...
+    async def cancel(self) -> None: ...
+
 class JobInfo:
    @property
    def job_id(self) -> str: ...
@@ -341,6 +350,9 @@ class Table:
    async def add_computed_columns(
        self, columns: list[tuple[str, str]]
    ) -> AddColumnsResult: ...
+    async def add_function_columns(
+        self, application_json: str, output_name: Optional[str]
+    ) -> AddColumnsResult: ...
    async def refresh_column(self, column: str) -> RefreshColumnResult: ...
    async def refresh_column_async(self, column: str) -> Job: ...
    async def add_columns_with_schema(self, schema: pa.Schema) -> AddColumnsResult: ...
@@ -45,7 +45,8 @@ from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError

 from . import __version__
 from ._lancedb import connect as lancedb_connect  # type: ignore
-from .job import AsyncJob, Job
+from .functions import FunctionVersion, UdfDefinition
+from .job import AsyncJob, Job, _function_job
 from .table import (
    AsyncTable,
    LanceTable,
@@ -616,6 +617,31 @@ class DBConnection(EnforceOverrides):
        """
        raise NotImplementedError("serialize is not supported for this connection type")

+    def create_function(self, definition: UdfDefinition) -> FunctionVersion:
+        """Register a scalar Python UDF and wait for its immutable version.
+
+        This is the blocking counterpart of :meth:`create_function_async`.
+        Local connections raise ``NotImplementedError``.
+        """
+        return self.create_function_async(definition).wait()
+
+    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
+        """Register a scalar Python UDF through the remote Function catalog.
+
+        Submission returns a typed job. The immutable Function version becomes
+        available only when :meth:`Job.wait` succeeds. Local connections raise
+        ``NotImplementedError``.
+        """
+        raise NotImplementedError(
+            "Function catalog operations are not supported for this connection type"
+        )
+
+    def get_function(self, name: str, *, version: str) -> FunctionVersion:
+        """Open one exact immutable Function version from the remote catalog."""
+        raise NotImplementedError(
+            "Function catalog operations are not supported for this connection type"
+        )
+
    def job(self, job_id: str) -> Job:
        """A [Job][lancedb.job.Job] handle for a server-side job by id.

@@ -1256,6 +1282,15 @@ class LanceDBConnection(DBConnection):
        """
        return Job(self._conn.job(job_id))

+    @override
+    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
+        job = LOOP.run(self._conn.create_function_async(definition))
+        return Job(job)
+
+    @override
+    def get_function(self, name: str, *, version: str) -> FunctionVersion:
+        return LOOP.run(self._conn.get_function(name, version=version))
+
    @override
    def list_jobs(self) -> List[JobInfo]:
        """List server-side jobs across the database's tables."""
@@ -2023,6 +2058,25 @@ class AsyncConnection(object):
        """
        return AsyncJob(self._inner.job(job_id))

+    async def create_function_async(
+        self, definition: UdfDefinition
+    ) -> AsyncJob[FunctionVersion]:
+        """Register a scalar Python UDF through the remote Function catalog.
+
+        The returned typed job resolves to the immutable Function version.
+        Local connections raise ``NotImplementedError``.
+        """
+        if not isinstance(definition, UdfDefinition):
+            raise TypeError("create_function_async requires a @udf definition")
+        inner = await self._inner.create_function_async(
+            definition.registration_request.to_canonical_json()
+        )
+        return _function_job(inner)
+
+    async def get_function(self, name: str, *, version: str) -> FunctionVersion:
+        """Open one exact immutable Function version from the remote catalog."""
+        return FunctionVersion.from_json(await self._inner.get_function(name, version))
+
    async def list_jobs(self) -> List[JobInfo]:
        """List server-side jobs across the database's tables."""
        return await self._inner.list_jobs()
@@ -26,7 +26,6 @@ class EmbeddingFunction(BaseModel, ABC):
    3. ndims() which returns the number of dimensions of the vector column
    """

-    __slots__ = ("__weakref__",)  # pydantic 1.x compatibility
    max_retries: int = (
        7  # Setting 0 disables retires. Maybe this should not be enabled by default,
    )
@@ -7,8 +7,7 @@ from functools import cached_property
 from typing import List, Union

 import numpy as np
-
-from lancedb.pydantic import PYDANTIC_VERSION
+from pydantic import ConfigDict

 from ..util import attempt_import_or_raise
 from .base import TextEmbeddingFunction
@@ -67,13 +66,7 @@ class BedRockText(TextEmbeddingFunction):
    source_input_type: str = "search_document"
    query_input_type: str = "search_query"

-    if PYDANTIC_VERSION.major < 2:  # Pydantic 1.x compat
-
-        class Config:
-            keep_untouched = (cached_property,)
-    else:
-        model_config = dict()
-        model_config["ignored_types"] = (cached_property,)
+    model_config = ConfigDict(ignored_types=(cached_property,))

    def ndims(self):
        # return len(self._generate_embedding("test"))
@@ -7,8 +7,7 @@ from functools import cached_property
 from typing import List, Optional, Union

 import numpy as np
-
-from lancedb.pydantic import PYDANTIC_VERSION
+from pydantic import ConfigDict

 from ..util import attempt_import_or_raise
 from .base import TextEmbeddingFunction
@@ -87,13 +86,7 @@ class GeminiText(TextEmbeddingFunction):
    query_task_type: str = "retrieval_query"
    source_task_type: str = "retrieval_document"

-    if PYDANTIC_VERSION.major < 2:  # Pydantic 1.x compat
-
-        class Config:
-            keep_untouched = (cached_property,)
-    else:
-        model_config = dict()
-        model_config["ignored_types"] = (cached_property,)
+    model_config = ConfigDict(ignored_types=(cached_property,))

    def ndims(self):
        if self.dim:
@@ -7,14 +7,13 @@ from typing import List, Union

 import numpy as np
 import pyarrow as pa
+from pydantic import ConfigDict

 from ..util import attempt_import_or_raise
 from .base import EmbeddingFunction
 from .registry import register
 from .utils import AUDIO, IMAGES, TEXT

-from lancedb.pydantic import PYDANTIC_VERSION
-

@register("imagebind")
 class ImageBindEmbeddings(EmbeddingFunction):
@@ -31,13 +30,7 @@ class ImageBindEmbeddings(EmbeddingFunction):
    device: str = "cpu"
    normalize: bool = False

-    if PYDANTIC_VERSION.major < 2:  # Pydantic 1.x compat
-
-        class Config:
-            keep_untouched = (cached_property,)
-    else:
-        model_config = dict()
-        model_config["ignored_types"] = (cached_property,)
+    model_config = ConfigDict(ignored_types=(cached_property,))

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
@@ -7,8 +7,7 @@ from typing import List, Any

 import numpy as np

-from pydantic import PrivateAttr
-from lancedb.pydantic import PYDANTIC_VERSION
+from pydantic import ConfigDict, PrivateAttr

 from ..util import attempt_import_or_raise
 from .base import EmbeddingFunction
@@ -59,13 +58,7 @@ class TransformersEmbeddingFunction(EmbeddingFunction):
        )
        self._model.to(self.device)

-    if PYDANTIC_VERSION.major < 2:  # Pydantic 1.x compat
-
-        class Config:
-            keep_untouched = (cached_property,)
-    else:
-        model_config = dict()
-        model_config["ignored_types"] = (cached_property,)
+    model_config = ConfigDict(ignored_types=(cached_property,))

    def ndims(self):
        self._ndims = self._model.config.hidden_size
@@ -85,8 +85,9 @@ class Expr:
    # for dict keys / set membership.
    __hash__ = None  # type: ignore[assignment]

-    def __init__(self, inner: PyExpr) -> None:
+    def __init__(self, inner: PyExpr, *, column_path: str | None = None) -> None:
        self._inner = inner
+        self._column_path = column_path

    # ── comparisons ──────────────────────────────────────────────────────────

@@ -273,7 +274,7 @@ def col(name: str) -> Expr:
    >>> col("age") > lit(18)
    Expr((age > 18))
    """
-    return Expr(expr_col(name))
+    return Expr(expr_col(name), column_path=name)


 def lit(value: Union[bool, int, float, str, bytes, date, datetime, Decimal]) -> Expr:
@@ -0,0 +1,977 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+"""Canonical values exchanged with LanceDB Enterprise Function services.
+
+These immutable models contain client/wire state only. Catalog persistence,
+environment bake, secret resolution, and execution are owned by Sophon.
+"""
+
+from __future__ import annotations
+
+import ast
+import base64
+import functools
+import hashlib
+import inspect
+import json
+import math
+import re
+import sys
+import textwrap
+import types
+import uuid
+from collections.abc import Mapping
+from datetime import date, datetime
+from typing import (
+    Annotated,
+    Any,
+    Callable,
+    Optional,
+    Union,
+    get_args,
+    get_origin,
+    get_type_hints,
+    overload,
+)
+
+import pyarrow as pa
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    conint,
+    field_validator,
+    model_validator,
+)
+
+_Int32 = conint(strict=True, ge=-(2**31), le=2**31 - 1)
+_UInt32 = conint(strict=True, ge=0, le=2**32 - 1)
+_UInt64 = conint(strict=True, ge=0, le=2**64 - 1)
+
+
+class _FrozenDict(dict):
+    def _immutable(self, *args, **kwargs):
+        raise TypeError("remote canonical values are immutable")
+
+    __setitem__ = _immutable
+    __delitem__ = _immutable
+    clear = _immutable
+    pop = _immutable
+    popitem = _immutable
+    setdefault = _immutable
+    update = _immutable
+
+    def __ior__(self, other):
+        self._immutable()
+
+
+def _freeze_value(value):
+    if isinstance(value, Mapping):
+        return _FrozenDict({key: _freeze_value(child) for key, child in value.items()})
+    if isinstance(value, (list, tuple)):
+        return tuple(_freeze_value(child) for child in value)
+    return value
+
+
+def _validate_literal(value):
+    if isinstance(value, float):
+        raise ValueError(
+            "floating-point Function literals are not part of the Slice 1 "
+            "canonical wire contract"
+        )
+    if isinstance(value, int) and not isinstance(value, bool):
+        if not -(2**63) <= value <= 2**64 - 1:
+            raise ValueError(
+                "Function integer literal is outside the canonical JSON range"
+            )
+    elif isinstance(value, Mapping):
+        for child in value.values():
+            _validate_literal(child)
+    elif isinstance(value, (list, tuple)):
+        for child in value:
+            _validate_literal(child)
+    return value
+
+
+def _known_wire_value(value):
+    if isinstance(value, _RemoteValue):
+        return value._known_dict()
+    if isinstance(value, Mapping):
+        return {key: _known_wire_value(child) for key, child in value.items()}
+    if isinstance(value, (list, tuple)):
+        return [_known_wire_value(child) for child in value]
+    return value
+
+
+class _RemoteValue(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    @model_validator(mode="after")
+    def _freeze_mappings(self):
+        for name, value in self.__dict__.items():
+            object.__setattr__(self, name, _freeze_value(value))
+        return self
+
+    @classmethod
+    def from_json(cls, payload: str):
+        return cls.model_validate_json(payload)
+
+    def _known_dict(self) -> dict[str, Any]:
+        known = {}
+        for name, field in self.__class__.model_fields.items():
+            value = getattr(self, name)
+            if value is None:
+                continue
+            if not field.is_required():
+                default_factory = field.default_factory
+                if default_factory is not None and value == default_factory():
+                    continue
+                if default_factory is None and value == field.default:
+                    continue
+            known[name] = _known_wire_value(value)
+        return known
+
+    def _copy(self, *, update: Mapping[str, Any]):
+        update = {name: _freeze_value(value) for name, value in update.items()}
+        return self.model_copy(update=update)
+
+    def to_canonical_json(self) -> str:
+        return json.dumps(
+            self._known_dict(),
+            ensure_ascii=False,
+            allow_nan=False,
+            sort_keys=True,
+            separators=(",", ":"),
+        )
+
+
+class _OpenRemoteValue(_RemoteValue):
+    """Forward-readable value whose extras stay out of canonical encoding."""
+
+    model_config = ConfigDict(extra="allow", frozen=True)
+
+    def _unknown_field_names(self) -> set[str]:
+        return set((self.__pydantic_extra__ or {}).keys())
+
+
+class FunctionArtifact(_RemoteValue):
+    """Content-addressed Python artifact identity."""
+
+    kind: str
+    digest: str
+    entrypoint: str
+
+
+class FunctionArtifactContent(_RemoteValue):
+    """Encoded artifact bytes uploaded during remote registration."""
+
+    encoding: str
+    data: str
+
+
+class PythonAdapterSpec(_RemoteValue):
+    """Internal scalar-callable to Arrow-batch adapter selection."""
+
+    kind: str
+    version: _UInt32
+
+
+class FunctionArtifactRequest(_RemoteValue):
+    """Source artifact uploaded while registering a Function."""
+
+    kind: str
+    digest: str
+    entrypoint: str
+    content: FunctionArtifactContent
+    adapter: PythonAdapterSpec
+
+
+class FunctionParameter(_RemoteValue):
+    name: str
+    arrow_type: str
+    nullable: bool
+
+
+class FunctionResultField(_OpenRemoteValue):
+    name: str
+    arrow_type: str
+    nullable: bool
+
+
+class FunctionOutput(_OpenRemoteValue):
+    """Scalar or ordered named-struct output; unknown kinds remain decodable."""
+
+    kind: str
+    arrow_type: Optional[str] = None
+    nullable: Optional[bool] = None
+    fields: tuple[FunctionResultField, ...] = ()
+
+
+class FunctionSignature(_RemoteValue):
+    inputs: tuple[FunctionParameter, ...]
+    output: FunctionOutput
+
+
+class PythonEnvironmentSpec(_RemoteValue):
+    """One Sophon-managed Python environment source."""
+
+    kind: str
+    packages: tuple[str, ...] = ()
+    path: Optional[str] = None
+    modules: tuple[str, ...] = ()
+    image: Optional[str] = None
+
+
+class PythonRuntimeSpec(_RemoteValue):
+    """Remote runtime definition with non-secret environment values.
+
+    V1 supports ``kind="python"``. Newer runtime kinds remain readable, while
+    their unknown payload fields are intentionally not retained by the client.
+    """
+
+    kind: str
+    python_version: Optional[str] = None
+    environment: Optional[PythonEnvironmentSpec] = None
+    env: Optional[Mapping[str, str]] = None
+
+    @model_validator(mode="after")
+    def _validate_runtime_kind(self):
+        if self.kind == "python":
+            if self.python_version is None:
+                raise ValueError("python runtime requires python_version")
+            if self.environment is None:
+                raise ValueError("python runtime requires environment")
+        else:
+            object.__setattr__(self, "python_version", None)
+            object.__setattr__(self, "environment", None)
+            object.__setattr__(self, "env", None)
+        return self
+
+
+class FunctionVersion(_RemoteValue):
+    """An exact immutable Function version returned by Enterprise.
+
+    Scheduling resources, priority, concurrency, and retry policy belong to
+    the submitting Job and are not part of this identity.
+    """
+
+    name: str
+    version: str
+    artifact: FunctionArtifact
+    signature: FunctionSignature
+    runtime: PythonRuntimeSpec
+    runtime_digest: str
+    environment_digest: str
+    required_secrets: tuple[str, ...] = ()
+    created_at: str
+
+    def __call__(self, **inputs: Any) -> FunctionApplication:
+        """Bind this exact version to named table columns.
+
+        Every input must be a direct [lancedb.col][lancedb.expr.col]
+        reference. The returned application is immutable and retains a
+        named-struct output as one sibling group, so every row's sibling values
+        come from one logical Function evaluation. Map result fields to table
+        columns with
+        [FunctionApplication.rename][lancedb.functions.FunctionApplication.rename],
+        then pass the application to
+        [Table.add_columns][lancedb.table.Table.add_columns].
+
+        Examples
+        --------
+        >>> from lancedb import col
+        >>> application = function(  # doctest: +SKIP
+        ...     title=col("title"),
+        ...     body=col("body"),
+        ... ).rename(columns={
+        ...     "normalized_text": "search_text",
+        ...     "token_count": "search_token_count",
+        ... })
+        >>> table.add_columns(application)  # doctest: +SKIP
+        """
+        from lancedb.expr import Expr
+
+        parameters = tuple(parameter.name for parameter in self.signature.inputs)
+        missing = [parameter for parameter in parameters if parameter not in inputs]
+        unknown = sorted(set(inputs) - set(parameters))
+        if missing or unknown:
+            details = []
+            if missing:
+                details.append(f"missing inputs: {missing!r}")
+            if unknown:
+                details.append(f"unknown inputs: {unknown!r}")
+            raise TypeError("invalid Function inputs (" + "; ".join(details) + ")")
+
+        bindings = []
+        for parameter in parameters:
+            value = inputs[parameter]
+            if not isinstance(value, Expr) or value._column_path is None:
+                raise TypeError(
+                    f"Function input {parameter!r} must be a direct col(...) reference"
+                )
+            bindings.append(
+                ApplicationInput(
+                    parameter=parameter,
+                    kind="column",
+                    value={"path": value._column_path},
+                )
+            )
+        return FunctionApplication(
+            function=FunctionVersionRef(name=self.name, version=self.version),
+            inputs=tuple(bindings),
+            output=self.signature.output,
+            group_id=f"fg_{uuid.uuid4().hex}",
+        )
+
+
+class FunctionRegistrationRequest(_RemoteValue):
+    """Stable remote registration envelope produced by :func:`udf`.
+
+    Only secret names are represented. Secret values are resolved inside the
+    remote service and have no client request field.
+    """
+
+    name: str
+    artifact: FunctionArtifactRequest
+    signature: FunctionSignature
+    runtime: PythonRuntimeSpec
+    required_secrets: tuple[str, ...] = ()
+
+
+class FunctionVersionRef(_OpenRemoteValue):
+    name: str
+    version: str
+
+
+class ApplicationInput(_OpenRemoteValue):
+    """One parameter value.
+
+    Slice 1 freezes integers, strings, booleans, nulls, arrays, and objects.
+    Floating-point literal encoding is deferred until Python authoring is
+    introduced with a language-neutral numeric representation.
+    """
+
+    parameter: str
+    kind: str
+    value: Any
+
+    @field_validator("value")
+    @classmethod
+    def _validate_value(cls, value):
+        return _validate_literal(value)
+
+
+class FunctionApplication(_OpenRemoteValue):
+    """Immutable pre-declaration application of an exact Function version.
+
+    A named-struct output remains one grouped application through table
+    declaration and execution.
+    [FunctionApplication.rename][lancedb.functions.FunctionApplication.rename]
+    records the result-field to table-column mapping without splitting sibling
+    outputs into separate UDF calls.
+    """
+
+    function: FunctionVersionRef
+    inputs: tuple[ApplicationInput, ...]
+    output: FunctionOutput
+    group_id: str
+    columns: Mapping[str, str] = Field(default_factory=dict)
+
+    def _known_dict(self) -> dict[str, Any]:
+        value = super()._known_dict()
+        for name in self._unknown_field_names():
+            value.pop(name, None)
+        return value
+
+    def _ensure_declarable(self) -> None:
+        unknown = {f"application.{name}" for name in self._unknown_field_names()}
+        unknown.update(
+            f"function.{name}" for name in self.function._unknown_field_names()
+        )
+        for index, input_value in enumerate(self.inputs):
+            unknown.update(
+                f"inputs[{index}].{name}" for name in input_value._unknown_field_names()
+            )
+        unknown.update(f"output.{name}" for name in self.output._unknown_field_names())
+        for index, field in enumerate(self.output.fields):
+            unknown.update(
+                f"output.fields[{index}].{name}"
+                for name in field._unknown_field_names()
+            )
+        if unknown:
+            raise ValueError(
+                "Function application contains fields from a newer contract: "
+                f"{sorted(unknown)!r}"
+            )
+
+    def rename(self, *, columns: Mapping[str, str]) -> FunctionApplication:
+        """Return a copy with result-field to table-column aliases."""
+        if self.output.kind != "named_struct":
+            raise ValueError("rename(columns=...) requires a named-struct application")
+        result_fields = {field.name for field in self.output.fields}
+        unknown = set(columns) - result_fields
+        if unknown:
+            raise ValueError(f"unknown Function result fields: {sorted(unknown)!r}")
+        merged = dict(self.columns)
+        merged.update(columns)
+        destinations = tuple(
+            merged.get(field.name, field.name) for field in self.output.fields
+        )
+        if len(set(destinations)) != len(destinations):
+            raise ValueError("FunctionApplication rename destinations must be unique")
+        return self._copy(update={"columns": merged})
+
+
+class InputBinding(_RemoteValue):
+    parameter: str
+    field_id: _Int32
+    field_path: str
+    arrow_type: str
+    nullable: bool
+
+
+class OutputMapping(_RemoteValue):
+    """One stable result-field mapping.
+
+    Assignment state is outside the Slice 1 client contract. During the NULL
+    transition Lance exposes no public cell-flag identifier to persist here.
+    """
+
+    result_field: str
+    output_name: str
+    output_field_id: _Int32
+    output_ordinal: _UInt32
+    arrow_type: str
+    nullable: bool
+
+
+class FunctionBinding(_RemoteValue):
+    """Immutable grouped binding persisted by the Enterprise table service."""
+
+    binding_id: str
+    revision: _UInt64
+    function: FunctionVersionRef
+    group_id: str
+    inputs: tuple[InputBinding, ...]
+    outputs: tuple[OutputMapping, ...]
+    input_schema: Optional[Mapping[str, Any]] = None
+    output_schema: Optional[Mapping[str, Any]] = None
+
+
+class RefreshColumnResult(_RemoteValue):
+    """Terminal result of a remote Function-column refresh Job."""
+
+    rows_assigned: _UInt64
+    rows_failed: _UInt64
+    rows_remaining: _UInt64
+    source_version: _UInt64
+    published_version: Optional[_UInt64] = None
+
+    @property
+    def rows_filled(self) -> int:
+        """Deprecated compatibility alias for :attr:`rows_assigned`."""
+        return self.rows_assigned
+
+    @property
+    def version(self) -> Optional[int]:
+        """Deprecated compatibility alias for :attr:`published_version`."""
+        return self.published_version
+
+
+_FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$")
+_SECRET_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+
+def _canonical_arrow_type(data_type: pa.DataType) -> str:
+    primitive_types = (
+        (pa.bool_(), "bool"),
+        (pa.int8(), "int8"),
+        (pa.int16(), "int16"),
+        (pa.int32(), "int32"),
+        (pa.int64(), "int64"),
+        (pa.uint8(), "uint8"),
+        (pa.uint16(), "uint16"),
+        (pa.uint32(), "uint32"),
+        (pa.uint64(), "uint64"),
+        (pa.float16(), "float16"),
+        (pa.float32(), "float32"),
+        (pa.float64(), "float64"),
+        (pa.string(), "utf8"),
+        (pa.large_utf8(), "large_utf8"),
+        (pa.binary(), "binary"),
+        (pa.large_binary(), "large_binary"),
+        (pa.date32(), "date32"),
+        (pa.date64(), "date64"),
+    )
+    for candidate, name in primitive_types:
+        if data_type == candidate:
+            return name
+    if pa.types.is_fixed_size_binary(data_type):
+        return f"fixed_size_binary[{data_type.byte_width}]"
+    if pa.types.is_list(data_type):
+        return f"list<{_canonical_arrow_type(data_type.value_type)}>"
+    if pa.types.is_large_list(data_type):
+        return f"large_list<{_canonical_arrow_type(data_type.value_type)}>"
+    if pa.types.is_fixed_size_list(data_type):
+        return (
+            f"fixed_size_list<{_canonical_arrow_type(data_type.value_type)}>"
+            f"[{data_type.list_size}]"
+        )
+    if pa.types.is_struct(data_type):
+        fields = ",".join(
+            f"{field.name}:{_canonical_arrow_type(field.type)}" for field in data_type
+        )
+        return f"struct<{fields}>"
+    if pa.types.is_timestamp(data_type):
+        timezone = f",tz={data_type.tz}" if data_type.tz is not None else ""
+        return f"timestamp[{data_type.unit}{timezone}]"
+    if pa.types.is_time32(data_type) or pa.types.is_time64(data_type):
+        return f"time[{data_type.unit}]"
+    if pa.types.is_duration(data_type):
+        return f"duration[{data_type.unit}]"
+    if pa.types.is_decimal(data_type):
+        bit_width = data_type.bit_width
+        return f"decimal{bit_width}({data_type.precision},{data_type.scale})"
+    raise TypeError(f"unsupported Arrow type for Function signature: {data_type}")
+
+
+def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]:
+    nullable = False
+    origin = get_origin(annotation)
+    if origin in (Union, types.UnionType):
+        arguments = get_args(annotation)
+        non_none = tuple(
+            argument for argument in arguments if argument is not type(None)
+        )
+        if len(non_none) != 1 or len(non_none) == len(arguments):
+            raise TypeError(f"unsupported union annotation: {annotation!r}")
+        annotation = non_none[0]
+        nullable = True
+
+    origin = get_origin(annotation)
+    if origin is Annotated:
+        base, *metadata = get_args(annotation)
+        arrow_types = [value for value in metadata if isinstance(value, pa.DataType)]
+        if len(arrow_types) != 1:
+            raise TypeError(
+                "Annotated Function types require exactly one PyArrow DataType"
+            )
+        _, base_nullable = _annotation_type(base)
+        return arrow_types[0], nullable or base_nullable
+
+    if isinstance(annotation, pa.DataType):
+        return annotation, nullable
+    if annotation is bool:
+        return pa.bool_(), nullable
+    if annotation is int:
+        return pa.int64(), nullable
+    if annotation is float:
+        return pa.float64(), nullable
+    if annotation is str:
+        return pa.string(), nullable
+    if annotation is bytes:
+        return pa.binary(), nullable
+    if annotation is date:
+        return pa.date32(), nullable
+    if annotation is datetime:
+        return pa.timestamp("us"), nullable
+    if get_origin(annotation) is list:
+        arguments = get_args(annotation)
+        if len(arguments) != 1:
+            raise TypeError(f"unsupported list annotation: {annotation!r}")
+        value_type, value_nullable = _annotation_type(arguments[0])
+        if value_nullable:
+            raise TypeError("nullable Function list elements are not supported")
+        return pa.list_(value_type), nullable
+    raise TypeError(f"unsupported Function annotation: {annotation!r}")
+
+
+def _callable_parameters(function: Callable[..., Any]) -> tuple[inspect.Parameter, ...]:
+    parameters = tuple(inspect.signature(function).parameters.values())
+    for parameter in parameters:
+        if parameter.kind in (
+            inspect.Parameter.POSITIONAL_ONLY,
+            inspect.Parameter.VAR_POSITIONAL,
+            inspect.Parameter.VAR_KEYWORD,
+        ):
+            raise TypeError("Function callables require named, non-variadic parameters")
+        if parameter.default is not inspect.Parameter.empty:
+            raise TypeError("Function callable defaults are not supported")
+    return parameters
+
+
+def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutput:
+    if isinstance(output, pa.Schema):
+        fields = tuple(output)
+    elif isinstance(output, pa.Field) and pa.types.is_struct(output.type):
+        if output.nullable:
+            raise ValueError("Function output must be non-nullable")
+        fields = tuple(output.type)
+    elif isinstance(output, pa.DataType) and pa.types.is_struct(output):
+        fields = tuple(output)
+    else:
+        field = (
+            output
+            if isinstance(output, pa.Field)
+            else pa.field("result", output, nullable=False)
+        )
+        if not isinstance(field, pa.Field):
+            raise TypeError(
+                "output_schema must be a PyArrow DataType, Field, or Schema"
+            )
+        if field.nullable:
+            raise ValueError("Function output must be non-nullable")
+        return FunctionOutput(
+            kind="scalar",
+            arrow_type=_canonical_arrow_type(field.type),
+            nullable=False,
+        )
+
+    if not fields:
+        raise ValueError("named-struct Function output must contain at least one field")
+    if any(field.nullable for field in fields):
+        raise ValueError("Function output fields must be non-nullable")
+    names = [field.name for field in fields]
+    if len(set(names)) != len(names):
+        raise ValueError("Function output field names must be unique")
+    return FunctionOutput(
+        kind="named_struct",
+        fields=tuple(
+            FunctionResultField(
+                name=field.name,
+                arrow_type=_canonical_arrow_type(field.type),
+                nullable=False,
+            )
+            for field in fields
+        ),
+    )
+
+
+def _infer_signature(
+    function: Callable[..., Any],
+    input_schema: Optional[pa.Schema],
+    output_schema: Optional[pa.DataType | pa.Field | pa.Schema],
+) -> FunctionSignature:
+    parameters = _callable_parameters(function)
+    if (input_schema is None) != (output_schema is None):
+        raise ValueError("input_schema and output_schema must be provided together")
+
+    if input_schema is not None:
+        if not isinstance(input_schema, pa.Schema):
+            raise TypeError("input_schema must be a PyArrow Schema")
+        expected = tuple(parameter.name for parameter in parameters)
+        actual = tuple(input_schema.names)
+        if actual != expected:
+            raise ValueError(
+                "input_schema fields must exactly match callable parameters in order: "
+                f"expected {expected!r}, got {actual!r}"
+            )
+        inputs = tuple(
+            FunctionParameter(
+                name=field.name,
+                arrow_type=_canonical_arrow_type(field.type),
+                nullable=field.nullable,
+            )
+            for field in input_schema
+        )
+        return FunctionSignature(inputs=inputs, output=_function_output(output_schema))
+
+    try:
+        annotations = get_type_hints(function, include_extras=True)
+    except Exception as error:
+        raise TypeError(f"failed to resolve Function annotations: {error}") from error
+    missing = [
+        parameter.name for parameter in parameters if parameter.name not in annotations
+    ]
+    if missing or "return" not in annotations:
+        names = missing + ([] if "return" in annotations else ["return"])
+        raise TypeError(f"missing Function annotations: {names!r}")
+    inputs = []
+    for parameter in parameters:
+        data_type, nullable = _annotation_type(annotations[parameter.name])
+        inputs.append(
+            FunctionParameter(
+                name=parameter.name,
+                arrow_type=_canonical_arrow_type(data_type),
+                nullable=nullable,
+            )
+        )
+    output_type, output_nullable = _annotation_type(annotations["return"])
+    if output_nullable:
+        raise ValueError("Function output must be non-nullable")
+    return FunctionSignature(
+        inputs=tuple(inputs),
+        output=_function_output(pa.field("result", output_type, nullable=False)),
+    )
+
+
+def _is_udf_decorator(node: ast.expr) -> bool:
+    if isinstance(node, ast.Call):
+        node = node.func
+    return (isinstance(node, ast.Name) and node.id == "udf") or (
+        isinstance(node, ast.Attribute) and node.attr == "udf"
+    )
+
+
+def _literal_source(value: Any) -> str:
+    if value is None or type(value) in (bool, int, str, bytes):
+        return repr(value)
+    if type(value) is float and math.isfinite(value):
+        return repr(value)
+    if type(value) is tuple:
+        children = ", ".join(_literal_source(child) for child in value)
+        if len(value) == 1:
+            children += ","
+        return f"({children})"
+    raise TypeError(
+        "Function source references an unsupported global value of type "
+        f"{type(value).__name__}"
+    )
+
+
+def _package_source(function: Callable[..., Any]) -> bytes:
+    if not inspect.isfunction(function) or inspect.iscoroutinefunction(function):
+        raise TypeError("@udf requires a synchronous Python function")
+    try:
+        source = textwrap.dedent(inspect.getsource(function))
+    except (OSError, TypeError) as error:
+        raise ValueError("@udf requires inspectable Python source") from error
+    module = ast.parse(source)
+    definitions = [
+        node
+        for node in module.body
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
+        and node.name == function.__name__
+    ]
+    if len(definitions) != 1 or not isinstance(definitions[0], ast.FunctionDef):
+        raise ValueError("@udf source must contain exactly one synchronous function")
+    definition = definitions[0]
+    if any(not _is_udf_decorator(decorator) for decorator in definition.decorator_list):
+        raise ValueError("@udf cannot package additional Python decorators")
+    definition.decorator_list = []
+
+    closure = inspect.getclosurevars(function)
+    if closure.nonlocals:
+        raise ValueError("@udf cannot package functions that capture closure values")
+    if closure.unbound:
+        raise ValueError(
+            f"@udf source contains unresolved global names: {sorted(closure.unbound)!r}"
+        )
+    globals_source = []
+    for name, value in sorted(closure.globals.items()):
+        if isinstance(value, types.ModuleType):
+            globals_source.append(f"import {value.__name__} as {name}")
+        else:
+            globals_source.append(f"{name} = {_literal_source(value)}")
+
+    function_source = ast.unparse(definition)
+    parts = ["from __future__ import annotations"]
+    if globals_source:
+        parts.extend(["", *globals_source])
+    parts.extend(["", function_source, ""])
+    return "\n".join(parts).encode("utf-8")
+
+
+class UdfDefinition:
+    """A scalar Python callable prepared for remote Function registration.
+
+    Instances are created with :func:`udf`. Calling an instance executes the
+    original scalar Python function, which keeps local unit testing ordinary.
+    Remote execution adapts that scalar callable to the internal Arrow batch
+    ABI described by the registration artifact.
+    """
+
+    def __init__(
+        self,
+        function: Callable[..., Any],
+        *,
+        name: Optional[str],
+        input_schema: Optional[pa.Schema],
+        output_schema: Optional[pa.DataType | pa.Field | pa.Schema],
+        pip: tuple[str, ...],
+        env: Mapping[str, str],
+        secrets: tuple[str, ...],
+        python_version: Optional[str],
+    ):
+        function_name = name or function.__name__
+        if not _FUNCTION_NAME.fullmatch(function_name):
+            raise ValueError(f"invalid Function name: {function_name!r}")
+        packages = tuple(sorted(set(pip)))
+        if any(not package or package != package.strip() for package in packages):
+            raise ValueError("pip requirements must be non-empty and trimmed")
+        environment = dict(env)
+        if any(
+            not isinstance(key, str) or not isinstance(value, str)
+            for key, value in environment.items()
+        ):
+            raise TypeError("Function env keys and values must be strings")
+        required_secrets = tuple(sorted(set(secrets)))
+        invalid_secrets = [
+            secret for secret in required_secrets if not _SECRET_NAME.fullmatch(secret)
+        ]
+        if invalid_secrets:
+            raise ValueError(f"invalid Function secret names: {invalid_secrets!r}")
+        overlap = set(environment) & set(required_secrets)
+        if overlap:
+            raise ValueError(
+                f"Function env and secret names must be disjoint: {sorted(overlap)!r}"
+            )
+
+        signature = _infer_signature(function, input_schema, output_schema)
+        source = _package_source(function)
+        digest = f"sha256:{hashlib.sha256(source).hexdigest()}"
+        runtime = PythonRuntimeSpec(
+            kind="python",
+            python_version=python_version
+            or f"{sys.version_info.major}.{sys.version_info.minor}",
+            environment=PythonEnvironmentSpec(kind="pip", packages=packages),
+            env=environment,
+        )
+        self._function = function
+        self._request = FunctionRegistrationRequest(
+            name=function_name,
+            artifact=FunctionArtifactRequest(
+                kind="python_callable",
+                digest=digest,
+                entrypoint=function.__name__,
+                content=FunctionArtifactContent(
+                    encoding="base64",
+                    data=base64.b64encode(source).decode("ascii"),
+                ),
+                adapter=PythonAdapterSpec(
+                    kind="scalar_to_arrow_batch",
+                    version=1,
+                ),
+            ),
+            signature=signature,
+            runtime=runtime,
+            required_secrets=required_secrets,
+        )
+        functools.update_wrapper(self, function)
+
+    @property
+    def registration_request(self) -> FunctionRegistrationRequest:
+        """The immutable request sent by ``create_function_async``."""
+        return self._request
+
+    def __call__(self, *args, **kwargs):
+        return self._function(*args, **kwargs)
+
+
+@overload
+def udf(function: Callable[..., Any]) -> UdfDefinition: ...
+
+
+@overload
+def udf(
+    function: None = None,
+    *,
+    name: Optional[str] = None,
+    input_schema: Optional[pa.Schema] = None,
+    output_schema: Optional[pa.DataType | pa.Field | pa.Schema] = None,
+    pip: tuple[str, ...] | list[str] = (),
+    env: Optional[Mapping[str, str]] = None,
+    secrets: tuple[str, ...] | list[str] = (),
+    python_version: Optional[str] = None,
+) -> Callable[[Callable[..., Any]], UdfDefinition]: ...
+
+
+def udf(
+    function: Optional[Callable[..., Any]] = None,
+    *,
+    name: Optional[str] = None,
+    input_schema: Optional[pa.Schema] = None,
+    output_schema: Optional[pa.DataType | pa.Field | pa.Schema] = None,
+    pip: tuple[str, ...] | list[str] = (),
+    env: Optional[Mapping[str, str]] = None,
+    secrets: tuple[str, ...] | list[str] = (),
+    python_version: Optional[str] = None,
+):
+    """Prepare a scalar Python callable for remote Function registration.
+
+    Input and output signatures are inferred from supported annotations. For
+    Arrow types annotations cannot express precisely, pass ``input_schema``
+    and ``output_schema`` together. Nullable outputs are rejected because V1
+    uses physical NULL to represent unassigned computed-column rows.
+
+    Parameters
+    ----------
+    function : Callable, optional
+        The synchronous scalar callable to package.
+    name : str, optional
+        The remote Function name. Defaults to the callable name.
+    input_schema : pyarrow.Schema, optional
+        Explicit input fields in the exact order of the callable parameters.
+        Must be provided together with ``output_schema``.
+    output_schema : pyarrow.DataType, pyarrow.Field, or pyarrow.Schema, optional
+        Explicit scalar or named-struct output. Must be non-nullable and be
+        provided together with ``input_schema``.
+    pip : sequence of str, optional
+        Pip requirements for the remote environment.
+    env : mapping of str to str, optional
+        Non-secret environment variables. Use ``secrets`` for credentials.
+    secrets : sequence of str, optional
+        Names of secrets resolved by the remote service. Secret values are not
+        accepted by this API or included in the registration request.
+    python_version : str, optional
+        Remote Python major/minor version. Defaults to the client version.
+
+    Returns
+    -------
+    UdfDefinition
+        A callable definition accepted by
+        :meth:`lancedb.db.DBConnection.create_function`,
+        :meth:`lancedb.db.AsyncConnection.create_function_async` and
+        :meth:`lancedb.db.DBConnection.create_function_async`.
+
+    Examples
+    --------
+    >>> from lancedb import udf
+    >>> @udf(pip=["numpy==2.2.0"], secrets=["MODEL_TOKEN"])
+    ... def score(value: float) -> float:
+    ...     return value * 2
+    >>> score(1.5)
+    3.0
+    """
+
+    def decorate(target: Callable[..., Any]) -> UdfDefinition:
+        return UdfDefinition(
+            target,
+            name=name,
+            input_schema=input_schema,
+            output_schema=output_schema,
+            pip=tuple(pip),
+            env={} if env is None else env,
+            secrets=tuple(secrets),
+            python_version=python_version,
+        )
+
+    if function is None:
+        return decorate
+    return decorate(function)
+
+
+__all__ = [
+    "ApplicationInput",
+    "FunctionApplication",
+    "FunctionArtifact",
+    "FunctionArtifactContent",
+    "FunctionArtifactRequest",
+    "FunctionBinding",
+    "FunctionOutput",
+    "FunctionParameter",
+    "FunctionRegistrationRequest",
+    "FunctionResultField",
+    "FunctionSignature",
+    "FunctionVersion",
+    "FunctionVersionRef",
+    "InputBinding",
+    "OutputMapping",
+    "PythonEnvironmentSpec",
+    "PythonAdapterSpec",
+    "PythonRuntimeSpec",
+    "RefreshColumnResult",
+    "UdfDefinition",
+    "udf",
+]
@@ -163,6 +163,15 @@ class FTS:
        The number of documents per compressed posting block. Supported values
        are 128 and 256. A value of 256 uses the experimental FTS V3 format
        and may introduce breaking changes.
+    memory_limit : int, optional
+        The total memory limit in MiB for the local FTS build stage. The limit
+        is divided evenly among indexing workers. This build-only setting is
+        not persisted with the index and does not apply to remote tables.
+    num_workers : int, optional
+        The number of workers for a local FTS build. By default Lance uses
+        roughly half of the available CPU cores. The effective value is
+        limited by the available compute capacity. This build-only setting is
+        not persisted with the index and does not apply to remote tables.

    Notes
    -----
@@ -185,6 +194,8 @@ class FTS:
    prefix_only: bool = False
    block_size: int = 128
    custom_stop_words: Optional[List[str]] = None
+    memory_limit: Optional[int] = None
+    num_workers: Optional[int] = None


@dataclass
@@ -5,20 +5,23 @@

 import asyncio
 from datetime import timedelta
-from typing import Optional
+from typing import Any, Generic, Optional, TypeVar, cast

 from lancedb.background_loop import LOOP

 from . import _lancedb
+from .functions import FunctionVersion
+
+T = TypeVar("T")


-class AsyncJob:
+class AsyncJob(Generic[T]):
    """A handle to an operation that may still be running.

    The operation may already be complete when the handle is created.
    """

-    def __init__(self, inner: Optional["_lancedb.Job"]):
+    def __init__(self, inner: Optional[Any]):
        self._inner = inner

    @property
@@ -44,18 +47,20 @@ class AsyncJob:
            return "finished"
        return await self._inner.status()

-    async def wait(self, timeout: Optional[timedelta] = None):
+    async def wait(self, timeout: Optional[timedelta] = None) -> T:
        """Wait until the operation reaches a terminal state.

        Raises `JobFailedError` if the operation failed, `JobCancelledError`
        if it was cancelled, and `TimeoutError` if `timeout` elapses first.
        """
        if self._inner is None:
-            return
+            return cast(T, None)
        if timeout is None:
-            await self._inner.wait()
-        else:
-            await asyncio.wait_for(self._inner.wait(), timeout.total_seconds())
+            return cast(T, await self._inner.wait())
+        return cast(
+            T,
+            await asyncio.wait_for(self._inner.wait(), timeout.total_seconds()),
+        )

    async def cancel(self):
        """Request cancellation. Cancelling a finished operation is a no-op."""
@@ -64,10 +69,10 @@ class AsyncJob:
        await self._inner.cancel()


-class Job:
+class Job(Generic[T]):
    """Synchronous counterpart of `AsyncJob`."""

-    def __init__(self, inner: Optional[AsyncJob]):
+    def __init__(self, inner: Optional[AsyncJob[T]]):
        self._inner = inner

    @property
@@ -88,18 +93,40 @@ class Job:
            return "finished"
        return LOOP.run(self._inner.status())

-    def wait(self, timeout: Optional[timedelta] = None):
+    def wait(self, timeout: Optional[timedelta] = None) -> T:
        """Block until the operation reaches a terminal state.

        Raises `JobFailedError` if the operation failed, `JobCancelledError`
        if it was cancelled, and `TimeoutError` if `timeout` elapses first.
        """
        if self._inner is None:
-            return
-        LOOP.run(self._inner.wait(timeout))
+            return cast(T, None)
+        return LOOP.run(self._inner.wait(timeout))

    def cancel(self):
        """Request cancellation. Cancelling a finished operation is a no-op."""
        if self._inner is None:
            return
        LOOP.run(self._inner.cancel())
+
+
+class _FunctionJobAdapter:
+    def __init__(self, inner: "_lancedb.FunctionJob"):
+        self._inner = inner
+
+    @property
+    def id(self) -> Optional[str]:
+        return self._inner.id
+
+    async def status(self) -> str:
+        return await self._inner.status()
+
+    async def wait(self) -> FunctionVersion:
+        return FunctionVersion.from_json(await self._inner.wait())
+
+    async def cancel(self):
+        await self._inner.cancel()
+
+
+def _function_job(inner: "_lancedb.FunctionJob") -> AsyncJob[FunctionVersion]:
+    return AsyncJob(_FunctionJobAdapter(inner))
@@ -41,21 +41,15 @@ class PermutationBuilder:
    The permutation is stored in memory and will be lost when the program exits.
    """

-    def __init__(self, table: LanceTable):
+    def __init__(self, table: Table):
        """
        Creates a new permutation builder for the given table.

        By default, the permutation builder will create a single split that contains all
        rows in the same order as the base table.
+
+        Tables with an LSM write spec are rejected: unflushed rows have no row id.
        """
-        if not hasattr(table, "_inner"):
-            raise TypeError(
-                f"PermutationBuilder requires a local LanceTable, "
-                f"got {type(table).__name__}. "
-                "The permutation API is not supported on remote tables. "
-                "Remote tables connect to LanceDB Cloud or Enterprise and do not have "
-                "direct access to the underlying Lance dataset needed for permutations."
-            )
        self._async = async_permutation_builder(table)

    def split_random(
@@ -231,7 +225,7 @@ class PermutationBuilder:
        return LOOP.run(do_execute())


-def permutation_builder(table: LanceTable) -> PermutationBuilder:
+def permutation_builder(table: Table) -> PermutationBuilder:
    return PermutationBuilder(table)


@@ -248,7 +242,7 @@ class Permutations:

    Attributes
    ----------
-    base_table: LanceTable
+    base_table: Table
        The base table that the permutations are based on.
    permutation_table: LanceTable
        The permutation table that defines the splits.
@@ -282,7 +276,7 @@ class Permutations:
    {'train': 0, 'test': 1}
    """

-    def __init__(self, base_table: LanceTable, permutation_table: LanceTable):
+    def __init__(self, base_table: Table, permutation_table: LanceTable):
        self.base_table = base_table
        self.permutation_table = permutation_table

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors

-"""Pydantic (v1 / v2) adapter for LanceDB"""
+"""Pydantic adapter for LanceDB."""

 from __future__ import annotations

@@ -14,9 +14,6 @@ from enum import Enum
 from typing import (
    TYPE_CHECKING,
    Any,
-    Callable,
-    Dict,
-    Generator,
    List,
    Type,
    Union,
@@ -24,17 +21,9 @@ from typing import (
    GenericAlias,
 )

-import numpy as np
 import pyarrow as pa
 import pydantic
-from packaging.version import Version
-
-PYDANTIC_VERSION = Version(pydantic.__version__)
-try:
-    from pydantic_core import CoreSchema, core_schema
-except ImportError:
-    if PYDANTIC_VERSION.major >= 2:
-        raise
+from pydantic_core import CoreSchema, core_schema

 if TYPE_CHECKING:
    from pydantic.fields import FieldInfo
@@ -131,25 +120,6 @@ def Vector(
                ),
            )

-        @classmethod
-        def __get_validators__(cls) -> Generator[Callable, None, None]:
-            yield cls.validate
-
-        # For pydantic v1
-        @classmethod
-        def validate(cls, v):
-            if not isinstance(v, (list, range, np.ndarray)) or len(v) != dim:
-                raise TypeError("A list of numbers or numpy.ndarray is needed")
-            return cls(v)
-
-        if PYDANTIC_VERSION.major < 2:
-
-            @classmethod
-            def __modify_schema__(cls, field_schema: Dict[str, Any]):
-                field_schema["items"] = {"type": "number"}
-                field_schema["maxItems"] = dim
-                field_schema["minItems"] = dim
-
    return FixedSizeList


@@ -157,9 +127,8 @@ def _raise_bare_vector_error(*_args):
    raise TypeError("Vector must be parameterized with a dimension, e.g. Vector(128).")


-# Pydantic v1 and v2 otherwise treat the bare Vector factory as a field validator
-# and inspect its signature, which produces misleading errors about internal types.
-setattr(Vector, "__get_validators__", _raise_bare_vector_error)
+# Pydantic otherwise inspects the bare factory as a field type and produces
+# misleading errors about its internal annotations.
 setattr(Vector, "__get_pydantic_core_schema__", _raise_bare_vector_error)


@@ -233,31 +202,6 @@ def MultiVector(
                ),
            )

-        @classmethod
-        def __get_validators__(cls) -> Generator[Callable, None, None]:
-            yield cls.validate
-
-        # For pydantic v1
-        @classmethod
-        def validate(cls, v):
-            if not isinstance(v, (list, range)):
-                raise TypeError("A list of vectors is needed")
-            for vec in v:
-                if not isinstance(vec, (list, range, np.ndarray)) or len(vec) != dim:
-                    raise TypeError(f"Each vector must be a list of {dim} numbers")
-            return cls(v)
-
-        if PYDANTIC_VERSION.major < 2:
-
-            @classmethod
-            def __modify_schema__(cls, field_schema: Dict[str, Any]):
-                field_schema["items"] = {
-                    "type": "array",
-                    "items": {"type": "number"},
-                    "minItems": dim,
-                    "maxItems": dim,
-                }
-
    return MultiVectorList


@@ -303,20 +247,10 @@ def _py_type_to_arrow_type(py_type: Type[Any], field: FieldInfo) -> pa.DataType:
    )


-if PYDANTIC_VERSION.major < 2:
-
-    def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
-        return [
-            _pydantic_to_field(name, field) for name, field in model.__fields__.items()
-        ]
-
-else:
-
-    def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
-        return [
-            _pydantic_to_field(name, field)
-            for name, field in model.model_fields.items()
-        ]
+def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
+    return [
+        _pydantic_to_field(name, field) for name, field in model.model_fields.items()
+    ]


 def _pydantic_type_to_arrow_type(tp: Any, field: FieldInfo) -> pa.DataType:
@@ -509,8 +443,6 @@ class LanceModel(pydantic.BaseModel):

    @classmethod
    def safe_get_fields(cls):
-        if PYDANTIC_VERSION.major < 2:
-            return cls.__fields__
        return cls.model_fields

    @classmethod
@@ -548,23 +480,9 @@ def get_extras(field_info: FieldInfo, key: str) -> Any:
    """
    Get the extra metadata from a Pydantic FieldInfo.
    """
-    if PYDANTIC_VERSION.major >= 2:
-        return (field_info.json_schema_extra or {}).get(key)
-    return (field_info.field_info.extra or {}).get("json_schema_extra", {}).get(key)
+    return (field_info.json_schema_extra or {}).get(key)


-if PYDANTIC_VERSION.major < 2:
-
-    def model_to_dict(model: pydantic.BaseModel) -> Dict[str, Any]:
-        """
-        Convert a Pydantic model to a dictionary.
-        """
-        return model.dict()
-
-else:
-
-    def model_to_dict(model: pydantic.BaseModel) -> Dict[str, Any]:
-        """
-        Convert a Pydantic model to a dictionary.
-        """
-        return model.model_dump()
+def model_to_dict(model: pydantic.BaseModel) -> dict[str, Any]:
+    """Convert a Pydantic model to a dictionary."""
+    return model.model_dump()
@@ -32,8 +32,6 @@ from typing_extensions import Annotated

 from lancedb._lancedb import fts_query_to_json
 from lancedb.background_loop import LOOP
-from lancedb.pydantic import PYDANTIC_VERSION
-
 from . import __version__
 from .arrow import AsyncRecordBatchReader
 from .dependencies import pandas as pd
@@ -827,12 +825,7 @@ class Query(pydantic.BaseModel):

    # This tells pydantic to allow custom types (needed for the `vector` query since
    # pa.Array wouln't be allowed otherwise)
-    if PYDANTIC_VERSION.major < 2:  # Pydantic 1.x compat
-
-        class Config:
-            arbitrary_types_allowed = True
-    else:
-        model_config = {"arbitrary_types_allowed": True}
+    model_config = pydantic.ConfigDict(arbitrary_types_allowed=True)


 class LanceQueryBuilder(ABC):
@@ -3251,12 +3244,7 @@ class AsyncStandardQuery(AsyncQueryBase):
        if ordering is None:
            self._inner.order_by(None)
        else:
-            self._inner.order_by(
-                [
-                    o.model_dump() if hasattr(o, "model_dump") else o.dict()
-                    for o in ordering
-                ]
-            )
+            self._inner.order_by([o.model_dump() for o in ordering])
        return self

    def fast_search(self) -> Self:
@@ -23,6 +23,7 @@ import pyarrow as pa

 from ..common import DATA
 from ..db import DBConnection, LOOP
+from ..functions import FunctionVersion, UdfDefinition
 from ..job import AsyncJob, Job

 if TYPE_CHECKING:
@@ -713,6 +714,14 @@ class RemoteDBConnection(DBConnection):
        """
        return Job(self._conn.job(job_id))

+    @override
+    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
+        return Job(LOOP.run(self._conn.create_function_async(definition)))
+
+    @override
+    def get_function(self, name: str, *, version: str) -> FunctionVersion:
+        return LOOP.run(self._conn.get_function(name, version=version))
+
    @override
    def list_jobs(self) -> List["JobInfo"]:
        """List server-side jobs across the database's tables."""
@@ -49,6 +49,7 @@ from lancedb.index import (
    LabelList,
 )
 from lancedb.job import Job
+from lancedb.functions import FunctionApplication
 from lancedb.remote.db import LOOP
 from lancedb.table import IndexConfigType, KNOWN_METRICS
 import pyarrow as pa
@@ -960,7 +961,9 @@ class RemoteTable(Table):

    def add_columns(
        self,
-        transforms: Dict[str, str] | None = None,
+        transforms: Dict[str, str | FunctionApplication]
+        | FunctionApplication
+        | None = None,
        *,
        computed: Dict[str, str] | None = None,
    ) -> AddColumnsResult:
@@ -72,6 +72,7 @@ from .index import (
    FTS,
 )
 from .expr import Expr
+from .functions import FunctionApplication
 from .merge import LanceMergeInsertBuilder
 from .pydantic import LanceModel, model_to_dict
 from .query import (
@@ -433,6 +434,20 @@ def _cast_to_target_schema(
    return pa.RecordBatchReader.from_batches(reordered_schema, gen())


+def _field_extension_name(field: pa.Field) -> Optional[str]:
+    extension_name = getattr(field.type, "extension_name", None)
+    if extension_name is not None:
+        return extension_name
+
+    metadata = field.metadata or {}
+    extension_name = metadata.get(b"ARROW:extension:name") or metadata.get(
+        "ARROW:extension:name"
+    )
+    if isinstance(extension_name, bytes):
+        return extension_name.decode()
+    return extension_name
+
+
 def _align_field_types(
    fields: List[pa.Field],
    target_fields: List[pa.Field],
@@ -445,6 +460,16 @@ def _align_field_types(
        target_field = next((f for f in target_fields if f.name == field.name), None)
        if target_field is None:
            raise ValueError(f"Field '{field.name}' not found in target schema")
+        # Preserve arrow.json input until it reaches Lance. LanceDB exposes stored
+        # JSON columns as lance.json (JSONB-backed LargeBinary), but casting the
+        # input to that storage type here merely relabels the raw JSON bytes as
+        # JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
+        if (
+            _field_extension_name(field) == "arrow.json"
+            and _field_extension_name(target_field) == "lance.json"
+        ):
+            new_fields.append(field)
+            continue
        if pa.types.is_struct(target_field.type):
            if pa.types.is_struct(field.type):
                new_type = pa.struct(
@@ -1918,7 +1943,8 @@ class Table(ABC):
    @abstractmethod
    def add_columns(
        self,
-        transforms: Dict[str, str]
+        transforms: Dict[str, str | FunctionApplication]
+        | FunctionApplication
        | pa.Field
        | List[pa.Field]
        | pa.Schema
@@ -1931,13 +1957,21 @@ class Table(ABC):

        Parameters
        ----------
-        transforms: Dict[str, str], pa.Field, List[pa.Field], pa.Schema
+        transforms: Dict[str, str | FunctionApplication], FunctionApplication,
+            pa.Field, List[pa.Field], pa.Schema
            A map of column name to a SQL expression to use to calculate the
            value of the new column. These expressions will be evaluated for
            each row in the table, and can reference existing columns.
            Alternatively, a pyarrow Field or Schema can be provided to add
            new columns with the specified data types. The new columns will
            be initialized with null values.
+
+            A mapping with one ``FunctionApplication`` value keeps its scalar
+            or named-struct result in the named table column. A bare
+            named-struct application expands its ordered result fields as one
+            atomic sibling group; aliases come from ``rename(columns=...)``.
+            Function columns are supported only on LanceDB Cloud and
+            Enterprise.
        computed: Dict[str, str], optional
            A map of column name to a SQL expression defining the column. The
            column's type and inputs are derived from the expression, so no
@@ -4032,9 +4066,10 @@ class LanceTable(Table):

    def add_columns(
        self,
-        transforms: Dict[str, str]
-        | pa.field
-        | List[pa.field]
+        transforms: Dict[str, str | FunctionApplication]
+        | FunctionApplication
+        | pa.Field
+        | List[pa.Field]
        | pa.Schema
        | None = None,
        *,
@@ -5968,9 +6003,10 @@ class AsyncTable:

    async def add_columns(
        self,
-        transforms: dict[str, str]
-        | pa.field
-        | List[pa.field]
+        transforms: dict[str, str | FunctionApplication]
+        | FunctionApplication
+        | pa.Field
+        | List[pa.Field]
        | pa.Schema
        | None = None,
        *,
@@ -5981,12 +6017,19 @@ class AsyncTable:

        Parameters
        ----------
-        transforms: Dict[str, str]
+        transforms: Dict[str, str | FunctionApplication] or FunctionApplication
            A map of column name to a SQL expression to use to calculate the
            value of the new column. These expressions will be evaluated for
            each row in the table, and can reference existing columns.
            Alternatively, you can pass a pyarrow field or schema to add
            new columns with NULLs.
+
+            A mapping with one ``FunctionApplication`` value keeps its scalar
+            or named-struct result in the named table column. A bare
+            named-struct application expands its ordered result fields as one
+            atomic sibling group; aliases come from ``rename(columns=...)``.
+            Function columns are supported only on LanceDB Cloud and
+            Enterprise.
        computed: Dict[str, str], optional
            A map of column name to a SQL expression defining the column. The
            column's type and inputs are derived from the expression.
@@ -6010,6 +6053,32 @@ class AsyncTable:
            version: the new version number of the table after adding columns.

        """
+        function_application = None
+        function_output_name = None
+        if isinstance(transforms, FunctionApplication):
+            function_application = transforms
+        elif isinstance(transforms, dict) and any(
+            isinstance(value, FunctionApplication) for value in transforms.values()
+        ):
+            if len(transforms) != 1 or not all(
+                isinstance(value, FunctionApplication) for value in transforms.values()
+            ):
+                raise ValueError(
+                    "one add_columns call declares exactly one Function sibling group"
+                )
+            function_output_name, function_application = next(iter(transforms.items()))
+
+        if function_application is not None:
+            if computed:
+                raise ValueError(
+                    "add_columns cannot mix a Function application with SQL "
+                    "computed columns"
+                )
+            function_application._ensure_declarable()
+            return await self._inner.add_function_columns(
+                function_application.to_canonical_json(), function_output_name
+            )
+
        if isinstance(transforms, pa.Field):
            transforms = [transforms]
        if isinstance(transforms, list) and all(
@@ -37,6 +37,11 @@ from unittest.mock import patch
 import lancedb
 import pyarrow as pa
 import pytest
+from utils import (
+    MockPermutationServer,
+    assert_server_safe_row_id_requests,
+    mock_remote_table,
+)

 torch = pytest.importorskip("torch")
 streaming = pytest.importorskip("lancedb.streaming")
@@ -2118,3 +2123,27 @@ def test_doc_example_checkpoint(lance_table):
    assert sorted(consumed + remaining_original) == list(range(NUM_ROWS)), (
        "Consumed + remaining must cover every row exactly once"
    )
+
+
+# ---------------------------------------------------------------------------
+# Remote tables (LanceDB Cloud / Enterprise)
+# ---------------------------------------------------------------------------
+
+
+def test_streaming_dataset_over_remote_table():
+    """StreamingDataset reads a remote table, with server-safe requests.
+
+    Builds a permutation over a remote table, then fetches batches from it by row id.
+    """
+    server = MockPermutationServer()
+
+    with mock_remote_table(server) as table:
+        ds = StreamingDataset(table, num_splits=2, shuffle_seed=SHUFFLE_SEED)
+        ids = [row["id"] for row in ds]
+
+    assert sorted(ids) == list(range(server.num_rows)), (
+        "Every row of the remote table must be yielded exactly once"
+    )
+    assert len(server.scans) == 1, "the permutation is built with one row-id scan"
+    assert server.takes, "rows must be fetched with row-id takes"
+    assert_server_safe_row_id_requests(server)
@@ -0,0 +1,389 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import json
+from pathlib import Path
+
+import pytest
+
+from lancedb import col
+import lancedb.functions as functions
+from lancedb.functions import (
+    FunctionApplication,
+    FunctionBinding,
+    FunctionVersion,
+    PythonRuntimeSpec,
+    RefreshColumnResult,
+)
+from lancedb.table import AsyncTable
+
+
+FIXTURES = (
+    Path(__file__).parents[3]
+    / "rust"
+    / "lancedb"
+    / "tests"
+    / "fixtures"
+    / "first_class_functions"
+    / "v1"
+)
+
+
+def fixture(name: str) -> str:
+    return (FIXTURES / name).read_text()
+
+
+def job_result(name: str) -> dict:
+    return json.loads(fixture(name))["result"]
+
+
+def assert_no_secret_values(value):
+    if isinstance(value, dict):
+        for key, child in value.items():
+            assert key not in {
+                "secret_value",
+                "secret_values",
+                "resolved_secret",
+                "resolved_secrets",
+            }
+            assert_no_secret_values(child)
+    elif isinstance(value, list):
+        for child in value:
+            assert_no_secret_values(child)
+
+
+def test_public_function_values_are_in_api_reference():
+    docs = Path(__file__).parents[3] / "docs" / "src" / "python" / "python.md"
+    rendered = docs.read_text()
+    for name in functions.__all__:
+        assert f"::: lancedb.functions.{name}" in rendered
+
+
+@pytest.mark.parametrize(
+    ("fixture_name", "canonical_name", "model", "nested_result"),
+    [
+        (
+            "remote_function_job.json",
+            "remote_function_version.canonical.json",
+            FunctionVersion,
+            True,
+        ),
+        (
+            "remote_function_application.json",
+            "remote_function_application.canonical.json",
+            FunctionApplication,
+            False,
+        ),
+        (
+            "remote_function_binding.json",
+            "remote_function_binding.canonical.json",
+            FunctionBinding,
+            False,
+        ),
+        (
+            "remote_refresh_job.json",
+            "remote_refresh_result.canonical.json",
+            RefreshColumnResult,
+            True,
+        ),
+        (
+            "remote_refresh_result_without_published_version.json",
+            "remote_refresh_result_without_published_version.canonical.json",
+            RefreshColumnResult,
+            False,
+        ),
+    ],
+)
+def test_python_and_rust_share_remote_canonical_goldens(
+    fixture_name, canonical_name, model, nested_result
+):
+    value = json.loads(fixture(fixture_name))
+    if nested_result:
+        value = value["result"]
+    decoded = model.from_json(json.dumps(value))
+    assert decoded.to_canonical_json() == fixture(canonical_name).strip()
+
+
+def test_function_version_identity_is_immutable_and_exact():
+    value = job_result("remote_function_job.json")
+    version = FunctionVersion.from_json(json.dumps(value))
+    assert version.name == "embed"
+    assert version.version == "fv_01K3EXACT"
+    assert version.required_secrets == ("HF_TOKEN",)
+
+    with pytest.raises((TypeError, ValueError)):
+        version.version = "fv_changed"
+    with pytest.raises(TypeError, match="immutable"):
+        version.runtime.env["TOKENIZERS_PARALLELISM"] = "true"
+
+    changed = dict(value)
+    changed["version"] = "fv_changed"
+    assert FunctionVersion(**changed) != version
+
+
+def test_function_version_binds_named_columns_as_one_immutable_group():
+    version = FunctionVersion.from_json(
+        json.dumps(job_result("remote_function_job.json"))
+    )
+
+    application = version(text=col("documents.body"))
+
+    assert application.function.name == version.name
+    assert application.function.version == version.version
+    assert application.output is version.signature.output
+    assert application.group_id.startswith("fg_")
+    assert [
+        (value.parameter, value.kind, value.value["path"])
+        for value in application.inputs
+    ] == [("text", "column", "documents.body")]
+    with pytest.raises((TypeError, ValueError)):
+        application.group_id = "fg_changed"
+
+
+def test_function_version_binding_validates_names_and_direct_columns():
+    version = FunctionVersion.from_json(
+        json.dumps(job_result("remote_function_job.json"))
+    )
+
+    with pytest.raises(TypeError, match=r"missing inputs: \['text'\]"):
+        version()
+    with pytest.raises(TypeError, match=r"unknown inputs: \['body'\]"):
+        version(text=col("text"), body=col("body"))
+    with pytest.raises(TypeError, match="direct col"):
+        version(text=col("text").lower())
+
+
+def test_function_version_keeps_named_struct_outputs_in_one_application():
+    value = job_result("remote_function_job.json")
+    value["name"] = "text_features"
+    value["version"] = "fv_grouped"
+    value["signature"] = {
+        "inputs": [
+            {"name": "title", "arrow_type": "utf8", "nullable": True},
+            {"name": "body", "arrow_type": "utf8", "nullable": True},
+        ],
+        "output": {
+            "kind": "named_struct",
+            "fields": [
+                {
+                    "name": "normalized_text",
+                    "arrow_type": "utf8",
+                    "nullable": False,
+                },
+                {
+                    "name": "token_count",
+                    "arrow_type": "int64",
+                    "nullable": False,
+                },
+            ],
+        },
+    }
+    version = FunctionVersion(**value)
+
+    application = version(body=col("body"), title=col("title")).rename(
+        columns={
+            "normalized_text": "search_text",
+            "token_count": "search_token_count",
+        }
+    )
+
+    assert [value.parameter for value in application.inputs] == ["title", "body"]
+    assert [field.name for field in application.output.fields] == [
+        "normalized_text",
+        "token_count",
+    ]
+    assert dict(application.columns) == {
+        "normalized_text": "search_text",
+        "token_count": "search_token_count",
+    }
+
+
+def test_unknown_fields_and_discriminators_are_forward_decodable():
+    value = job_result("remote_function_job.json")
+    value["future_version_metadata"] = {"retention_class": "catalog"}
+    value["runtime"] = {"kind": "wasm", "module_digest": "sha256:wasm"}
+    value["signature"]["output"]["kind"] = "future_output_shape"
+
+    version = FunctionVersion.from_json(json.dumps(value))
+    assert version.runtime.kind == "wasm"
+    assert version.runtime.python_version is None
+    assert version.runtime.environment is None
+    assert json.loads(version.to_canonical_json())["runtime"] == {"kind": "wasm"}
+    assert version.signature.output.kind == "future_output_shape"
+
+
+def test_function_application_uses_rename_columns_only():
+    application = FunctionApplication.from_json(
+        fixture("remote_function_application.json")
+    )
+    renamed = application.rename(columns={"normalized_text": "body_normalized"})
+
+    assert application.columns["normalized_text"] == "search_text"
+    assert renamed.columns["normalized_text"] == "body_normalized"
+    assert renamed.function == application.function
+    assert renamed.group_id == application.group_id
+    assert not hasattr(application, "rename_outputs")
+    with pytest.raises(TypeError, match="immutable"):
+        renamed.columns["normalized_text"] = "changed"
+    with pytest.raises(TypeError, match="immutable"):
+        application.inputs[0].value["path"] = "changed"
+
+    with pytest.raises(ValueError, match="unknown Function result fields"):
+        application.rename(columns={"missing": "search_text"})
+    with pytest.raises(ValueError, match="destinations must be unique"):
+        application.rename(columns={"normalized_text": "same", "token_count": "same"})
+
+    bare_value = json.loads(fixture("remote_function_application.json"))
+    bare_value.pop("columns")
+    bare = FunctionApplication(**bare_value)
+    with pytest.raises(ValueError, match="destinations must be unique"):
+        bare.rename(columns={"normalized_text": "token_count"})
+
+
+def test_binding_and_refresh_result_keep_stable_remote_fields():
+    binding = FunctionBinding.from_json(fixture("remote_function_binding.json"))
+    assert binding.revision == 3
+    assert binding.function.version == "fv_01K3TEXT"
+    assert [output.output_ordinal for output in binding.outputs] == [0, 1]
+    assert binding.input_schema is not None
+    assert binding.output_schema is not None
+
+    result = RefreshColumnResult.from_json(
+        json.dumps(job_result("remote_refresh_job.json"))
+    )
+    assert result.rows_filled == result.rows_assigned
+    assert result.version == result.published_version
+
+    result = RefreshColumnResult.from_json(
+        fixture("remote_refresh_result_without_published_version.json")
+    )
+    assert result.published_version is None
+    assert RefreshColumnResult.from_json(result.to_canonical_json()) == result
+
+
+def test_function_literal_numeric_domain_matches_rust():
+    with pytest.raises(ValueError, match="floating-point Function literals"):
+        FunctionApplication.from_json(fixture("remote_function_application_float.json"))
+
+    value = json.loads(fixture("remote_function_application_float.json"))
+    value["inputs"][0]["value"] = 2**64
+    with pytest.raises(ValueError, match="outside the canonical JSON range"):
+        FunctionApplication.from_json(json.dumps(value))
+
+
+def test_empty_default_maps_have_stable_canonical_bytes():
+    runtime = PythonRuntimeSpec(
+        kind="python", python_version="3.12", environment={"kind": "pip"}
+    )
+    assert runtime.to_canonical_json() == (
+        '{"environment":{"kind":"pip"},"kind":"python","python_version":"3.12"}'
+    )
+
+    value = json.loads(fixture("remote_function_application.json"))
+    value.pop("columns")
+    application = FunctionApplication.from_json(json.dumps(value))
+    assert "columns" not in json.loads(application.to_canonical_json())
+
+
+@pytest.mark.parametrize("field", ["rows_assigned", "source_version"])
+def test_refresh_result_rejects_non_u64_values(field):
+    value = job_result("remote_refresh_job.json")
+    value[field] = -1
+    with pytest.raises(ValueError):
+        RefreshColumnResult.from_json(json.dumps(value))
+
+    value[field] = "1"
+    with pytest.raises(ValueError):
+        RefreshColumnResult.from_json(json.dumps(value))
+
+
+def test_canonical_client_values_contain_secret_names_only():
+    version = FunctionVersion.from_json(
+        json.dumps(job_result("remote_function_job.json"))
+    )
+    canonical = json.loads(version.to_canonical_json())
+    assert canonical["required_secrets"] == ["HF_TOKEN"]
+    assert_no_secret_values(canonical)
+
+
+class _FunctionDeclarationInner:
+    def __init__(self):
+        self.calls = []
+
+    async def add_function_columns(self, application_json, output_name):
+        self.calls.append((json.loads(application_json), output_name))
+        return "declared"
+
+
+def known_application() -> FunctionApplication:
+    value = json.loads(fixture("remote_function_application.json"))
+    value.pop("future_application")
+    return FunctionApplication(**value)
+
+
+@pytest.mark.asyncio
+async def test_add_columns_routes_struct_as_one_and_grouped_expansion_atomically():
+    inner = _FunctionDeclarationInner()
+    table = AsyncTable(inner)
+    application = known_application()
+
+    result = await table.add_columns(
+        {"features": application._copy(update={"columns": {}})}
+    )
+    assert result == "declared"
+    assert inner.calls[-1][1] == "features"
+
+    bare = application._copy(update={"columns": {}}).rename(
+        columns={"normalized_text": "search_text"}
+    )
+    result = await table.add_columns(bare)
+    assert result == "declared"
+    assert inner.calls[-1][1] is None
+    assert inner.calls[-1][0]["columns"] == {"normalized_text": "search_text"}
+
+
+@pytest.mark.asyncio
+async def test_add_columns_rejects_mixed_groups_and_unknown_newer_application():
+    inner = _FunctionDeclarationInner()
+    table = AsyncTable(inner)
+    application = known_application()
+
+    with pytest.raises(ValueError, match="exactly one Function sibling group"):
+        await table.add_columns({"a": application, "b": application})
+
+    future = json.loads(fixture("remote_function_application.json"))
+    application = FunctionApplication(**future)
+    with pytest.raises(ValueError, match="newer contract"):
+        await table.add_columns(application)
+
+    future.pop("future_application")
+    future["output"]["assignment"] = "cell_flag"
+    application = FunctionApplication(**future)
+    assert "assignment" not in json.loads(application.to_canonical_json())["output"]
+    with pytest.raises(ValueError, match="output.assignment"):
+        await table.add_columns(application)
+    assert inner.calls == []
+
+
+def test_rename_requires_named_struct_and_keeps_partial_mapping_immutable():
+    scalar = FunctionApplication.from_json(
+        json.dumps(
+            {
+                "function": {"name": "embed", "version": "fv_exact"},
+                "inputs": [],
+                "output": {
+                    "kind": "scalar",
+                    "arrow_type": "list<float32>",
+                    "nullable": False,
+                },
+                "group_id": "fg_scalar",
+            }
+        )
+    )
+    with pytest.raises(ValueError, match="named-struct"):
+        scalar.rename(columns={"value": "embedding"})
+
+    application = known_application()._copy(update={"columns": {}})
+    renamed = application.rename(columns={"normalized_text": "search_text"})
+    assert dict(application.columns) == {}
+    assert dict(renamed.columns) == {"normalized_text": "search_text"}
@@ -0,0 +1,254 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+from __future__ import annotations
+
+import contextlib
+import http.server
+import json
+from pathlib import Path
+import threading
+from typing import Optional
+
+import pyarrow as pa
+import pytest
+
+import lancedb
+from lancedb.functions import UdfDefinition, udf
+
+
+FIXTURES = (
+    Path(__file__).parents[3]
+    / "rust"
+    / "lancedb"
+    / "tests"
+    / "fixtures"
+    / "first_class_functions"
+    / "v1"
+)
+
+
+@udf(
+    pip=["numpy>=2"],
+    env={"MODE": "test"},
+    secrets=["API_TOKEN"],
+    python_version="3.12",
+)
+def normalize_score(value: float) -> float:
+    return value / 100.0
+
+
+def _assert_no_secret_values(value):
+    if isinstance(value, dict):
+        for key, child in value.items():
+            assert key not in {
+                "secret_value",
+                "secret_values",
+                "resolved_secret",
+                "resolved_secrets",
+            }
+            _assert_no_secret_values(child)
+    elif isinstance(value, list):
+        for child in value:
+            _assert_no_secret_values(child)
+
+
+def test_scalar_udf_matches_shared_registration_golden_and_remains_callable():
+    assert isinstance(normalize_score, UdfDefinition)
+    assert normalize_score(25.0) == 0.25
+    assert (
+        normalize_score.registration_request.to_canonical_json()
+        == (FIXTURES / "remote_function_registration_request.canonical.json")
+        .read_text()
+        .strip()
+    )
+    request = json.loads(normalize_score.registration_request.to_canonical_json())
+    assert request["artifact"]["adapter"] == {
+        "kind": "scalar_to_arrow_batch",
+        "version": 1,
+    }
+    assert request["required_secrets"] == ["API_TOKEN"]
+    _assert_no_secret_values(request)
+
+
+def test_explicit_arrow_schema_is_deterministic():
+    input_schema = pa.schema([pa.field("value", pa.float32(), nullable=True)])
+    output_schema = pa.field("embedding", pa.list_(pa.float32(), 3), nullable=False)
+
+    @udf(input_schema=input_schema, output_schema=output_schema)
+    def explicit(value):
+        return [value, value, value]
+
+    signature = explicit.registration_request.signature
+    assert signature.inputs[0].arrow_type == "float32"
+    assert signature.inputs[0].nullable is True
+    assert signature.output.arrow_type == "fixed_size_list<float32>[3]"
+    assert signature.output.nullable is False
+
+
+def test_annotation_and_explicit_schema_validation_fail_closed():
+    with pytest.raises(TypeError, match="missing Function annotations"):
+
+        @udf
+        def missing(value):
+            return value
+
+    with pytest.raises(TypeError, match="unsupported Function annotation"):
+
+        @udf
+        def unsupported(value: set[str]) -> str:
+            return ""
+
+    with pytest.raises(ValueError, match="output must be non-nullable"):
+
+        @udf
+        def nullable_output(value: int) -> Optional[int]:
+            return value
+
+    with pytest.raises(ValueError, match="provided together"):
+
+        @udf(input_schema=pa.schema([pa.field("value", pa.int64())]))
+        def partial_schema(value):
+            return value
+
+    with pytest.raises(ValueError, match="exactly match callable parameters"):
+
+        @udf(
+            input_schema=pa.schema([pa.field("other", pa.int64())]),
+            output_schema=pa.int64(),
+        )
+        def wrong_name(value):
+            return value
+
+    with pytest.raises(ValueError, match="output must be non-nullable"):
+
+        @udf(
+            input_schema=pa.schema([pa.field("value", pa.int64())]),
+            output_schema=pa.field("result", pa.int64(), nullable=True),
+        )
+        def nullable_explicit(value):
+            return value
+
+
+def test_environment_rejects_secret_value_overlap():
+    with pytest.raises(ValueError, match="must be disjoint"):
+
+        @udf(env={"TOKEN": "plaintext"}, secrets=["TOKEN"])
+        def overlapping(value: int) -> int:
+            return value
+
+
+def test_local_function_catalog_operations_are_not_supported(tmp_path):
+    db = lancedb.connect(tmp_path)
+    message = "Function catalog operations are not supported by this database"
+    with pytest.raises(NotImplementedError, match=message):
+        db.create_function(normalize_score)
+    with pytest.raises(NotImplementedError, match=message):
+        db.create_function_async(normalize_score)
+    with pytest.raises(NotImplementedError, match=message):
+        db.get_function("normalize_score", version="fv_exact")
+
+
+@contextlib.contextmanager
+def _mock_remote_function_catalog():
+    state = {"requests": [], "version": None}
+
+    class Handler(http.server.BaseHTTPRequestHandler):
+        def log_message(self, *args):
+            pass
+
+        def do_POST(self):
+            length = int(self.headers.get("Content-Length", "0"))
+            body = json.loads(self.rfile.read(length) or b"{}")
+            state["requests"].append((self.path, body))
+            status = 200
+            if self.path == "/v1/functions/create":
+                state["version"] = {
+                    "name": body["name"],
+                    "version": "fv_exact",
+                    "artifact": {
+                        key: body["artifact"][key]
+                        for key in ("kind", "digest", "entrypoint")
+                    },
+                    "signature": body["signature"],
+                    "runtime": body["runtime"],
+                    "runtime_digest": "sha256:runtime",
+                    "environment_digest": "sha256:environment",
+                    "required_secrets": body.get("required_secrets", []),
+                    "created_at": "2026-08-21T00:00:00Z",
+                }
+                response = {"job_id": "job-register"}
+                status = 202
+            elif self.path == "/v1/jobs/describe":
+                assert body == {"job_id": "job-register"}
+                response = {
+                    "job_id": "job-register",
+                    "job_type": "create_function",
+                    "job_state": "DONE",
+                    "result": state["version"],
+                }
+            elif self.path == "/v1/functions/get":
+                assert body == {
+                    "name": "normalize_score",
+                    "version": "fv_exact",
+                }
+                response = state["version"]
+            else:
+                status = 404
+                response = {"error": "not found"}
+            encoded = json.dumps(response).encode()
+            self.send_response(status)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(encoded)))
+            self.end_headers()
+            self.wfile.write(encoded)
+
+    with http.server.HTTPServer(("localhost", 0), Handler) as server:
+        thread = threading.Thread(target=server.serve_forever)
+        thread.start()
+        try:
+            yield f"http://localhost:{server.server_address[1]}", state
+        finally:
+            server.shutdown()
+            thread.join()
+
+
+def test_remote_registration_job_and_exact_version_reopen_round_trip():
+    with _mock_remote_function_catalog() as (host, state):
+        db = lancedb.connect(
+            "db://dev",
+            api_key="fake",
+            host_override=host,
+            client_config={"retry_config": {"retries": 0}},
+        )
+        registration = db.create_function_async(normalize_score)
+        assert registration.id == "job-register"
+        created = registration.wait()
+        reopened = db.get_function("normalize_score", version=created.version)
+
+    assert created == reopened
+    assert reopened.name == "normalize_score"
+    assert reopened.version == "fv_exact"
+    create_request = state["requests"][0][1]
+    assert create_request == json.loads(
+        normalize_score.registration_request.to_canonical_json()
+    )
+    _assert_no_secret_values(create_request)
+
+
+def test_blocking_remote_registration_returns_function_version():
+    with _mock_remote_function_catalog() as (host, state):
+        db = lancedb.connect(
+            "db://dev",
+            api_key="fake",
+            host_override=host,
+            client_config={"retry_config": {"retries": 0}},
+        )
+        created = db.create_function(normalize_score)
+
+    assert created.name == "normalize_score"
+    assert created.version == "fv_exact"
+    assert [path for path, _ in state["requests"]] == [
+        "/v1/functions/create",
+        "/v1/jobs/describe",
+    ]
@@ -245,6 +245,14 @@ def test_create_inverted_index_rejects_invalid_block_size(table):
        table.create_index("text", config=FTS(block_size=129))


+def test_create_inverted_index_respects_build_memory_limit(table):
+    with pytest.raises(ValueError, match="exceeds worker memory limit"):
+        table.create_index(
+            "text",
+            config=FTS(memory_limit=0, num_workers=1),
+        )
+
+
 def test_custom_stop_words_list(table):
    table.create_index(
        "text",
@@ -21,6 +21,11 @@ SCHEMA = pa.schema(
 )


+def test_lsm_write_spec_module_metadata():
+    assert lancedb.LsmWriteSpec is LsmWriteSpec
+    assert LsmWriteSpec.__module__ == "lancedb._lancedb"
+
+
 def _batch(ids, vs):
    return pa.RecordBatch.from_arrays(
        [pa.array(ids, type=pa.utf8()), pa.array(vs, type=pa.int32())],
@@ -8,6 +8,11 @@ import pytest
 from lancedb import DBConnection, Table, connect
 from lancedb.background_loop import LOOP
 from lancedb.permutation import Permutation, Permutations, permutation_builder
+from utils import (
+    MockPermutationServer,
+    assert_server_safe_row_id_requests,
+    mock_remote_table,
+)


 def test_split_random_ratios(mem_db):
@@ -1214,3 +1219,57 @@ def test_remove_rowid_after_select(some_permutation: Permutation):
    perm_without_rowid = perm_with_rowid.remove_columns(["_rowid"])
    assert "_rowid" not in perm_without_rowid.column_names
    assert perm_without_rowid.column_names == ["id"]
+
+
+def test_permutation_is_stable_when_remote_scan_order_varies():
+    """Splits are assigned by scan position, and every rank builds its own
+    permutation, so two ranks seeing different scan orders must still agree."""
+    server = MockPermutationServer(num_rows=16, vary_scan_order=True)
+
+    def split_of_each_row(permutation_tbl):
+        # Sequential splits are assigned by position, so a reversed scan would put
+        # the last rows in split 0.  Compare the mapping rather than the table order,
+        # which the split-id sort does not pin down.
+        rows = permutation_tbl.search(None).to_arrow().to_pydict()
+        return dict(zip(rows["row_id"], rows["split_id"]))
+
+    with mock_remote_table(server) as table:
+        first = split_of_each_row(
+            permutation_builder(table).split_sequential(fixed=2).execute()
+        )
+        second = split_of_each_row(
+            permutation_builder(table).split_sequential(fixed=2).execute()
+        )
+
+    assert server.scan_calls == 2, "both builds must have scanned"
+    assert first == second
+    assert first[0] == 0 and first[server.num_rows - 1] == 1, first
+
+
+def test_permutation_over_remote_table():
+    """The permutation API accepts a remote table, addressing rows by `_rowid` just
+    as `take_row_ids` does.  Also pins the request shapes sent to the server.
+    """
+    server = MockPermutationServer()
+
+    with mock_remote_table(server) as table:
+        permutation_tbl = permutation_builder(table).split_sequential(fixed=2).execute()
+        assert permutation_tbl.count_rows() == server.num_rows
+
+        permutation = Permutation.from_tables(table, permutation_tbl, 0)
+        assert permutation.num_rows == server.num_rows // 2
+
+        # Compare against the permutation's own order; the split-id sort is not stable.
+        rows = permutation_tbl.search(None).to_arrow().to_pydict()
+        split0 = [
+            row_id
+            for row_id, split in zip(rows["row_id"], rows["split_id"])
+            if not split
+        ]
+        # The mock table's `id` equals its `_rowid`.
+        assert permutation.take_offsets([2, 0]) == [
+            {"id": split0[2]},
+            {"id": split0[0]},
+        ]
+
+    assert_server_safe_row_id_requests(server)
@@ -10,11 +10,10 @@ import pyarrow as pa
 import pydantic
 import pytest
 from lancedb.pydantic import (
-    PYDANTIC_VERSION,
    LanceModel,
+    MultiVector,
    Vector,
    pydantic_to_schema,
-    MultiVector,
 )
 from pydantic import BaseModel
 from pydantic import Field
@@ -432,16 +431,10 @@ def test_fixed_size_list_field():
        li: List[int]

    data = TestModel(vec=list(range(16)), li=[1, 2, 3])
-    if PYDANTIC_VERSION.major >= 2:
-        assert json.loads(data.model_dump_json()) == {
-            "vec": list(range(16)),
-            "li": [1, 2, 3],
-        }
-    else:
-        assert data.dict() == {
-            "vec": list(range(16)),
-            "li": [1, 2, 3],
-        }
+    assert json.loads(data.model_dump_json()) == {
+        "vec": list(range(16)),
+        "li": [1, 2, 3],
+    }

    schema = pydantic_to_schema(TestModel)
    assert schema == pa.schema(
@@ -451,10 +444,7 @@ def test_fixed_size_list_field():
        ]
    )

-    if PYDANTIC_VERSION.major >= 2:
-        json_schema = TestModel.model_json_schema()
-    else:
-        json_schema = TestModel.schema()
+    json_schema = TestModel.model_json_schema()

    assert json_schema == {
        "properties": {
@@ -2772,6 +2772,56 @@ async def test_merge_insert_async(mem_db_async: AsyncConnection):
    assert (await table.to_arrow()).sort_by("a") == expected


+@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
+@pytest.mark.asyncio
+async def test_merge_insert_encodes_json(mem_db_async: AsyncConnection):
+    json_type = pa.json_()
+    schema = pa.schema([pa.field("id", pa.string()), pa.field("j", json_type)])
+
+    def json_table(rows):
+        json_values = pa.ExtensionArray.from_storage(
+            json_type,
+            pa.array([value for _, value in rows], type=json_type.storage_type),
+        )
+        return pa.Table.from_arrays(
+            [pa.array([row_id for row_id, _ in rows]), json_values], schema=schema
+        )
+
+    table = await mem_db_async.create_table("json_merge", schema=schema)
+    await table.add(json_table([("a", '{"k": 1}'), ("b", '{"k": 9}')]))
+
+    await (
+        table.merge_insert("id")
+        .when_matched_update_all()
+        .execute(json_table([("a", '{"k": 2}')]))
+    )
+
+    rows = sorted(await table.query().to_list(), key=lambda row: row["id"])
+    assert rows == [
+        {"id": "a", "j": '{"k":2}'},
+        {"id": "b", "j": '{"k":9}'},
+    ]
+    filtered = await table.query().where("json_extract(j, '$.k') = '2'").to_list()
+    assert filtered == [{"id": "a", "j": '{"k":2}'}]
+
+
+@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
+@pytest.mark.asyncio
+async def test_add_sanitization_encodes_json(mem_db_async: AsyncConnection):
+    json_type = pa.json_()
+    schema = pa.schema([pa.field("id", pa.string()), pa.field("j", json_type)])
+    json_values = pa.ExtensionArray.from_storage(
+        json_type, pa.array(['{"k": 3}'], type=json_type.storage_type)
+    )
+    data = pa.Table.from_arrays([pa.array(["c"]), json_values], schema=schema)
+
+    table = await mem_db_async.create_table("json_add", schema=schema)
+    await table.add(data, on_bad_vectors="fill")
+
+    rows = await table.query().where("json_extract(j, '$.k') = '3'").to_list()
+    assert rows == [{"id": "c", "j": '{"k":3}'}]
+
+
 def test_create_with_embedding_function(mem_db: DBConnection):
    class MyTable(LanceModel):
        text: str
@@ -3897,7 +3947,7 @@ def test_refresh_column_async_returns_job(tmp_path):

    job = table.refresh_column_async("doubled")
    assert job.id is None  # in-process jobs have no server id
-    job.wait()
+    assert job.wait() is None
    assert job.status() == "finished"
    assert sorted(table.to_arrow()["doubled"].to_pylist()) == [2, 4]

@@ -3913,6 +3963,6 @@ async def test_refresh_column_async_job_async_table(tmp_path):
    await table.add_columns(computed={"tripled": "x * 3"})

    job = await table.refresh_column_async("tripled")
-    await job.wait()
+    assert await job.wait() is None
    assert await job.status() == "finished"
    assert (await table.to_arrow())["tripled"].to_pylist() == [9]
@@ -1,7 +1,17 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import contextlib
+import http.server
+import json
+import re
+import threading
+
+import lancedb
+import pyarrow as pa
 import pytest

+ARROW_FILE_CONTENT_TYPE = "application/vnd.apache.arrow.file"
+

 def exception_output(e_info: pytest.ExceptionInfo):
    import traceback
@@ -9,3 +19,199 @@ def exception_output(e_info: pytest.ExceptionInfo):
    # skip traceback part, since it's not worth checking in tests
    lines = traceback.format_exception_only(e_info.type, e_info.value)
    return "".join(lines).strip()
+
+
+def parse_in_list(filter_sql: str) -> list[int]:
+    """Pull the integers out of a `<col> IN (a, b, c)` predicate.
+
+    Scoped to the parenthesised list so a cast in the SQL adds no phantom values.
+    """
+    match = re.search(r"\bIN\s*\(([^)]*)\)", filter_sql, re.IGNORECASE)
+    assert match is not None, f"expected an IN list, got: {filter_sql}"
+    return [int(m) for m in re.findall(r"-?\d+", match.group(1))]
+
+
+def is_row_id_take(body) -> bool:
+    """True when a query body fetches specific rows by row id."""
+    return "_rowid" in (body.get("filter") or "")
+
+
+def arrow_file_bytes(table: pa.Table) -> bytes:
+    """Serialize to the Arrow IPC *file* framing the /query/ route answers with."""
+    sink = pa.BufferOutputStream()
+    with pa.ipc.new_file(sink, table.schema) as writer:
+        writer.write_table(table)
+    return sink.getvalue().to_pybytes()
+
+
+class MockPermutationServer:
+    """A stand-in LanceDB server hosting one table whose ``id`` equals its ``_rowid``.
+
+    Records every ``/query/`` body so tests can assert on the request shapes sent to
+    the server, which is the part that has to stay compatible.
+    """
+
+    def __init__(self, name="remote_data", num_rows=8, vary_scan_order=False):
+        self.name = name
+        self.num_rows = num_rows
+        self.query_bodies = []
+        # Stand in for a distributed scan that answers in no fixed order.
+        self.vary_scan_order = vary_scan_order
+        self.scan_calls = 0
+
+    def __call__(self, request):
+        path = request.path
+        if path == f"/v1/table/{self.name}/describe/":
+            return self._json(
+                request,
+                {
+                    "version": 1,
+                    "schema": {
+                        "fields": [
+                            {"name": "id", "type": {"type": "int64"}, "nullable": False}
+                        ]
+                    },
+                },
+            )
+        if path == f"/v1/table/{self.name}/get_lsm_write_spec/":
+            self._read_body(request)
+            # Null spec: this table has no LSM write path.
+            return self._json(request, {"lsm_write_spec": None})
+        if path == f"/v1/table/{self.name}/count_rows/":
+            self._read_body(request)
+            return self._json(request, self.num_rows)
+        if path == f"/v1/table/{self.name}/query/":
+            return self._query(request, self._read_body(request))
+
+        # Drain first, so an unexpected route cannot desync a keep-alive connection.
+        self._read_body(request)
+        request.send_response(404)
+        request.end_headers()
+
+    @property
+    def scans(self):
+        """Bodies of the permutation build scan: the row id column, nothing else."""
+        return [b for b in self.query_bodies if b.get("columns") == ["_rowid"]]
+
+    @property
+    def takes(self):
+        """Bodies of the row-id takes the loader fetches batches with.
+
+        Keyed on `_rowid`, not "has a filter": the schema probe also has a predicate.
+        """
+        return [b for b in self.query_bodies if is_row_id_take(b)]
+
+    @staticmethod
+    def _read_body(request):
+        content_len = int(request.headers.get("Content-Length") or 0)
+        return json.loads(request.rfile.read(content_len)) if content_len else {}
+
+    @staticmethod
+    def _json(request, payload):
+        request.send_response(200)
+        request.send_header("Content-Type", "application/json")
+        request.end_headers()
+        request.wfile.write(json.dumps(payload).encode())
+
+    @staticmethod
+    def _arrow(request, table):
+        body = arrow_file_bytes(table)
+        request.send_response(200)
+        request.send_header("Content-Type", ARROW_FILE_CONTENT_TYPE)
+        request.send_header("Content-Length", str(len(body)))
+        request.end_headers()
+        request.wfile.write(body)
+
+    def _query(self, request, body):
+        self.query_bodies.append(body)
+
+        if is_row_id_take(body):
+            # A row-id take. Answer ascending, so tests prove the client reorders.
+            row_ids = sorted(parse_in_list(body["filter"]))
+            return self._arrow(
+                request,
+                pa.table(
+                    {
+                        "id": pa.array(row_ids, pa.int64()),
+                        "_rowid": pa.array(row_ids, pa.uint64()),
+                    }
+                ),
+            )
+
+        if body.get("columns") == ["_rowid"]:
+            # The permutation build scan: row ids and nothing else.
+            row_ids = list(range(self.num_rows))
+            if self.vary_scan_order and self.scan_calls % 2:
+                row_ids.reverse()
+            self.scan_calls += 1
+            return self._arrow(
+                request,
+                pa.table({"_rowid": pa.array(row_ids, pa.uint64())}),
+            )
+
+        # The schema probe: filtered to nothing, so it carries schema and no rows.
+        return self._arrow(request, pa.table({"id": pa.array([], pa.int64())}))
+
+
+def _make_handler(serve):
+    class MockLanceDBHandler(http.server.BaseHTTPRequestHandler):
+        def do_GET(self):
+            serve(self)
+
+        def do_POST(self):
+            serve(self)
+
+        def log_message(self, *args):
+            pass  # keep pytest output readable
+
+    return MockLanceDBHandler
+
+
+@contextlib.contextmanager
+def mock_remote_table(server):
+    """Run ``server`` on a local port and yield an open remote table against it.
+
+    Threading: the loader fans out fetch threads a single-threaded server would
+    serialize, hiding the prefetch overlap under test.
+    """
+    with http.server.ThreadingHTTPServer(
+        ("localhost", 0), _make_handler(server)
+    ) as srv:
+        thread = threading.Thread(target=srv.serve_forever)
+        thread.start()
+        try:
+            db = lancedb.connect(
+                "db://dev",
+                api_key="fake",
+                host_override=f"http://localhost:{srv.server_address[1]}",
+                client_config={"timeout_config": {"connect_timeout": 5}},
+            )
+            yield db.open_table(server.name)
+        finally:
+            srv.shutdown()
+            thread.join()
+
+
+def assert_server_safe_row_id_requests(server):
+    """Assert the loader fetched rows by row id and bounded everything else.
+
+    `.get`, not `[...]`, so a dropped field reads as the assertion, not a KeyError.
+    """
+    for body in server.takes:
+        # The fetch needs the row id back to restore the requested order.
+        assert body.get("with_row_id") is True, body
+        assert "_rowid" in body["filter"], body
+
+    # Only the one-off permutation scan may scan the whole table; the schema probe is
+    # built once per split per epoch. `k == 0` counts as unbounded: lance reads a zero
+    # limit as "no limit".
+    def is_unbounded(body):
+        if is_row_id_take(body):
+            return False
+        k = body.get("k")
+        return k is None or k == 0 or k > server.num_rows
+
+    unbounded = [b for b in server.query_bodies if is_unbounded(b)]
+    assert unbounded == server.scans, (
+        f"only the permutation scan may be unbounded, got {unbounded}"
+    )
@@ -563,6 +563,38 @@ impl Connection {
        Ok(crate::job::Job::new(inner.job(job_id).infer_error()?))
    }

+    pub fn create_function_async(
+        self_: PyRef<'_, Self>,
+        request_json: String,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        let request = lancedb::function::FunctionRegistrationRequest::from_json(&request_json)
+            .infer_error()?;
+        future_into_py(self_.py(), async move {
+            inner
+                .create_function_async(request)
+                .await
+                .infer_error()
+                .map(crate::job::FunctionJob::new)
+        })
+    }
+
+    pub fn get_function(
+        self_: PyRef<'_, Self>,
+        name: String,
+        version: String,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        future_into_py(self_.py(), async move {
+            inner
+                .get_function(name, version)
+                .await
+                .infer_error()?
+                .to_canonical_json()
+                .infer_error()
+        })
+    }
+
    pub fn list_jobs(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
        let inner = self_.get_inner()?.clone();
        future_into_py(self_.py(), async move {
@@ -42,7 +42,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
            "Fm" => Ok(LanceDbIndex::Fm(FmIndexBuilder::default())),
            "FTS" => {
                let params = source.extract::<FtsParams>()?;
-                let inner_opts = FtsIndexBuilder::default()
+                let mut inner_opts = FtsIndexBuilder::default()
                    .base_tokenizer(params.base_tokenizer)
                    .language(&params.language)
                    .map_err(|_| {
@@ -61,6 +61,12 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                    .ngram_max_length(params.ngram_max_length)
                    .ngram_prefix_only(params.prefix_only)
                    .custom_stop_words(params.custom_stop_words);
+                if let Some(memory_limit) = params.memory_limit {
+                    inner_opts = inner_opts.memory_limit_mb(memory_limit);
+                }
+                if let Some(num_workers) = params.num_workers {
+                    inner_opts = inner_opts.num_workers(num_workers);
+                }
                let inner_opts = inner_opts
                    .block_size(params.block_size)
                    .map_err(|err| PyValueError::new_err(err.to_string()))?;
@@ -213,6 +219,8 @@ struct FtsParams {
    ngram_max_length: u32,
    prefix_only: bool,
    block_size: usize,
+    memory_limit: Option<u64>,
+    num_workers: Option<usize>,
 }

 #[derive(FromPyObject)]
@@ -444,3 +452,51 @@ impl IndexConfig {
        }
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use pyo3::types::{PyDict, PyDictMethods};
+    use serde_json::json;
+
+    #[test]
+    fn fts_build_controls_are_forwarded() {
+        Python::initialize();
+        Python::attach(|py| {
+            let locals = PyDict::new(py);
+            py.run(
+                c"class FTS:
+    with_position = True
+    base_tokenizer = 'simple'
+    language = 'English'
+    max_token_length = None
+    lower_case = True
+    stem = False
+    remove_stop_words = False
+    custom_stop_words = None
+    ascii_folding = False
+    ngram_min_length = 3
+    ngram_max_length = 3
+    prefix_only = False
+    block_size = 128
+    memory_limit = 2048
+    num_workers = 7
+
+config = FTS()",
+                None,
+                Some(&locals),
+            )
+            .unwrap();
+
+            let config = locals.get_item("config").unwrap().unwrap();
+            let index = extract_index_params(&Some(config)).unwrap();
+            let LanceDbIndex::FTS(params) = index else {
+                panic!("expected FTS index parameters");
+            };
+            let training_json = params.to_training_json().unwrap();
+
+            assert_eq!(training_json.get("memory_limit"), Some(&json!(2048)));
+            assert_eq!(training_json.get("num_workers"), Some(&json!(7)));
+        });
+    }
+}
@@ -13,6 +13,23 @@ pub struct Job {
    inner: Arc<lancedb::Job>,
 }

+/// Python bridge for a typed remote Function registration job.
+///
+/// The public Python layer decodes the canonical JSON returned by `wait`
+/// into its immutable `FunctionVersion` model.
+#[pyclass]
+pub struct FunctionJob {
+    inner: Arc<lancedb::Job<lancedb::function::FunctionVersion>>,
+}
+
+impl FunctionJob {
+    pub(crate) fn new(inner: lancedb::Job<lancedb::function::FunctionVersion>) -> Self {
+        Self {
+            inner: Arc::new(inner),
+        }
+    }
+}
+
 impl Job {
    pub(crate) fn new(inner: lancedb::Job) -> Self {
        Self {
@@ -21,6 +38,42 @@ impl Job {
    }
 }

+#[pymethods]
+impl FunctionJob {
+    #[getter]
+    pub fn id(&self) -> Option<String> {
+        self.inner.id().map(str::to_string)
+    }
+
+    pub fn status(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        future_into_py(
+            self_.py(),
+            async move { inner.status().await.infer_error() },
+        )
+    }
+
+    pub fn wait(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        future_into_py(self_.py(), async move {
+            inner
+                .wait()
+                .await
+                .infer_error()?
+                .to_canonical_json()
+                .infer_error()
+        })
+    }
+
+    pub fn cancel(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        future_into_py(self_.py(), async move {
+            inner.cancel().await.infer_error()?;
+            Ok(())
+        })
+    }
+}
+
 #[pymethods]
 impl Job {
    #[getter]
@@ -40,7 +93,7 @@ impl Job {
        let inner = self_.inner.clone();
        future_into_py(self_.py(), async move {
            inner.wait().await.infer_error()?;
-            Ok(())
+            Ok(None::<()>)
        })
    }

@@ -47,6 +47,7 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
    m.add_class::<Session>()?;
    m.add_class::<Table>()?;
    m.add_class::<crate::job::Job>()?;
+    m.add_class::<crate::job::FunctionJob>()?;
    m.add_class::<crate::job::JobInfo>()?;
    m.add_class::<crate::job::JobDescription>()?;
    m.add_class::<crate::job::JobFailureInfo>()?;
@@ -268,7 +268,9 @@ impl PyPermutationReader {
                    .await
                    .infer_error()?
            } else {
-                PermutationReader::identity(base_table).await
+                PermutationReader::identity(base_table)
+                    .await
+                    .infer_error()?
            };
            Ok(Self::from_reader(reader))
        })
@@ -262,7 +262,7 @@ fn fmt_maintained(maintained: &Option<Vec<String>>) -> String {
 /// classmethods, then optionally chain `with_maintained_indexes(...)` and
 /// `with_writer_config_defaults(...)`. A fresh spec maintains every index the
 /// MemWAL supports, resolved on install.
-#[pyclass(from_py_object)]
+#[pyclass(module = "lancedb._lancedb", from_py_object)]
 #[derive(Clone, Debug)]
 pub struct LsmWriteSpec {
    inner: lancedb::table::LsmWriteSpec,
@@ -1551,6 +1551,24 @@ impl Table {
        })
    }

+    pub fn add_function_columns(
+        self_: PyRef<'_, Self>,
+        application_json: String,
+        output_name: Option<String>,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let application =
+            lancedb::function::FunctionApplication::from_json(&application_json).infer_error()?;
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            let builder = match output_name {
+                Some(name) => inner.add_columns().function_as(name, application),
+                None => inner.add_columns().function(application),
+            };
+            let result = builder.execute().await.infer_error()?;
+            Ok(AddColumnsResult::from(result))
+        })
+    }
+
    pub fn refresh_column(self_: PyRef<'_, Self>, column: String) -> PyResult<Bound<'_, PyAny>> {
        let inner = self_.inner_ref()?.clone();
        future_into_py(self_.py(), async move {
@@ -2003,7 +2003,7 @@ requires-dist = [
    { name = "pyarrow", specifier = ">=16" },
    { name = "pyarrow", marker = "extra == 'tests'", specifier = "<25" },
    { name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" },
-    { name = "pydantic", specifier = ">=1.10" },
+    { name = "pydantic", specifier = ">=2.7.4,<3" },
    { name = "pylance", marker = "extra == 'pylance'", specifier = ">=5.0.0b5" },
    { name = "pylance", marker = "extra == 'tests'", specifier = "==9.0.0rc1" },
    { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" },
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.38.0-beta.2"
+version = "0.38.0-beta.3"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
@@ -51,20 +51,20 @@ metrics = { workspace = true, optional = true }
 metrics-util = { workspace = true, optional = true }
 moka = { workspace = true }
 pin-project = { workspace = true }
-tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
+tokio = { workspace = true }
 log.workspace = true
-async-trait = "0"
-bytes = "1"
+async-trait = { workspace = true }
+bytes = { workspace = true }
 futures.workspace = true
 num-traits.workspace = true
 url.workspace = true
 rand.workspace = true
 regex.workspace = true
-serde = { version = "^1" }
-serde_json = { version = "1" }
+serde = { workspace = true }
+serde_json = { workspace = true }
 async-openai = { version = "0.20.0", optional = true }
 serde_with = { version = "3.8.1" }
-tempfile = "3.5.0"
+tempfile = { workspace = true }
 aws-sdk-bedrockruntime = { version = "1.27.0", optional = true }
 # For remote feature
 reqwest = { version = "0.12.0", default-features = false, features = [
@@ -79,7 +79,7 @@ reqwest = { version = "0.12.0", default-features = false, features = [
 ], optional = true }
 http = { version = "1", optional = true } # Matching what is in reqwest
 urlencoding = { version = "2", optional = true }
-uuid = { version = "1.7.0", features = ["v4", "v5"] }
+uuid = { workspace = true, features = ["v5"] }
 polars-arrow = { version = ">=0.37,<0.40.0", optional = true }
 polars = { version = ">=0.37,<0.40.0", optional = true }
 hf-hub = { version = "0.4.1", optional = true, default-features = false, features = [
@@ -96,11 +96,11 @@ semver = { workspace = true }
 [dev-dependencies]
 anyhow = "1"
 lance-testing = { workspace = true }
-tempfile = "3.5.0"
+tempfile = { workspace = true }
 random_word = { version = "0.4.3", features = ["en"] }
 roaring = "0.11.4"
-tokio = { version = "1.23", features = ["io-util", "macros", "net", "rt-multi-thread", "sync", "test-util"] }
-uuid = { version = "1.7.0", features = ["v4"] }
+tokio = { workspace = true, features = ["io-util", "macros", "net", "test-util"] }
+uuid = { workspace = true }
 walkdir = "2"
 aws-sdk-dynamodb = { version = "1.55.0" }
 aws-sdk-s3 = { version = "1.55.0" }
@@ -1,14 +1,14 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors

-// Release benchmark for opening a missing table as sibling-table cardinality grows.
+// Benchmark for opening a missing table as sibling-table cardinality grows.
 //
 // The fixture uses real `.lance` directories and marker files. Fixture creation is
 // outside the timed section. Defaults intentionally cover 1k, 10k, and 100k siblings
 // with 10 warmups and 100 distinct missing-table opens per scale:
 //
 // ```text
-// cargo run --release -p lancedb --example bench_open_missing_table
+// cargo run --profile release-no-lto -p lancedb --example bench_open_missing_table
 // ```
 //
 // `BENCH_SIBLINGS`, `BENCH_WARMUPS`, and `BENCH_TRIALS` override those defaults.
--- a/Show More
+++ b/Show More