diff --git a/.bumpversion.toml b/.bumpversion.toml index e32f518c9..a6912bed5 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.40.0-beta.11" +current_version = "0.40.0-beta.12" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 98acccf02..2e290166e 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -164,6 +164,71 @@ jobs: python ci/mock_openai.py & cd nodejs/examples node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose + arrow-compat: + # The main jobs resolve `apache-arrow` to the version pinned in + # devDependencies (the newest supported release). This job re-runs the + # suite with the library itself running on each older Arrow release the + # peer range still accepts, which is what a consumer on that version gets. + name: Linux (Arrow ${{ matrix.arrow-version }}) + timeout-minutes: 30 + strategy: + matrix: + arrow-version: [ "19.0.1", "20.0.0" ] + runs-on: "ubuntu-22.04" + defaults: + run: + shell: bash + working-directory: nodejs + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + lfs: true + - uses: pnpm/action-setup@v6 + with: + version: 11.1.1 + - uses: actions/setup-node@v6 + with: + node-version: 24 + cache: 'pnpm' + cache-dependency-path: nodejs/pnpm-lock.yaml + - uses: Swatinem/rust-cache@v2 + with: + # Same crate, profile and target as the `linux` job above, so reuse + # its cache: rust-cache keys on the job id by default, and this job + # would otherwise start cold on every run until it had saved its own. + shared-key: linux + save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y protobuf-compiler libssl-dev + - name: Build + run: | + pnpm install --frozen-lockfile + # No `--` separator: pnpm forwards it literally, which would + # make napi-rs treat `--profile ci` as a cargo passthrough arg. + pnpm build:debug --profile ci + - name: Resolve apache-arrow to ${{ matrix.arrow-version }} + run: | + # The older releases are already installed as the `apache-arrow-NN` + # test aliases. With pnpm's hoisted linker each alias is a complete, + # self-contained directory, so swap it in for the library's own + # `apache-arrow` instead of running a second install, which has been + # seen to hang after printing "Done" in CI. A copy, not a symlink: the + # cross-version tests rely on the alias being a separate module + # instance from the one the library loads. + major="$(echo '${{ matrix.arrow-version }}' | cut -d. -f1)" + mv node_modules/apache-arrow node_modules/apache-arrow.pinned + cp -R "node_modules/apache-arrow-${major}" node_modules/apache-arrow + # `require("apache-arrow/package.json")` is blocked by the package's exports map. + node -p "'apache-arrow resolved to ' + JSON.parse(require('fs').readFileSync('node_modules/apache-arrow/package.json')).version" + pnpm tsc + - name: Test + env: + # Newer @smithy/core uses dynamic ESM imports. + NODE_OPTIONS: "--experimental-vm-modules" + run: node_modules/.bin/jest --verbose macos: timeout-minutes: 30 # macos-15 ships a newer linker; the older macos-14 linker fails to insert diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 4ac2c3070..585abf5ee 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -180,7 +180,8 @@ jobs: env: CC: clang-18 CXX: clang++-18 - GH_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }} + GH_TOKEN: ${{ github.token }} + SOPHON_READ_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }} steps: - uses: actions/checkout@v6 with: diff --git a/Cargo.lock b/Cargo.lock index 367ba7805..77c7412f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -141,7 +141,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -152,7 +152,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -181,9 +181,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" dependencies = [ "rustversion", ] @@ -1110,9 +1110,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.6.3" +version = "1.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9" +checksum = "0b791f3ac597193fe1d08b82366986eb1f5bc31f2ac6c194c0855276116c76cd" dependencies = [ "base64-simd", "bytes", @@ -1819,7 +1819,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3087,7 +3087,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3315,7 +3315,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3588,8 +3588,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4538,7 +4538,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.3", "system-configuration", "tokio", "tower-service", @@ -4880,7 +4880,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5143,8 +5143,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "arc-swap", "arrow", @@ -5161,6 +5161,7 @@ dependencies = [ "async-trait", "async_cell", "aws-sdk-dynamodb", + "blake3", "byteorder", "bytes", "chrono", @@ -5192,7 +5193,7 @@ dependencies = [ "lance-tokenizer", "log", "moka", - "object_store 0.14.1", + "object_store 0.14.2", "permutation", "pin-project", "prost", @@ -5216,8 +5217,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5239,7 +5240,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5253,7 +5254,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "arrow-array", "arrow-schema", @@ -5262,8 +5263,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "arrayref", "crunchy", @@ -5273,8 +5274,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5293,7 +5294,7 @@ dependencies = [ "log", "moka", "num_cpus", - "object_store 0.14.1", + "object_store 0.14.2", "pin-project", "prost", "quick_cache", @@ -5311,8 +5312,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "arrow", "arrow-array", @@ -5343,8 +5344,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "arrow", "arrow-array", @@ -5361,8 +5362,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "proc-macro2", "quote", @@ -5371,8 +5372,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "arrow-arith", "arrow-array", @@ -5405,8 +5406,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "arrow-arith", "arrow-array", @@ -5427,7 +5428,7 @@ dependencies = [ "lance-io", "log", "num-traits", - "object_store 0.14.1", + "object_store 0.14.2", "prost", "prost-build", "prost-types", @@ -5438,8 +5439,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "datafusion", "geo-traits", @@ -5453,8 +5454,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "arc-swap", "arrow", @@ -5481,6 +5482,7 @@ dependencies = [ "geoarrow-array", "geoarrow-schema", "half", + "hashbrown 0.16.1", "hex", "itertools 0.14.0", "jieba-rs", @@ -5503,7 +5505,7 @@ dependencies = [ "log", "ndarray", "num-traits", - "object_store 0.14.1", + "object_store 0.14.2", "prost", "prost-build", "prost-types", @@ -5524,8 +5526,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "arrow-array", "arrow-schema", @@ -5543,12 +5545,13 @@ dependencies = [ "roaring", "serde", "serde_json", + "tokio", ] [[package]] name = "lance-io" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "arrow", "arrow-array", @@ -5568,7 +5571,7 @@ dependencies = [ "log", "metrics", "moka", - "object_store 0.14.1", + "object_store 0.14.2", "object_store_opendal", "opendal", "opendal-service-goosefs", @@ -5590,8 +5593,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "arrow-array", "arrow-schema", @@ -5605,8 +5608,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "arrow", "async-trait", @@ -5620,9 +5623,10 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ + "arc-swap", "arrow", "arrow-ipc", "arrow-schema", @@ -5643,7 +5647,7 @@ dependencies = [ "lance-namespace", "lance-table", "log", - "object_store 0.14.1", + "object_store 0.14.2", "quick-xml 0.40.1", "rand 0.9.5", "reqwest 0.12.28", @@ -5660,9 +5664,9 @@ dependencies = [ [[package]] name = "lance-namespace-reqwest-client" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8d23e54b1634d5bbb434f8dd33dc3c05f6e58d876a9a27b3b4aef58ddbe11af" +checksum = "828f291764feda98898452bea4ce3756b73e5d5c7c67b62edda27456b7cc72c1" dependencies = [ "reqwest 0.12.28", "serde", @@ -5674,8 +5678,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5689,8 +5693,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "arrow", "arrow-array", @@ -5711,7 +5715,7 @@ dependencies = [ "lance-io", "lance-select", "log", - "object_store 0.14.1", + "object_store 0.14.2", "prost", "prost-build", "prost-types", @@ -5730,8 +5734,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "arrow-array", "arrow-schema", @@ -5744,8 +5748,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "13.0.0-beta.12" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f" +version = "13.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d" dependencies = [ "frostem", "icu_segmenter", @@ -5758,7 +5762,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.40.0-beta.11" +version = "0.40.0-beta.12" dependencies = [ "ahash", "anyhow", @@ -5822,7 +5826,7 @@ dependencies = [ "moka", "num-traits", "oauth2", - "object_store 0.14.1", + "object_store 0.14.2", "pin-project", "polars", "polars-arrow", @@ -5855,7 +5859,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.40.0-beta.11" +version = "0.40.0-beta.12" dependencies = [ "arrow-array", "arrow-buffer", @@ -5880,7 +5884,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.40.0-beta.11" +version = "0.40.0-beta.12" dependencies = [ "arc-swap", "arrow", @@ -6471,9 +6475,9 @@ dependencies = [ [[package]] name = "napi" -version = "3.12.2" +version = "3.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58c5f4d5375213fdb7be2655e152386e82f026f9a5ba36a75556e11359aafe09" +checksum = "f0c007d4a8ead952a81887661d41fd56b7e7a70d3d4d921f445fb37a8f6efa1e" dependencies = [ "bitflags 2.11.1", "chrono", @@ -6491,15 +6495,15 @@ dependencies = [ [[package]] name = "napi-build" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "860e7c40864f95cfb83cde99f9ebadd88ef3d9bdccd7dd2cee0cc96a2dd4ffa7" +checksum = "0941983559cd62bfa7f04646a16428967a847365adeef27711e4e3927046bbed" [[package]] name = "napi-derive" -version = "3.6.5" +version = "3.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350057056a30368aa76c11a0d406b0aa61321710be2c36656ab4f6efe0b785a2" +checksum = "6b1d232c24dc6754979b27efdf2047a5849efc7c92c1245de87e6bedbb9f5c3c" dependencies = [ "convert_case", "ctor", @@ -6511,9 +6515,9 @@ dependencies = [ [[package]] name = "napi-derive-backend" -version = "6.1.3" +version = "6.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c1c87a71568f3fe5c736b10878ff55064b250bb4ea4b48c8055713e07b9e463" +checksum = "fd6856fe1686867da90ed0dc292d7c0c87a695eb7c578b913360f33619ee01e3" dependencies = [ "convert_case", "proc-macro2", @@ -6524,9 +6528,9 @@ dependencies = [ [[package]] name = "napi-sys" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85fbf1fa9f1babfe396d74bbbf52b3643770243e8f5b0b46715d4caf7f0dfc9a" +checksum = "e22a4f25c16a5c5411d987cd6fbd48313522ae1789ade2d6dd3efdc6d40a0fc8" dependencies = [ "libloading", ] @@ -6633,7 +6637,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6762,7 +6766,7 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "chrono", "getrandom 0.2.17", "http 1.5.0", @@ -6879,13 +6883,13 @@ dependencies = [ [[package]] name = "object_store" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50" +checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94" dependencies = [ "async-trait", "aws-lc-rs", - "base64 0.22.1", + "base64 0.23.1", "bytes", "chrono", "crc-fast", @@ -6931,7 +6935,7 @@ dependencies = [ "bytes", "chrono", "futures", - "object_store 0.14.1", + "object_store 0.14.2", "opendal", "pin-project", "tokio", @@ -8136,8 +8140,8 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "heck 0.4.1", - "itertools 0.11.0", + "heck 0.5.0", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -8156,7 +8160,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", @@ -8378,7 +8382,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls 0.23.45", - "socket2 0.5.10", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -8416,7 +8420,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] @@ -9216,7 +9220,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -9287,7 +9291,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -9578,9 +9582,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.23.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "935177bb8c0cd8ca1a4e6d1a2ac8988bea69cab4f9d3a31311e012ad27868ea4" +checksum = "df9adc193c780ef8f159aee8b61e2d5801aaa555e6eb0947fe45530ec506296f" dependencies = [ "base64 0.23.1", "bs58", @@ -9599,9 +9603,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.23.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc" +checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -9834,7 +9838,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.117", @@ -9846,7 +9850,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.117", @@ -9869,7 +9873,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -9988,7 +9992,7 @@ dependencies = [ "cfg-if 1.0.4", "libc", "psm", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -10280,7 +10284,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -11301,7 +11305,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c64305048..183e5f4f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=13.0.0-beta.12", default-features = false, "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=13.0.0-beta.12", default-features = false, "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=13.0.0-beta.12", default-features = false, "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=13.1.0-beta.1", default-features = false, "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=13.1.0-beta.1", default-features = false, "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=13.1.0-beta.1", default-features = false, "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/docs/src/java/java.md b/docs/src/java/java.md index 5946e5862..0565156dc 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.40.0-beta.11 + 0.40.0-beta.12 ``` diff --git a/docs/src/js/classes/AutoQuery.md b/docs/src/js/classes/AutoQuery.md index 1d7ea6952..12f517dbd 100644 --- a/docs/src/js/classes/AutoQuery.md +++ b/docs/src/js/classes/AutoQuery.md @@ -350,7 +350,7 @@ input to this method would be: #### Parameters -* **columns**: `string` \| `string`[] \| `Record`<`string`, `string`> \| `Map`<`string`, `string`> +* **columns**: `string` \| `string`[] \| `Map`<`string`, `string`> \| `Record`<`string`, `string`> #### Returns @@ -417,6 +417,8 @@ Collect the results as an Arrow ArrowTable. +Empty results retain the query's output schema. + #### Inherited from `StandardQueryBase.toArrow` diff --git a/docs/src/js/classes/BlobFile.md b/docs/src/js/classes/BlobFile.md index 84a596d1a..ee012be25 100644 --- a/docs/src/js/classes/BlobFile.md +++ b/docs/src/js/classes/BlobFile.md @@ -10,16 +10,52 @@ A lazy handle to blob bytes. Create one with [Table.fetchBlobFiles](Table.md#fet ## Methods +### close() + +```ts +close(): Promise +``` + +Releases the handle. Reads after `close()` fail. Calling it again does +nothing. + +#### Returns + +`Promise`<`void`> + +*** + +### isClosed() + +```ts +isClosed(): boolean +``` + +Returns true after [BlobFile.close](BlobFile.md#close). + +#### Returns + +`boolean` + +*** + ### read() ```ts -read(): Promise +read(maxBytes?, options?): Promise ``` -Reads from the cursor to the end and advances the cursor. +Reads from the cursor and advances the cursor. -A second call returns an empty buffer. [BlobFile.readRange](BlobFile.md#readrange) does -not move the cursor. +Reads to the end when `maxBytes` is omitted, or at most `maxBytes` bytes +otherwise. Returns an empty buffer at the end of the blob. +[BlobFile.readRange](BlobFile.md#readrange) does not move the cursor. + +#### Parameters + +* **maxBytes?**: `bigint` + +* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md) #### Returns @@ -30,7 +66,10 @@ not move the cursor. ### readRange() ```ts -readRange(start, end): Promise +readRange( + start, + end, + options?): Promise ``` Reads the half-open byte range `[start, end)`. @@ -43,12 +82,55 @@ Fails when `end` is past the blob size. Does not move the cursor. * **end**: `bigint` +* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md) + #### Returns `Promise`<`Buffer`> *** +### readRanges() + +```ts +readRanges(ranges, options?): Promise +``` + +Reads several half-open byte ranges. Returns one buffer per range, in +the order given. + +Fails when any `end` is past the blob size. Does not move the cursor. + +#### Parameters + +* **ranges**: [`BlobRange`](../type-aliases/BlobRange.md)[] + +* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md) + +#### Returns + +`Promise`<`Buffer`[]> + +*** + +### seek() + +```ts +seek(position): Promise +``` + +Moves the cursor to `position`, in bytes from the start of the blob. + +#### Parameters + +* **position**: `bigint` + +#### Returns + +`Promise`<`void`> + +*** + ### size() ```ts @@ -60,3 +142,17 @@ Returns the blob size in bytes. #### Returns `bigint` + +*** + +### tell() + +```ts +tell(): Promise +``` + +Returns the cursor position in bytes. + +#### Returns + +`Promise`<`bigint`> diff --git a/docs/src/js/classes/Query.md b/docs/src/js/classes/Query.md index 6ebaebd75..3038bf2b3 100644 --- a/docs/src/js/classes/Query.md +++ b/docs/src/js/classes/Query.md @@ -424,7 +424,7 @@ input to this method would be: #### Parameters -* **columns**: `string` \| `string`[] \| `Record`<`string`, `string`> \| `Map`<`string`, `string`> +* **columns**: `string` \| `string`[] \| `Map`<`string`, `string`> \| `Record`<`string`, `string`> #### Returns @@ -491,6 +491,8 @@ Collect the results as an Arrow ArrowTable. +Empty results retain the query's output schema. + #### Inherited from `StandardQueryBase.toArrow` diff --git a/docs/src/js/classes/QueryBase.md b/docs/src/js/classes/QueryBase.md index 35c071525..0ef4fa286 100644 --- a/docs/src/js/classes/QueryBase.md +++ b/docs/src/js/classes/QueryBase.md @@ -193,7 +193,7 @@ input to this method would be: #### Parameters -* **columns**: `string` \| `string`[] \| `Record`<`string`, `string`> \| `Map`<`string`, `string`> +* **columns**: `string` \| `string`[] \| `Map`<`string`, `string`> \| `Record`<`string`, `string`> #### Returns @@ -252,6 +252,8 @@ Collect the results as an Arrow ArrowTable. +Empty results retain the query's output schema. + *** ### withRowId() diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index a8c382c27..ac16b8a81 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -516,7 +516,10 @@ Drop an index from the table. ### fetchBlobFiles() ```ts -abstract fetchBlobFiles(column, rowIds): Promise<(null | BlobFile)[]> +abstract fetchBlobFiles( + column, + rowIds, + options?): Promise<(null | BlobFile)[]> ``` Opens lazy blob handles for `column` at the given row IDs using the @@ -531,6 +534,8 @@ See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions. * **rowIds**: readonly (`number` \| `bigint`)[] +* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md) + #### Returns `Promise`<(`null` \| [`BlobFile`](BlobFile.md))[]> @@ -540,7 +545,10 @@ See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions. ### fetchBlobs() ```ts -abstract fetchBlobs(column, rowIds): Promise<(null | Buffer)[]> +abstract fetchBlobs( + column, + rowIds, + options?): Promise<(null | Buffer)[]> ``` Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid). @@ -548,6 +556,10 @@ Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid). Reads the table's current checkout. IDs from another version can fail after compaction unless stable row ids are enabled. Results keep input order and duplicates. Null blobs are `null`. Empty blobs are empty buffers. +Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes. +The client splits requests automatically and reads an individual larger +blob through the Range route. This method still materializes all bytes in +memory; use [Table.fetchBlobFiles](Table.md#fetchblobfiles) for large values. #### Parameters @@ -555,6 +567,8 @@ duplicates. Null blobs are `null`. Empty blobs are empty buffers. * **rowIds**: readonly (`number` \| `bigint`)[] +* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md) + #### Returns `Promise`<(`null` \| `Buffer`)[]> @@ -1367,7 +1381,7 @@ repeatedly calling this method. ##### Parameters -* **updates**: `Record`<`string`, `string`> \| `Map`<`string`, `string`> +* **updates**: `Map`<`string`, `string`> \| `Record`<`string`, `string`> the columns to update diff --git a/docs/src/js/classes/TakeQuery.md b/docs/src/js/classes/TakeQuery.md index 6ae2f9c8c..f3d59a2a3 100644 --- a/docs/src/js/classes/TakeQuery.md +++ b/docs/src/js/classes/TakeQuery.md @@ -200,7 +200,7 @@ input to this method would be: #### Parameters -* **columns**: `string` \| `string`[] \| `Record`<`string`, `string`> \| `Map`<`string`, `string`> +* **columns**: `string` \| `string`[] \| `Map`<`string`, `string`> \| `Record`<`string`, `string`> #### Returns @@ -267,6 +267,8 @@ Collect the results as an Arrow ArrowTable. +Empty results retain the query's output schema. + #### Inherited from [`QueryBase`](QueryBase.md).[`toArrow`](QueryBase.md#toarrow) diff --git a/docs/src/js/classes/VectorQuery.md b/docs/src/js/classes/VectorQuery.md index f9412c76d..a0a77fd7c 100644 --- a/docs/src/js/classes/VectorQuery.md +++ b/docs/src/js/classes/VectorQuery.md @@ -447,6 +447,8 @@ nprobes(nprobes): VectorQuery Set the number of partitions to search (probe) +The number of probes must be greater than 0. + This argument is only used when the vector column has an IVF PQ index. If there is no index then this value is ignored. @@ -673,7 +675,7 @@ input to this method would be: #### Parameters -* **columns**: `string` \| `string`[] \| `Record`<`string`, `string`> \| `Map`<`string`, `string`> +* **columns**: `string` \| `string`[] \| `Map`<`string`, `string`> \| `Record`<`string`, `string`> #### Returns @@ -740,6 +742,8 @@ Collect the results as an Arrow ArrowTable. +Empty results retain the query's output schema. + #### Inherited from `StandardQueryBase.toArrow` diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 2f49578b1..edb0bacc3 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -158,8 +158,11 @@ - [AnalyzePlanDistributedMetrics](type-aliases/AnalyzePlanDistributedMetrics.md) - [BaseTokenizer](type-aliases/BaseTokenizer.md) - [BlobOptions](type-aliases/BlobOptions.md) +- [BlobRange](type-aliases/BlobRange.md) +- [BlobReadOptions](type-aliases/BlobReadOptions.md) - [Data](type-aliases/Data.md) - [DataLike](type-aliases/DataLike.md) +- [DataTypeLike](type-aliases/DataTypeLike.md) - [FieldLike](type-aliases/FieldLike.md) - [IntoSql](type-aliases/IntoSql.md) - [IntoVector](type-aliases/IntoVector.md) diff --git a/docs/src/js/interfaces/AddDataOptions.md b/docs/src/js/interfaces/AddDataOptions.md index 6976787f0..eb10fdb0f 100644 --- a/docs/src/js/interfaces/AddDataOptions.md +++ b/docs/src/js/interfaces/AddDataOptions.md @@ -10,6 +10,22 @@ Options for adding data to a table. ## Properties +### allowExternalBlobOutsideBases? + +```ts +optional allowExternalBlobOutsideBases: boolean; +``` + +Whether blob URIs outside registered external bases may be written. + +Defaults to `false`. This option is supported only for local/native +tables; remote tables return an error when it is enabled. An enabled write +stores the absolute URI reference without registering a base or copying +the external object into the database. The object must remain accessible +when the blob is read later. + +*** + ### mode ```ts diff --git a/docs/src/js/interfaces/CherryPickResult.md b/docs/src/js/interfaces/CherryPickResult.md index c837ab143..fdec8eedd 100644 --- a/docs/src/js/interfaces/CherryPickResult.md +++ b/docs/src/js/interfaces/CherryPickResult.md @@ -38,9 +38,9 @@ preview: CherryPickPreview; ```ts status: - | "failed" - | "unknown" | "ready" + | "failed" | "notImplemented" - | "cherryPicked"; + | "cherryPicked" + | "unknown"; ``` diff --git a/docs/src/js/interfaces/DropNamespaceOptions.md b/docs/src/js/interfaces/DropNamespaceOptions.md index 1863bb0cd..6e5f01d34 100644 --- a/docs/src/js/interfaces/DropNamespaceOptions.md +++ b/docs/src/js/interfaces/DropNamespaceOptions.md @@ -21,7 +21,7 @@ Refuse to drop if non-empty (restrict) or drop recursively (cascade). ### mode? ```ts -optional mode: "fail" | "skip"; +optional mode: "skip" | "fail"; ``` Whether to skip if the namespace doesn't exist, or fail. diff --git a/docs/src/js/namespaces/embedding/classes/EmbeddingFunction.md b/docs/src/js/namespaces/embedding/classes/EmbeddingFunction.md index 574a0d71f..b6291e0be 100644 --- a/docs/src/js/namespaces/embedding/classes/EmbeddingFunction.md +++ b/docs/src/js/namespaces/embedding/classes/EmbeddingFunction.md @@ -52,7 +52,7 @@ new EmbeddingFunction(): EmbeddingFunction ### computeQueryEmbeddings() ```ts -computeQueryEmbeddings(data): Promise +computeQueryEmbeddings(data): Promise ``` Compute the embeddings for a single query @@ -63,7 +63,7 @@ Compute the embeddings for a single query #### Returns -`Promise`<`number`[] \| `Uint8Array` \| `Float32Array` \| `Float64Array`> +`Promise`<`Float32Array` \| `Float64Array` \| `Uint8Array` \| `number`[]> *** diff --git a/docs/src/js/namespaces/embedding/classes/TextEmbeddingFunction.md b/docs/src/js/namespaces/embedding/classes/TextEmbeddingFunction.md index 444c4c3f0..0a0cc619d 100644 --- a/docs/src/js/namespaces/embedding/classes/TextEmbeddingFunction.md +++ b/docs/src/js/namespaces/embedding/classes/TextEmbeddingFunction.md @@ -37,7 +37,7 @@ new TextEmbeddingFunction(): TextEmbeddingFunction ### computeQueryEmbeddings() ```ts -computeQueryEmbeddings(data): Promise +computeQueryEmbeddings(data): Promise ``` Compute the embeddings for a single query @@ -48,7 +48,7 @@ Compute the embeddings for a single query #### Returns -`Promise`<`number`[] \| `Uint8Array` \| `Float32Array` \| `Float64Array`> +`Promise`<`Float32Array` \| `Float64Array` \| `Uint8Array` \| `number`[]> #### Overrides diff --git a/docs/src/js/type-aliases/BlobRange.md b/docs/src/js/type-aliases/BlobRange.md new file mode 100644 index 000000000..85cd48242 --- /dev/null +++ b/docs/src/js/type-aliases/BlobRange.md @@ -0,0 +1,27 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / BlobRange + +# Type Alias: BlobRange + +```ts +type BlobRange: object; +``` + +A half-open byte range `[start, end)` for [BlobFile.readRanges](../classes/BlobFile.md#readranges). + +## Type declaration + +### end + +```ts +end: bigint; +``` + +### start + +```ts +start: bigint; +``` diff --git a/docs/src/js/type-aliases/BlobReadOptions.md b/docs/src/js/type-aliases/BlobReadOptions.md new file mode 100644 index 000000000..17264b0e4 --- /dev/null +++ b/docs/src/js/type-aliases/BlobReadOptions.md @@ -0,0 +1,24 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / BlobReadOptions + +# Type Alias: BlobReadOptions + +```ts +type BlobReadOptions: object; +``` + +Options for blob reads. + +## Type declaration + +### signal? + +```ts +optional signal: AbortSignal; +``` + +Cancels the read. The call rejects with `signal.reason`, and the native +read stops, including in-flight requests to a remote table. diff --git a/docs/src/js/type-aliases/DataTypeLike.md b/docs/src/js/type-aliases/DataTypeLike.md new file mode 100644 index 000000000..1f0ee6b81 --- /dev/null +++ b/docs/src/js/type-aliases/DataTypeLike.md @@ -0,0 +1,18 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / DataTypeLike + +# Type Alias: DataTypeLike + +```ts +type DataTypeLike: DataType | object; +``` + +A `DataType` from any copy or version of apache-arrow. + +Arrow 21 brands its classes with `unique symbol` properties, so a type +object from a second copy of the library no longer satisfies the `DataType` +type of this one even though it is structurally identical. Inputs that only +need to be sanitized accept this looser shape instead. diff --git a/docs/src/python/python.md b/docs/src/python/python.md index ccc44a55e..66fda4378 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -198,6 +198,33 @@ listing a storage directory. ::: lancedb.materialized_view.MaterializedViewDefinition +::: lancedb.vector_dedup + +::: lancedb.materialized_view.VectorDedupSource + +For an existing indexed source, declare and materialize a dedup result without +writing SQL. Creation captures the source version once and waits for the complete +result; remote connections use the service's MV jobs. + +```python +from lancedb import vector_dedup + +view = db.create_materialized_view( + "images_clean", + vector_dedup("images", column="phash", distance_threshold=4), +) +cleaned = view.table +``` + +To submit without waiting, call `db.create_materialized_view_async(...)` with the +same source expression and wait on the returned job. To declare only, pass +`with_no_data=True`, then use `view.refresh_async().wait()` when ready. Pass +`dataset_version=...` to `vector_dedup` for an explicit snapshot. The default +policy retains direct representatives (A-B-C without A-C keeps A and C), preserves +the source, and materializes its original columns. Only within-segment/partition +pairs are covered, using the index's distance representation. + + ## Views ::: lancedb.view.ViewDescription diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index b7cdb9e41..77635c74e 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.40.0-beta.11 + 0.40.0-beta.12 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 7fe65841f..9d7249f92 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.40.0-beta.11 + 0.40.0-beta.12 pom ${project.artifactId} LanceDB Java SDK Parent POM @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 13.0.0-beta.12 + 13.1.0-beta.1 false 2.30.0 1.7 diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 87d4973d2..051e16474 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.40.0-beta.11" +version = "0.40.0-beta.12" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/NODEJS_THIRD_PARTY_LICENSES.md b/nodejs/NODEJS_THIRD_PARTY_LICENSES.md index c0881f61e..550d2888c 100644 --- a/nodejs/NODEJS_THIRD_PARTY_LICENSES.md +++ b/nodejs/NODEJS_THIRD_PARTY_LICENSES.md @@ -275,6 +275,7 @@ [@types/node@20.16.10](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT [@types/node@20.17.9](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT [@types/node@22.7.4](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT +[@types/node@25.9.8](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT [@types/semver@7.5.6](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT [@types/stack-utils@2.0.3](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT [@types/tmp@0.2.6](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT @@ -306,6 +307,9 @@ [apache-arrow@16.0.0](https://github.com/apache/arrow) - Apache-2.0 [apache-arrow@17.0.0](https://github.com/apache/arrow) - Apache-2.0 [apache-arrow@18.0.0](https://github.com/apache/arrow) - Apache-2.0 +[apache-arrow@19.0.1](https://github.com/apache/arrow) - Apache-2.0 +[apache-arrow@20.0.0](https://github.com/apache/arrow) - Apache-2.0 +[apache-arrow@21.2.0](https://github.com/apache/arrow-js) - Apache-2.0 [argparse@1.0.10](https://github.com/nodeca/argparse) - MIT [argparse@2.0.1](https://github.com/nodeca/argparse) - Python-2.0 [array-back@3.1.0](https://github.com/75lb/array-back) - MIT @@ -348,6 +352,7 @@ [color@4.2.3](https://github.com/Qix-/color) - MIT [combined-stream@1.0.8](https://github.com/felixge/node-combined-stream) - MIT [command-line-args@5.2.1](https://github.com/75lb/command-line-args) - MIT +[command-line-args@6.0.2](https://github.com/75lb/command-line-args) - MIT [command-line-usage@7.0.1](https://github.com/75lb/command-line-usage) - MIT [concat-map@0.0.1](https://github.com/substack/node-concat-map) - MIT [convert-source-map@2.0.0](https://github.com/thlorenz/convert-source-map) - MIT @@ -398,12 +403,14 @@ [file-entry-cache@6.0.1](https://github.com/royriojas/file-entry-cache) - MIT [fill-range@7.1.1](https://github.com/jonschlinkert/fill-range) - MIT [find-replace@3.0.0](https://github.com/75lb/find-replace) - MIT +[find-replace@5.0.2](https://github.com/75lb/find-replace) - MIT [find-up@4.1.0](https://github.com/sindresorhus/find-up) - MIT [find-up@5.0.0](https://github.com/sindresorhus/find-up) - MIT [flat-cache@3.2.0](https://github.com/jaredwray/flat-cache) - MIT [flatbuffers@1.12.0](https://github.com/google/flatbuffers) - Apache* [flatbuffers@23.5.26](https://github.com/google/flatbuffers) - Apache* [flatbuffers@24.3.25](https://github.com/google/flatbuffers) - Apache-2.0 +[flatbuffers@25.9.23](https://github.com/google/flatbuffers) - Apache-2.0 [flatted@3.2.9](https://github.com/WebReflection/flatted) - ISC [follow-redirects@1.15.6](https://github.com/follow-redirects/follow-redirects) - MIT [foreground-child@3.3.0](https://github.com/tapjs/foreground-child) - ISC @@ -644,6 +651,7 @@ [uc.micro@2.1.0](https://github.com/markdown-it/uc.micro) - MIT [undici-types@5.26.5](https://github.com/nodejs/undici) - MIT [undici-types@6.19.8](https://github.com/nodejs/undici) - MIT +[undici-types@7.24.6](https://github.com/nodejs/undici) - MIT [update-browserslist-db@1.0.13](https://github.com/browserslist/update-db) - MIT [uri-js@4.4.1](https://github.com/garycourt/uri-js) - BSD-2-Clause [uuid@9.0.1](https://github.com/uuidjs/uuid) - MIT diff --git a/nodejs/__test__/arrow.test.ts b/nodejs/__test__/arrow.test.ts index df02b4936..e054a5495 100644 --- a/nodejs/__test__/arrow.test.ts +++ b/nodejs/__test__/arrow.test.ts @@ -2,10 +2,8 @@ // SPDX-FileCopyrightText: Copyright The LanceDB Authors import * as fs from "node:fs"; import * as vm from "node:vm"; -import * as arrow15 from "apache-arrow-15"; -import * as arrow16 from "apache-arrow-16"; -import * as arrow17 from "apache-arrow-17"; -import * as arrow18 from "apache-arrow-18"; + +import { type ApacheArrow, arrow15, arrowVersions } from "./arrow_versions"; import { Field as CurrentField, @@ -110,1541 +108,1502 @@ it("preserves field metadata from a provided schema", async function () { expect(roundTripped.schema.fields[0].metadata).toEqual(jsonMetadata); }); -describe.each([arrow15, arrow16, arrow17, arrow18])( - "Arrow", - ( - arrow: typeof arrow15 | typeof arrow16 | typeof arrow17 | typeof arrow18, - ) => { - type ApacheArrow = - | typeof arrow15 - | typeof arrow16 - | typeof arrow17 - | typeof arrow18; - const { - Schema, - Field, - Binary, - Bool, - Utf8, - Float64, - Struct, - List, - Map_, - Int16, - Int32, - Int64, - Float, - Float16, - Float32, - FixedSizeList, - Precision, - tableFromIPC, - DataType, - Dictionary, - RecordBatch: ArrowRecordBatch, - Table: ArrowTable, - Uint8: ArrowUint8, - makeData: arrowMakeData, - vectorFromArray, +describe.each(arrowVersions)("Arrow %#", (arrow: ApacheArrow) => { + const { + Schema, + Field, + Binary, + Bool, + Utf8, + Float64, + Struct, + List, + Map_, + Int16, + Int32, + Int64, + Float, + Float16, + Float32, + FixedSizeList, + Precision, + tableFromIPC, + DataType, + Dictionary, + RecordBatch: ArrowRecordBatch, + Table: ArrowTable, + Uint8: ArrowUint8, + makeData: arrowMakeData, + vectorFromArray, + // biome-ignore lint/suspicious/noExplicitAny: + } = arrow; + type Schema = ApacheArrow["Schema"]; + type Table = ApacheArrow["Table"]; + + function expectValidMapField( + // biome-ignore lint/suspicious/noExplicitAny: Arrow Field types vary across supported versions + field: any, + ): void { + expect(DataType.isMap(field.type)).toBe(true); + expect(field.type.keysSorted).toBe(true); + expect(field.type.children).toHaveLength(1); + + const entries = field.type.children[0]; + expect(entries.name).toBe("entries"); + expect(entries.nullable).toBe(false); + expect(DataType.isStruct(entries.type)).toBe(true); + expect(entries.type.children).toHaveLength(2); + + const [key, value] = entries.type.children; + expect([key.name, value.name]).toEqual(["key", "value"]); + expect(key.nullable).toBe(false); + expect(DataType.isUtf8(key.type)).toBe(true); + expect(value.nullable).toBe(true); + expect(DataType.isInt(value.type)).toBe(true); + expect(value.type.bitWidth).toBe(32); + expect(value.type.isSigned).toBe(true); + } + + // Helper method to verify various ways to create a table + async function checkTableCreation( + tableCreationMethod: ( + records: Record[], + recordsReversed: Record[], + schema: Schema, + ) => Promise, + infersTypes: boolean, + ): Promise { + const records = sampleRecords(); + const recordsReversed = [ + { + list: ["anime", "action", "comedy"], + struct: { x: 0, y: 0 }, + string: "hello", + number: 7, + boolean: false, + binary: Buffer.alloc(5), + }, + ]; + const schema = new Schema([ + new Field("binary", new Binary(), false), + new Field("boolean", new Bool(), false), + new Field("number", new Float64(), false), + new Field("string", new Utf8(), false), + new Field( + "struct", + new Struct([ + new Field("x", new Float64(), false), + new Field("y", new Float64(), false), + ]), + ), + new Field("list", new List(new Field("item", new Utf8(), false)), false), + ]); + const table = (await tableCreationMethod( + records, + recordsReversed, + schema, // biome-ignore lint/suspicious/noExplicitAny: - } = arrow; - type Schema = ApacheArrow["Schema"]; - type Table = ApacheArrow["Table"]; + )) as any; - function expectValidMapField( - // biome-ignore lint/suspicious/noExplicitAny: Arrow Field types vary across supported versions - field: any, - ): void { - expect(DataType.isMap(field.type)).toBe(true); - expect(field.type.keysSorted).toBe(true); - expect(field.type.children).toHaveLength(1); + // We expect deterministic ordering of the fields + expect(table.schema.names).toEqual(schema.names); - const entries = field.type.children[0]; - expect(entries.name).toBe("entries"); - expect(entries.nullable).toBe(false); - expect(DataType.isStruct(entries.type)).toBe(true); - expect(entries.type.children).toHaveLength(2); + schema.fields.forEach( + ( + // biome-ignore lint/suspicious/noExplicitAny: + field: { name: any; type: { toString: () => any } }, + idx: string | number, + ) => { + const actualField = table.schema.fields[idx]; + // Type inference always assumes nullable=true + if (infersTypes) { + expect(actualField.nullable).toBe(true); + } else { + expect(actualField.nullable).toBe(false); + } + expect(table.getChild(field.name)?.type.toString()).toEqual( + field.type.toString(), + ); + expect(table.getChildAt(idx)?.type.toString()).toEqual( + field.type.toString(), + ); + }, + ); + } - const [key, value] = entries.type.children; - expect([key.name, value.name]).toEqual(["key", "value"]); - expect(key.nullable).toBe(false); - expect(DataType.isUtf8(key.type)).toBe(true); - expect(value.nullable).toBe(true); - expect(DataType.isInt(value.type)).toBe(true); - expect(value.type.bitWidth).toBe(32); - expect(value.type.isSigned).toBe(true); - } - - // Helper method to verify various ways to create a table - async function checkTableCreation( - tableCreationMethod: ( - records: Record[], - recordsReversed: Record[], - schema: Schema, - ) => Promise
, - infersTypes: boolean, - ): Promise { - const records = sampleRecords(); - const recordsReversed = [ - { - list: ["anime", "action", "comedy"], - struct: { x: 0, y: 0 }, - string: "hello", - number: 7, - boolean: false, - binary: Buffer.alloc(5), - }, + describe("The function makeArrowTable", function () { + it("accepts snake_case embedding metadata like camelCase", function () { + const spellings = [ + // biome-ignore lint/style/useNamingConvention: the Python wire spelling + { source_column: "text", vector_column: "vector" }, + { sourceColumn: "text", vectorColumn: "vector" }, ]; - const schema = new Schema([ - new Field("binary", new Binary(), false), - new Field("boolean", new Bool(), false), - new Field("number", new Float64(), false), - new Field("string", new Utf8(), false), - new Field( - "struct", - new Struct([ - new Field("x", new Float64(), false), - new Field("y", new Float64(), false), + for (const columns of spellings) { + const schema = new Schema( + [ + new Field("text", new Utf8(), false), + new Field( + "vector", + new FixedSizeList(3, new Field("item", new Float32(), true)), + false, + ), + ], + new Map([ + [ + "embedding_functions", + JSON.stringify([{ name: "mock", model: {}, ...columns }]), + ], ]), - ), + ); + // The vector field is non-nullable and absent from the data; only a + // recognized embedding config makes that acceptable. + const table = makeArrowTable([{ text: "hello" }], { schema }); + expect(table.numRows).toBe(1); + } + }); + + it("will use data types from a provided schema instead of inference", async function () { + const schema = new Schema([ + new Field("a", new Int32(), false), + new Field("b", new Float32(), true), + new Field("c", new FixedSizeList(3, new Field("item", new Float16()))), + new Field("d", new Int64(), true), + ]); + const table = makeArrowTable( + [ + { a: 1, b: 2, c: [1, 2, 3], d: 9 }, + { a: 4, b: 5, c: [4, 5, 6], d: 10 }, + { a: 7, b: 8, c: [7, 8, 9], d: null }, + ], + { schema }, + ); + + const buf = await fromTableToBuffer(table); + expect(buf.byteLength).toBeGreaterThan(0); + + const actual = tableFromIPC(buf); + expect(actual.numRows).toBe(3); + const actualSchema = actual.schema; + expect(actualSchema).toEqual(schema); + expect(table.getChild("a")?.toJSON()).toEqual([1, 4, 7]); + expect(table.getChild("b")?.toJSON()).toEqual([2, 5, 8]); + expect(table.getChild("d")?.toJSON()).toEqual([9n, 10n, null]); + }); + + it("will use a provided FixedSizeList schema with typed array values", function () { + const schema = new Schema([ + new Field("text", new Utf8(), false), new Field( - "list", - new List(new Field("item", new Utf8(), false)), + "vector", + new FixedSizeList(3, new Field("item", new Float32(), false)), false, ), ]); - const table = (await tableCreationMethod( - records, - recordsReversed, - schema, - // biome-ignore lint/suspicious/noExplicitAny: - )) as any; - // We expect deterministic ordering of the fields - expect(table.schema.names).toEqual(schema.names); + const table = makeArrowTable( + [ + { + text: "foo", + vector: new Float32Array([1, 2, 3]), + }, + ], + { schema }, + ); - schema.fields.forEach( - ( - // biome-ignore lint/suspicious/noExplicitAny: - field: { name: any; type: { toString: () => any } }, - idx: string | number, - ) => { - const actualField = table.schema.fields[idx]; - // Type inference always assumes nullable=true - if (infersTypes) { - expect(actualField.nullable).toBe(true); - } else { - expect(actualField.nullable).toBe(false); - } - expect(table.getChild(field.name)?.type.toString()).toEqual( - field.type.toString(), - ); - expect(table.getChildAt(idx)?.type.toString()).toEqual( - field.type.toString(), - ); + expect(table.getChild("text")?.toJSON()).toEqual(["foo"]); + expect( + table + .getChild("vector") + ?.toJSON() + .map((value) => value.toJSON()), + ).toEqual([[1, 2, 3]]); + }); + + it("will assume the column `vector` is FixedSizeList by default", async function () { + const schema = new Schema([ + new Field("a", new Float(Precision.DOUBLE), true), + new Field("b", new Int64(), true), + new Field( + "vector", + new FixedSizeList( + 3, + new Field("item", new Float(Precision.SINGLE), true), + ), + true, + ), + ]); + const table = makeArrowTable([ + { a: 1, b: 2n, vector: [1, 2, 3] }, + { a: 4, b: 5n, vector: [4, 5, 6] }, + { a: 7, b: 8n, vector: [7, 8, 9] }, + ]); + + const buf = await fromTableToBuffer(table); + expect(buf.byteLength).toBeGreaterThan(0); + + const actual = tableFromIPC(buf); + expect(actual.numRows).toBe(3); + const actualSchema = actual.schema; + expect(actualSchema).toEqual(schema); + + expect(table.getChild("a")?.toJSON()).toEqual([1, 4, 7]); + expect(table.getChild("b")?.toJSON()).toEqual([2n, 5n, 8n]); + expect( + table + .getChild("vector") + ?.toJSON() + .map((v) => v.toJSON()), + ).toEqual([ + [1, 2, 3], + [4, 5, 6], + [7, 8, 9], + ]); + }); + + it("can support multiple vector columns", async function () { + const schema = new Schema([ + new Field("a", new Float(Precision.DOUBLE), true), + new Field("b", new Float(Precision.DOUBLE), true), + new Field( + "vec1", + new FixedSizeList(3, new Field("item", new Float16(), true)), + true, + ), + new Field( + "vec2", + new FixedSizeList(3, new Field("item", new Float64(), true)), + true, + ), + ]); + const table = makeArrowTable( + [ + { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] }, + { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] }, + { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] }, + ], + { + vectorColumns: { + vec1: { type: new Float16() }, + vec2: { type: new Float64() }, + }, }, ); - } - describe("The function makeArrowTable", function () { - it("accepts snake_case embedding metadata like camelCase", function () { - const spellings = [ - // biome-ignore lint/style/useNamingConvention: the Python wire spelling - { source_column: "text", vector_column: "vector" }, - { sourceColumn: "text", vectorColumn: "vector" }, - ]; - for (const columns of spellings) { - const schema = new Schema( - [ - new Field("text", new Utf8(), false), - new Field( - "vector", - new FixedSizeList(3, new Field("item", new Float32(), true)), - false, - ), - ], - new Map([ - [ - "embedding_functions", - JSON.stringify([{ name: "mock", model: {}, ...columns }]), - ], - ]), - ); - // The vector field is non-nullable and absent from the data; only a - // recognized embedding config makes that acceptable. - const table = makeArrowTable([{ text: "hello" }], { schema }); - expect(table.numRows).toBe(1); - } - }); + const buf = await fromTableToBuffer(table); + expect(buf.byteLength).toBeGreaterThan(0); - it("will use data types from a provided schema instead of inference", async function () { - const schema = new Schema([ - new Field("a", new Int32(), false), - new Field("b", new Float32(), true), - new Field( - "c", - new FixedSizeList(3, new Field("item", new Float16())), - ), - new Field("d", new Int64(), true), - ]); - const table = makeArrowTable( - [ - { a: 1, b: 2, c: [1, 2, 3], d: 9 }, - { a: 4, b: 5, c: [4, 5, 6], d: 10 }, - { a: 7, b: 8, c: [7, 8, 9], d: null }, - ], - { schema }, - ); + const actual = tableFromIPC(buf); + expect(actual.numRows).toBe(3); + const actualSchema = actual.schema; + expect(actualSchema).toEqual(schema); + }); - const buf = await fromTableToBuffer(table); - expect(buf.byteLength).toBeGreaterThan(0); + it("will detect vector columns when name contains 'vector' or 'embedding'", async function () { + // Test various naming patterns that should be detected as vector columns + const floatVectorTable = makeArrowTable([ + { + // Float vectors (use decimal values to ensure they're treated as floats) + // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns + user_vector: [1.1, 2.2], + // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns + text_embedding: [3.3, 4.4], + // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns + doc_embeddings: [5.5, 6.6], + // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns + my_vector_field: [7.7, 8.8], + // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns + embedding_model: [9.9, 10.1], + // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns + VECTOR_COL: [11.1, 12.2], // uppercase + // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns + Vector_Mixed: [13.3, 14.4], // mixed case + }, + ]); - const actual = tableFromIPC(buf); - expect(actual.numRows).toBe(3); - const actualSchema = actual.schema; - expect(actualSchema).toEqual(schema); - expect(table.getChild("a")?.toJSON()).toEqual([1, 4, 7]); - expect(table.getChild("b")?.toJSON()).toEqual([2, 5, 8]); - expect(table.getChild("d")?.toJSON()).toEqual([9n, 10n, null]); - }); + // Check that columns with 'vector' or 'embedding' in name are converted to FixedSizeList + const floatVectorColumns = [ + "user_vector", + "text_embedding", + "doc_embeddings", + "my_vector_field", + "embedding_model", + "VECTOR_COL", + "Vector_Mixed", + ]; - it("will use a provided FixedSizeList schema with typed array values", function () { - const schema = new Schema([ - new Field("text", new Utf8(), false), - new Field( - "vector", - new FixedSizeList(3, new Field("item", new Float32(), false)), - false, - ), - ]); - - const table = makeArrowTable( - [ - { - text: "foo", - vector: new Float32Array([1, 2, 3]), - }, - ], - { schema }, - ); - - expect(table.getChild("text")?.toJSON()).toEqual(["foo"]); + for (const columnName of floatVectorColumns) { expect( - table - .getChild("vector") - ?.toJSON() - .map((value) => value.toJSON()), - ).toEqual([[1, 2, 3]]); - }); - - it("will assume the column `vector` is FixedSizeList by default", async function () { - const schema = new Schema([ - new Field("a", new Float(Precision.DOUBLE), true), - new Field("b", new Int64(), true), - new Field( - "vector", - new FixedSizeList( - 3, - new Field("item", new Float(Precision.SINGLE), true), - ), - true, - ), - ]); - const table = makeArrowTable([ - { a: 1, b: 2n, vector: [1, 2, 3] }, - { a: 4, b: 5n, vector: [4, 5, 6] }, - { a: 7, b: 8n, vector: [7, 8, 9] }, - ]); - - const buf = await fromTableToBuffer(table); - expect(buf.byteLength).toBeGreaterThan(0); - - const actual = tableFromIPC(buf); - expect(actual.numRows).toBe(3); - const actualSchema = actual.schema; - expect(actualSchema).toEqual(schema); - - expect(table.getChild("a")?.toJSON()).toEqual([1, 4, 7]); - expect(table.getChild("b")?.toJSON()).toEqual([2n, 5n, 8n]); + DataType.isFixedSizeList(floatVectorTable.getChild(columnName)?.type), + ).toBe(true); + // Check that float vectors use Float32 by default expect( - table - .getChild("vector") - ?.toJSON() - .map((v) => v.toJSON()), - ).toEqual([ - [1, 2, 3], - [4, 5, 6], - [7, 8, 9], - ]); - }); + floatVectorTable + .getChild(columnName) + ?.type.children[0].type.toString(), + ).toEqual(new Float32().toString()); + } - it("can support multiple vector columns", async function () { - const schema = new Schema([ - new Field("a", new Float(Precision.DOUBLE), true), - new Field("b", new Float(Precision.DOUBLE), true), - new Field( - "vec1", - new FixedSizeList(3, new Field("item", new Float16(), true)), - true, - ), - new Field( - "vec2", - new FixedSizeList(3, new Field("item", new Float64(), true)), - true, - ), - ]); - const table = makeArrowTable( - [ - { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] }, - { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] }, - { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] }, - ], - { - vectorColumns: { - vec1: { type: new Float16() }, - vec2: { type: new Float64() }, - }, - }, - ); + // Test that regular integer arrays still get treated as float vectors + // (since JavaScript doesn't distinguish integers from floats at runtime) + const integerArrayTable = makeArrowTable([ + { + // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns + vector_int: [1, 2], // Regular array with integers - should be Float32 + // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns + embedding_int: [3, 4], // Regular array with integers - should be Float32 + }, + ]); - const buf = await fromTableToBuffer(table); - expect(buf.byteLength).toBeGreaterThan(0); - - const actual = tableFromIPC(buf); - expect(actual.numRows).toBe(3); - const actualSchema = actual.schema; - expect(actualSchema).toEqual(schema); - }); - - it("will detect vector columns when name contains 'vector' or 'embedding'", async function () { - // Test various naming patterns that should be detected as vector columns - const floatVectorTable = makeArrowTable([ - { - // Float vectors (use decimal values to ensure they're treated as floats) - // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns - user_vector: [1.1, 2.2], - // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns - text_embedding: [3.3, 4.4], - // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns - doc_embeddings: [5.5, 6.6], - // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns - my_vector_field: [7.7, 8.8], - // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns - embedding_model: [9.9, 10.1], - // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns - VECTOR_COL: [11.1, 12.2], // uppercase - // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns - Vector_Mixed: [13.3, 14.4], // mixed case - }, - ]); - - // Check that columns with 'vector' or 'embedding' in name are converted to FixedSizeList - const floatVectorColumns = [ - "user_vector", - "text_embedding", - "doc_embeddings", - "my_vector_field", - "embedding_model", - "VECTOR_COL", - "Vector_Mixed", - ]; - - for (const columnName of floatVectorColumns) { - expect( - DataType.isFixedSizeList( - floatVectorTable.getChild(columnName)?.type, - ), - ).toBe(true); - // Check that float vectors use Float32 by default - expect( - floatVectorTable - .getChild(columnName) - ?.type.children[0].type.toString(), - ).toEqual(new Float32().toString()); - } - - // Test that regular integer arrays still get treated as float vectors - // (since JavaScript doesn't distinguish integers from floats at runtime) - const integerArrayTable = makeArrowTable([ - { - // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns - vector_int: [1, 2], // Regular array with integers - should be Float32 - // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns - embedding_int: [3, 4], // Regular array with integers - should be Float32 - }, - ]); - - const integerArrayColumns = ["vector_int", "embedding_int"]; - - for (const columnName of integerArrayColumns) { - expect( - DataType.isFixedSizeList( - integerArrayTable.getChild(columnName)?.type, - ), - ).toBe(true); - // Regular integer arrays should use Float32 (avoiding false positives) - expect( - integerArrayTable - .getChild(columnName) - ?.type.children[0].type.toString(), - ).toEqual(new Float32().toString()); - } - - // Test normal list should NOT be converted to FixedSizeList - const normalListTable = makeArrowTable([ - { - // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns - normal_list: [15.5, 16.6], // should NOT be detected as vector - }, - ]); + const integerArrayColumns = ["vector_int", "embedding_int"]; + for (const columnName of integerArrayColumns) { expect( DataType.isFixedSizeList( - normalListTable.getChild("normal_list")?.type, + integerArrayTable.getChild(columnName)?.type, ), - ).toBe(false); - expect( - DataType.isList(normalListTable.getChild("normal_list")?.type), ).toBe(true); - }); - - it("will allow different vector column types", async function () { - const table = makeArrowTable([{ fp16: [1], fp32: [1], fp64: [1] }], { - vectorColumns: { - fp16: { type: new Float16() }, - fp32: { type: new Float32() }, - fp64: { type: new Float64() }, - }, - }); - + // Regular integer arrays should use Float32 (avoiding false positives) expect( - table.getChild("fp16")?.type.children[0].type.toString(), - ).toEqual(new Float16().toString()); - expect( - table.getChild("fp32")?.type.children[0].type.toString(), + integerArrayTable + .getChild(columnName) + ?.type.children[0].type.toString(), ).toEqual(new Float32().toString()); - expect( - table.getChild("fp64")?.type.children[0].type.toString(), - ).toEqual(new Float64().toString()); + } + + // Test normal list should NOT be converted to FixedSizeList + const normalListTable = makeArrowTable([ + { + // biome-ignore lint/style/useNamingConvention: Testing vector column detection patterns + normal_list: [15.5, 16.6], // should NOT be detected as vector + }, + ]); + + expect( + DataType.isFixedSizeList(normalListTable.getChild("normal_list")?.type), + ).toBe(false); + expect( + DataType.isList(normalListTable.getChild("normal_list")?.type), + ).toBe(true); + }); + + it("will allow different vector column types", async function () { + const table = makeArrowTable([{ fp16: [1], fp32: [1], fp64: [1] }], { + vectorColumns: { + fp16: { type: new Float16() }, + fp32: { type: new Float32() }, + fp64: { type: new Float64() }, + }, }); - it("will infer FixedSizeList from Float32Array values", async function () { - const table = makeArrowTable([ - { id: "a", vector: new Float32Array([0.1, 0.2, 0.3]) }, - { id: "b", vector: new Float32Array([0.4, 0.5, 0.6]) }, - ]); + expect(table.getChild("fp16")?.type.children[0].type.toString()).toEqual( + new Float16().toString(), + ); + expect(table.getChild("fp32")?.type.children[0].type.toString()).toEqual( + new Float32().toString(), + ); + expect(table.getChild("fp64")?.type.children[0].type.toString()).toEqual( + new Float64().toString(), + ); + }); - expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe( + it("will infer FixedSizeList from Float32Array values", async function () { + const table = makeArrowTable([ + { id: "a", vector: new Float32Array([0.1, 0.2, 0.3]) }, + { id: "b", vector: new Float32Array([0.4, 0.5, 0.6]) }, + ]); + + expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe( + true, + ); + const vectorType = table.getChild("vector")?.type; + expect(vectorType.listSize).toBe(3); + expect(vectorType.children[0].type.toString()).toEqual( + new Float32().toString(), + ); + }); + + it("will infer FixedSizeList from Uint8Array values", async function () { + const table = makeArrowTable([ + { id: "a", vector: new Uint8Array([1, 2, 3]) }, + { id: "b", vector: new Uint8Array([4, 5, 6]) }, + ]); + + expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe( + true, + ); + const vectorType = table.getChild("vector")?.type; + expect(vectorType.listSize).toBe(3); + expect(vectorType.children[0].type.toString()).toEqual( + new ArrowUint8().toString(), + ); + }); + + it("will use dictionary encoded strings if asked", async function () { + const table = makeArrowTable([{ str: "hello" }]); + expect(DataType.isUtf8(table.getChild("str")?.type)).toBe(true); + + const tableWithDict = makeArrowTable([{ str: "hello" }], { + dictionaryEncodeStrings: true, + }); + expect(DataType.isDictionary(tableWithDict.getChild("str")?.type)).toBe( + true, + ); + + const schema = new Schema([ + new Field("str", new Dictionary(new Utf8(), new Int32())), + ]); + + const tableWithDict2 = makeArrowTable([{ str: "hello" }], { schema }); + expect(DataType.isDictionary(tableWithDict2.getChild("str")?.type)).toBe( + true, + ); + }); + + it("will infer data types correctly", async function () { + await checkTableCreation( + // biome-ignore lint/suspicious/noExplicitAny: + async (records) => (makeArrowTable)(records), + true, + ); + }); + + it("will allow matching inferred types across records", function () { + expect(() => makeArrowTable([{ value: 1 }, { value: 2 }])).not.toThrow(); + }); + + it("will reject mismatched inferred types across records", function () { + expect(() => makeArrowTable([{ value: 1 }, { value: "two" }])).toThrow( + "Failed to infer schema for data. Previously inferred type Float64 but found Utf8 for field value at row 1. Consider providing an explicit schema.", + ); + }); + + it.each([ + ["ArrayBuffer", new Uint8Array([104]).buffer], + ["URL", new URL("https://example.com/")], + ["empty object", {}], + [ + "class instance with enumerable fields", + Object.assign(new Blob(["x"]), { position: 0 }), + ], + ])("rejects %s values without a schema", (_name, value) => { + expect(() => makeArrowTable([{ id: 1, value }])).toThrow( + /field value at row 0/, + ); + expect(() => makeArrowTable([{ value }])).toThrow(/field value at row 0/); + }); + + it.each([ + ["Binary", new Binary(), new Uint8Array([104]).buffer], + ["Utf8", new Utf8(), new URL("https://example.com/")], + ])("rejects %s values with a schema", (_name, type, value) => { + const schema = new Schema([new Field("value", type, true)]); + expect(() => makeArrowTable([{ value }], { schema })).toThrow( + /field value at row 0/, + ); + }); + + it("rejects unsupported values in nested struct fields", () => { + const schema = new Schema([ + new Field( + "metadata", + new Struct([new Field("bytes", new Binary(), true)]), true, - ); - const vectorType = table.getChild("vector")?.type; - expect(vectorType.listSize).toBe(3); - expect(vectorType.children[0].type.toString()).toEqual( - new Float32().toString(), - ); - }); + ), + ]); + expect(() => + makeArrowTable( + [{ metadata: { bytes: new Uint8Array([104]).buffer } }], + { schema }, + ), + ).toThrow(/field metadata\.bytes at row 0/); + }); - it("will infer FixedSizeList from Uint8Array values", async function () { - const table = makeArrowTable([ - { id: "a", vector: new Uint8Array([1, 2, 3]) }, - { id: "b", vector: new Uint8Array([4, 5, 6]) }, - ]); - - expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe( + it("rejects unsupported objects in list elements", () => { + const schema = new Schema([ + new Field( + "images", + new List(new Field("item", new Binary(), true)), true, - ); - const vectorType = table.getChild("vector")?.type; - expect(vectorType.listSize).toBe(3); - expect(vectorType.children[0].type.toString()).toEqual( - new ArrowUint8().toString(), - ); - }); + ), + ]); + expect(() => + makeArrowTable([{ images: [new Blob(["abc"])] }], { schema }), + ).toThrow(/field images\[0\] at row 0/); + }); - it("will use dictionary encoded strings if asked", async function () { - const table = makeArrowTable([{ str: "hello" }]); - expect(DataType.isUtf8(table.getChild("str")?.type)).toBe(true); - - const tableWithDict = makeArrowTable([{ str: "hello" }], { - dictionaryEncodeStrings: true, - }); - expect(DataType.isDictionary(tableWithDict.getChild("str")?.type)).toBe( - true, - ); - - const schema = new Schema([ - new Field("str", new Dictionary(new Utf8(), new Int32())), - ]); - - const tableWithDict2 = makeArrowTable([{ str: "hello" }], { schema }); - expect( - DataType.isDictionary(tableWithDict2.getChild("str")?.type), - ).toBe(true); - }); - - it("will infer data types correctly", async function () { - await checkTableCreation( - // biome-ignore lint/suspicious/noExplicitAny: - async (records) => (makeArrowTable)(records), - true, - ); - }); - - it("will allow matching inferred types across records", function () { - expect(() => - makeArrowTable([{ value: 1 }, { value: 2 }]), - ).not.toThrow(); - }); - - it("will reject mismatched inferred types across records", function () { - expect(() => makeArrowTable([{ value: 1 }, { value: "two" }])).toThrow( - "Failed to infer schema for data. Previously inferred type Float64 but found Utf8 for field value at row 1. Consider providing an explicit schema.", - ); - }); - - it.each([ - ["ArrayBuffer", new Uint8Array([104]).buffer], - ["URL", new URL("https://example.com/")], - ["empty object", {}], - [ - "class instance with enumerable fields", - Object.assign(new Blob(["x"]), { position: 0 }), - ], - ])("rejects %s values without a schema", (_name, value) => { - expect(() => makeArrowTable([{ id: 1, value }])).toThrow( - /field value at row 0/, - ); - expect(() => makeArrowTable([{ value }])).toThrow( - /field value at row 0/, - ); - }); - - it.each([ - ["Binary", new Binary(), new Uint8Array([104]).buffer], - ["Utf8", new Utf8(), new URL("https://example.com/")], - ])("rejects %s values with a schema", (_name, type, value) => { - const schema = new Schema([new Field("value", type, true)]); - expect(() => makeArrowTable([{ value }], { schema })).toThrow( - /field value at row 0/, - ); - }); - - it("rejects unsupported values in nested struct fields", () => { - const schema = new Schema([ - new Field( - "metadata", - new Struct([new Field("bytes", new Binary(), true)]), - true, - ), - ]); - expect(() => - makeArrowTable( - [{ metadata: { bytes: new Uint8Array([104]).buffer } }], - { schema }, - ), - ).toThrow(/field metadata\.bytes at row 0/); - }); - - it("rejects unsupported objects in list elements", () => { - const schema = new Schema([ - new Field( - "images", - new List(new Field("item", new Binary(), true)), - true, - ), - ]); - expect(() => - makeArrowTable([{ images: [new Blob(["abc"])] }], { schema }), - ).toThrow(/field images\[0\] at row 0/); - }); - - it("rejects unsupported objects in nested lists", () => { - const schema = new Schema([ - new Field( - "images", - new List( - new Field( - "item", - new List(new Field("item", new Binary(), true)), - true, - ), + it("rejects unsupported objects in nested lists", () => { + const schema = new Schema([ + new Field( + "images", + new List( + new Field( + "item", + new List(new Field("item", new Binary(), true)), + true, ), - true, ), - ]); - expect(() => - makeArrowTable([{ images: [[new Blob(["abc"])]] }], { schema }), - ).toThrow(/field images\[0\]\[0\] at row 0/); - }); + true, + ), + ]); + expect(() => + makeArrowTable([{ images: [[new Blob(["abc"])]] }], { schema }), + ).toThrow(/field images\[0\]\[0\] at row 0/); + }); - it("rejects unsupported nested struct fields in lists", () => { - const schema = new Schema([ - new Field( - "items", - new List( - new Field( - "item", - new Struct([new Field("bytes", new Binary(), true)]), - true, - ), + it("rejects unsupported nested struct fields in lists", () => { + const schema = new Schema([ + new Field( + "items", + new List( + new Field( + "item", + new Struct([new Field("bytes", new Binary(), true)]), + true, ), - true, ), - ]); - expect(() => - makeArrowTable( - [{ items: [{ bytes: new Uint8Array([104]).buffer }] }], - { schema }, - ), - ).toThrow(/field items\[0\]\.bytes at row 0/); + true, + ), + ]); + expect(() => + makeArrowTable([{ items: [{ bytes: new Uint8Array([104]).buffer }] }], { + schema, + }), + ).toThrow(/field items\[0\]\.bytes at row 0/); + }); + + it("will ignore generated dictionary IDs when comparing inferred types", function () { + const table = makeArrowTable([{ str: "a" }, { str: "b" }], { + dictionaryEncodeStrings: true, }); - it("will ignore generated dictionary IDs when comparing inferred types", function () { - const table = makeArrowTable([{ str: "a" }, { str: "b" }], { - dictionaryEncodeStrings: true, - }); + expect(table.getChild("str")?.toJSON()).toEqual(["a", "b"]); + }); - expect(table.getChild("str")?.toJSON()).toEqual(["a", "b"]); - }); + it("will preserve null values without treating them as type mismatches", function () { + for (const records of [ + [{ vector: [1, 2, 3] }, { vector: null }], + [{ vector: null }, { vector: [1, 2, 3] }], + ]) { + const table = makeArrowTable(records); - it("will preserve null values without treating them as type mismatches", function () { - for (const records of [ - [{ vector: [1, 2, 3] }, { vector: null }], - [{ vector: null }, { vector: [1, 2, 3] }], - ]) { - const table = makeArrowTable(records); + expect(table.numRows).toBe(2); + expect(table.getChild("vector")?.nullCount).toBe(1); + } + }); - expect(table.numRows).toBe(2); - expect(table.getChild("vector")?.nullCount).toBe(1); - } - }); - - it("will preserve empty variable-size lists", function () { - for (const records of [ - [{ items: [1] }, { items: [] }], - [{ items: [] }, { items: [1] }], - ]) { - const table = makeArrowTable(records); - expect( - table - .getChild("items") - ?.toJSON() - .map((value) => value.toJSON()), - ).toEqual(records.map((record) => record.items)); - } - }); - - it("will propagate deferred evidence through nested lists", function () { - for (const records of [ - [{ items: [1] }, { items: [null] }], - [{ items: [null] }, { items: [1] }], - [{ items: [null, 1] }, { items: [2, null] }], - ]) { - const table = makeArrowTable(records); - expect( - table - .getChild("items") - ?.toJSON() - .map((value) => value.toJSON()), - ).toEqual(records.map((record) => record.items)); - } - - const nestedRecords = [{ items: [[1]] }, { items: [[null]] }]; - const nestedTable = makeArrowTable(nestedRecords); + it("will preserve empty variable-size lists", function () { + for (const records of [ + [{ items: [1] }, { items: [] }], + [{ items: [] }, { items: [1] }], + ]) { + const table = makeArrowTable(records); expect( - nestedTable + table .getChild("items") ?.toJSON() - .map((value) => - value - .toJSON() - .map((nestedValue: { toJSON: () => unknown[] }) => - nestedValue.toJSON(), - ), - ), - ).toEqual(nestedRecords.map((record) => record.items)); - }); + .map((value) => value.toJSON()), + ).toEqual(records.map((record) => record.items)); + } + }); - it("will reject incompatible deferred evidence within a list", function () { - for (const items of [ - [[], 1], - [1, []], - [[null], 1], - [1, [null]], - ]) { - expect(() => makeArrowTable([{ items }])).toThrow( - "Failed to infer data type for field items at row 0.", - ); - } - }); + it("will propagate deferred evidence through nested lists", function () { + for (const records of [ + [{ items: [1] }, { items: [null] }], + [{ items: [null] }, { items: [1] }], + [{ items: [null, 1] }, { items: [2, null] }], + ]) { + const table = makeArrowTable(records); + expect( + table + .getChild("items") + ?.toJSON() + .map((value) => value.toJSON()), + ).toEqual(records.map((record) => record.items)); + } - it("will reject empty fixed-size lists", function () { - expect(() => - makeArrowTable([{ vector: [1, 2, 3] }, { vector: [] }]), - ).toThrow( - "Failed to infer schema for data. Previously inferred type FixedSizeList[3] but found List[0] for field vector at row 1.", + const nestedRecords = [{ items: [[1]] }, { items: [[null]] }]; + const nestedTable = makeArrowTable(nestedRecords); + expect( + nestedTable + .getChild("items") + ?.toJSON() + .map((value) => + value + .toJSON() + .map((nestedValue: { toJSON: () => unknown[] }) => + nestedValue.toJSON(), + ), + ), + ).toEqual(nestedRecords.map((record) => record.items)); + }); + + it("will reject incompatible deferred evidence within a list", function () { + for (const items of [ + [[], 1], + [1, []], + [[null], 1], + [1, [null]], + ]) { + expect(() => makeArrowTable([{ items }])).toThrow( + "Failed to infer data type for field items at row 0.", ); - }); + } + }); - it("will reject inferred leaf and branch shape changes", function () { - expect(() => - makeArrowTable([{ value: 1 }, { value: { nested: 2 } }]), - ).toThrow( - "Failed to infer schema for data. Previously inferred type Float64 but found Struct for field value at row 1.", - ); - expect(() => - makeArrowTable([{ value: { nested: 1 } }, { value: 2 }]), - ).toThrow( - "Failed to infer schema for data. Previously inferred type Struct but found Float64 for field value at row 1.", - ); - }); + it("will reject empty fixed-size lists", function () { + expect(() => + makeArrowTable([{ vector: [1, 2, 3] }, { vector: [] }]), + ).toThrow( + "Failed to infer schema for data. Previously inferred type FixedSizeList[3] but found List[0] for field vector at row 1.", + ); + }); - it("will allow null values around inferred struct values", function () { - for (const { records, nullIndex } of [ - { - records: [{ value: null }, { value: { nested: 2 } }], - nullIndex: 0, - }, - { - records: [{ value: { nested: 1 } }, { value: null }], - nullIndex: 1, - }, - ]) { - const table = makeArrowTable(records); - const values = table.getChild("value"); + it("will reject inferred leaf and branch shape changes", function () { + expect(() => + makeArrowTable([{ value: 1 }, { value: { nested: 2 } }]), + ).toThrow( + "Failed to infer schema for data. Previously inferred type Float64 but found Struct for field value at row 1.", + ); + expect(() => + makeArrowTable([{ value: { nested: 1 } }, { value: 2 }]), + ).toThrow( + "Failed to infer schema for data. Previously inferred type Struct but found Float64 for field value at row 1.", + ); + }); - expect(values?.nullCount).toBe(1); - expect(values?.get(nullIndex)).toBeNull(); - } - }); + it("will allow null values around inferred struct values", function () { + for (const { records, nullIndex } of [ + { + records: [{ value: null }, { value: { nested: 2 } }], + nullIndex: 0, + }, + { + records: [{ value: { nested: 1 } }, { value: null }], + nullIndex: 1, + }, + ]) { + const table = makeArrowTable(records); + const values = table.getChild("value"); - it("will allow a schema to be provided", async function () { - await checkTableCreation( - async (records, _, schema) => - // biome-ignore lint/suspicious/noExplicitAny: - (makeArrowTable)(records, { schema }), - false, - ); - }); + expect(values?.nullCount).toBe(1); + expect(values?.get(nullIndex)).toBeNull(); + } + }); - it("will use the field order of any provided schema", async function () { - await checkTableCreation( - async (_, recordsReversed, schema) => - // biome-ignore lint/suspicious/noExplicitAny: - (makeArrowTable)(recordsReversed, { schema }), - false, - ); - }); - - it("will make an empty table", async function () { - await checkTableCreation( + it("will allow a schema to be provided", async function () { + await checkTableCreation( + async (records, _, schema) => // biome-ignore lint/suspicious/noExplicitAny: - async (_, __, schema) => (makeArrowTable)([], { schema }), - false, - ); - }); + (makeArrowTable)(records, { schema }), + false, + ); + }); - it("will allow subsets of columns if nullable", async function () { - const schema = new Schema([ - new Field("a", new Int64(), true), + it("will use the field order of any provided schema", async function () { + await checkTableCreation( + async (_, recordsReversed, schema) => + // biome-ignore lint/suspicious/noExplicitAny: + (makeArrowTable)(recordsReversed, { schema }), + false, + ); + }); + + it("will make an empty table", async function () { + await checkTableCreation( + // biome-ignore lint/suspicious/noExplicitAny: + async (_, __, schema) => (makeArrowTable)([], { schema }), + false, + ); + }); + + it("will allow subsets of columns if nullable", async function () { + const schema = new Schema([ + new Field("a", new Int64(), true), + new Field( + "s", + new Struct([ + new Field("x", new Int32(), true), + new Field("y", new Int32(), true), + ]), + true, + ), + new Field("d", new Int16(), true), + ]); + + const table = makeArrowTable([{ a: 1n }], { schema }); + expect(table.numCols).toBe(1); + expect(table.numRows).toBe(1); + + const table2 = makeArrowTable([{ a: 1n, d: 2 }], { schema }); + expect(table2.numCols).toBe(2); + + const table3 = makeArrowTable([{ s: { y: 3 } }], { schema }); + expect(table3.numCols).toBe(1); + const expectedSchema = new Schema([ + new Field("s", new Struct([new Field("y", new Int32(), true)]), true), + ]); + expect(table3.schema).toEqual(expectedSchema); + }); + + it("will work even if columns are sparsely provided", async function () { + const sparseRecords = [{ a: 1n }, { b: 2n }, { c: 3n }, { d: 4n }]; + const table = makeArrowTable(sparseRecords); + expect(table.numCols).toBe(4); + expect(table.numRows).toBe(4); + + const schema = new Schema([ + new Field("a", new Int64(), true), + new Field("b", new Int32(), true), + new Field("c", new Int64(), true), + new Field("d", new Int16(), true), + ]); + const table2 = makeArrowTable(sparseRecords, { schema }); + expect(table2.numCols).toBe(4); + expect(table2.numRows).toBe(4); + expect(table2.schema).toEqual(schema); + }); + + it("will handle missing columns in schema alignment when using embeddings", async function () { + const schema = new Schema( + [ + new Field("domain", new Utf8(), true), + new Field("name", new Utf8(), true), + new Field("description", new Utf8(), true), + ], + new Map([["embedding_functions", JSON.stringify([])]]), + ); + + const data = [ + { domain: "google.com", name: "Google" }, + { domain: "facebook.com", name: "Facebook" }, + ]; + + const table = await convertToTable(data, undefined, { schema }); + + expect(table.numCols).toBe(3); + expect(table.numRows).toBe(2); + + const descriptionColumn = table.getChild("description"); + expect(descriptionColumn).toBeDefined(); + expect(descriptionColumn?.nullCount).toBe(2); + expect(descriptionColumn?.toArray()).toEqual([null, null]); + + expect(table.getChild("domain")?.toArray()).toEqual([ + "google.com", + "facebook.com", + ]); + expect(table.getChild("name")?.toArray()).toEqual(["Google", "Facebook"]); + }); + + it("will handle completely missing nested struct columns", async function () { + const schema = new Schema( + [ + new Field("id", new Utf8(), true), + new Field("name", new Utf8(), true), new Field( - "s", + "metadata", new Struct([ - new Field("x", new Int32(), true), - new Field("y", new Int32(), true), + new Field("version", new Int32(), true), + new Field("author", new Utf8(), true), + new Field( + "tags", + new List(new Field("item", new Utf8(), true)), + true, + ), ]), true, ), - new Field("d", new Int16(), true), - ]); - - const table = makeArrowTable([{ a: 1n }], { schema }); - expect(table.numCols).toBe(1); - expect(table.numRows).toBe(1); - - const table2 = makeArrowTable([{ a: 1n, d: 2 }], { schema }); - expect(table2.numCols).toBe(2); - - const table3 = makeArrowTable([{ s: { y: 3 } }], { schema }); - expect(table3.numCols).toBe(1); - const expectedSchema = new Schema([ - new Field("s", new Struct([new Field("y", new Int32(), true)]), true), - ]); - expect(table3.schema).toEqual(expectedSchema); - }); - - it("will work even if columns are sparsely provided", async function () { - const sparseRecords = [{ a: 1n }, { b: 2n }, { c: 3n }, { d: 4n }]; - const table = makeArrowTable(sparseRecords); - expect(table.numCols).toBe(4); - expect(table.numRows).toBe(4); - - const schema = new Schema([ - new Field("a", new Int64(), true), - new Field("b", new Int32(), true), - new Field("c", new Int64(), true), - new Field("d", new Int16(), true), - ]); - const table2 = makeArrowTable(sparseRecords, { schema }); - expect(table2.numCols).toBe(4); - expect(table2.numRows).toBe(4); - expect(table2.schema).toEqual(schema); - }); - - it("will handle missing columns in schema alignment when using embeddings", async function () { - const schema = new Schema( - [ - new Field("domain", new Utf8(), true), - new Field("name", new Utf8(), true), - new Field("description", new Utf8(), true), - ], - new Map([["embedding_functions", JSON.stringify([])]]), - ); - - const data = [ - { domain: "google.com", name: "Google" }, - { domain: "facebook.com", name: "Facebook" }, - ]; - - const table = await convertToTable(data, undefined, { schema }); - - expect(table.numCols).toBe(3); - expect(table.numRows).toBe(2); - - const descriptionColumn = table.getChild("description"); - expect(descriptionColumn).toBeDefined(); - expect(descriptionColumn?.nullCount).toBe(2); - expect(descriptionColumn?.toArray()).toEqual([null, null]); - - expect(table.getChild("domain")?.toArray()).toEqual([ - "google.com", - "facebook.com", - ]); - expect(table.getChild("name")?.toArray()).toEqual([ - "Google", - "Facebook", - ]); - }); - - it("will handle completely missing nested struct columns", async function () { - const schema = new Schema( - [ - new Field("id", new Utf8(), true), - new Field("name", new Utf8(), true), - new Field( - "metadata", - new Struct([ - new Field("version", new Int32(), true), - new Field("author", new Utf8(), true), - new Field( - "tags", - new List(new Field("item", new Utf8(), true)), - true, - ), - ]), - true, - ), - ], - new Map([["embedding_functions", JSON.stringify([])]]), - ); - - const data = [ - { id: "doc1", name: "Document 1" }, - { id: "doc2", name: "Document 2" }, - ]; - - const table = await convertToTable(data, undefined, { schema }); - - expect(table.numCols).toBe(3); - expect(table.numRows).toBe(2); - - const buf = await fromTableToBuffer(table); - const retrievedTable = tableFromIPC(buf); - - const rows = []; - for (let i = 0; i < retrievedTable.numRows; i++) { - rows.push(retrievedTable.get(i)); - } - - expect(rows[0].metadata.version).toBe(null); - expect(rows[0].metadata.author).toBe(null); - expect(rows[0].metadata.tags).toBe(null); - expect(rows[0].id).toBe("doc1"); - expect(rows[0].name).toBe("Document 1"); - }); - - it("will handle partially missing nested struct fields", async function () { - const schema = new Schema( - [ - new Field("id", new Utf8(), true), - new Field( - "metadata", - new Struct([ - new Field("version", new Int32(), true), - new Field("author", new Utf8(), true), - new Field("created_at", new Utf8(), true), - ]), - true, - ), - ], - new Map([["embedding_functions", JSON.stringify([])]]), - ); - - const data = [ - { id: "doc1", metadata: { version: 1, author: "Alice" } }, - { id: "doc2", metadata: { version: 2 } }, - ]; - - const table = await convertToTable(data, undefined, { schema }); - - expect(table.numCols).toBe(2); - expect(table.numRows).toBe(2); - - const metadataColumn = table.getChild("metadata"); - expect(metadataColumn).toBeDefined(); - expect(metadataColumn?.type.toString()).toBe( - "Struct<{version:Int32, author:Utf8, created_at:Utf8}>", - ); - }); - - it("will handle multiple levels of nested structures", async function () { - const schema = new Schema( - [ - new Field("id", new Utf8(), true), - new Field( - "config", - new Struct([ - new Field("database", new Utf8(), true), - new Field( - "connection", - new Struct([ - new Field("host", new Utf8(), true), - new Field("port", new Int32(), true), - new Field( - "ssl", - new Struct([ - new Field("enabled", new Bool(), true), - new Field("cert_path", new Utf8(), true), - ]), - true, - ), - ]), - true, - ), - ]), - true, - ), - ], - new Map([["embedding_functions", JSON.stringify([])]]), - ); - - const data = [ - { - id: "config1", - config: { - database: "postgres", - connection: { host: "localhost" }, - }, - }, - { - id: "config2", - config: { database: "mysql" }, - }, - { - id: "config3", - }, - ]; - - const table = await convertToTable(data, undefined, { schema }); - - expect(table.numCols).toBe(2); - expect(table.numRows).toBe(3); - - const configColumn = table.getChild("config"); - expect(configColumn).toBeDefined(); - expect(configColumn?.type.toString()).toBe( - "Struct<{database:Utf8, connection:Struct<{host:Utf8, port:Int32, ssl:Struct<{enabled:Bool, cert_path:Utf8}>}>}>", - ); - }); - - it("will handle missing columns in Arrow table input when using embeddings", async function () { - const incompleteTable = makeArrowTable([ - { domain: "google.com", name: "Google" }, - { domain: "facebook.com", name: "Facebook" }, - ]); - - const schema = new Schema( - [ - new Field("domain", new Utf8(), true), - new Field("name", new Utf8(), true), - new Field("description", new Utf8(), true), - ], - new Map([["embedding_functions", JSON.stringify([])]]), - ); - - const buf = await fromDataToBuffer(incompleteTable, undefined, schema); - - expect(buf.byteLength).toBeGreaterThan(0); - - const retrievedTable = tableFromIPC(buf); - expect(retrievedTable.numCols).toBe(3); - expect(retrievedTable.numRows).toBe(2); - - const descriptionColumn = retrievedTable.getChild("description"); - expect(descriptionColumn).toBeDefined(); - expect(descriptionColumn?.nullCount).toBe(2); - expect(descriptionColumn?.toArray()).toEqual([null, null]); - - expect(retrievedTable.getChild("domain")?.toArray()).toEqual([ - "google.com", - "facebook.com", - ]); - expect(retrievedTable.getChild("name")?.toArray()).toEqual([ - "Google", - "Facebook", - ]); - }); - - it("should correctly retain values in nested struct fields", async function () { - const testData = [ - { - id: "doc1", - vector: [1, 2, 3], - metadata: { - filePath: "/path/to/file1.ts", - startLine: 10, - endLine: 20, - text: "function test() { return true; }", - }, - }, - { - id: "doc2", - vector: [4, 5, 6], - metadata: { - filePath: "/path/to/file2.ts", - startLine: 30, - endLine: 40, - text: "function test2() { return false; }", - }, - }, - ]; - - const table = makeArrowTable(testData); - - const metadataField = table.schema.fields.find( - (f) => f.name === "metadata", - ); - expect(metadataField).toBeDefined(); - // biome-ignore lint/suspicious/noExplicitAny: accessing fields in different Arrow versions - const childNames = metadataField?.type.children.map((c: any) => c.name); - expect(childNames).toEqual([ - "filePath", - "startLine", - "endLine", - "text", - ]); - - const buf = await fromTableToBuffer(table); - const retrievedTable = tableFromIPC(buf); - - const rows = []; - for (let i = 0; i < retrievedTable.numRows; i++) { - rows.push(retrievedTable.get(i)); - } - - const firstRow = rows[0]; - expect(firstRow.id).toBe("doc1"); - expect(firstRow.vector.toJSON()).toEqual([1, 2, 3]); - expect(firstRow.metadata.filePath).toBe("/path/to/file1.ts"); - expect(firstRow.metadata.startLine).toBe(10); - expect(firstRow.metadata.endLine).toBe(20); - expect(firstRow.metadata.text).toBe("function test() { return true; }"); - }); - }); - - class DummyEmbedding extends EmbeddingFunction { - toJSON(): Partial { - return {}; - } - - async computeSourceEmbeddings(data: string[]): Promise { - return data.map(() => [0.0, 0.0]); - } - - ndims(): number { - return 2; - } - - embeddingDataType() { - return new Float16(); - } - } - - class DummyEmbeddingWithNoDimension extends EmbeddingFunction { - toJSON(): Partial { - return {}; - } - - embeddingDataType() { - return new Float16(); - } - - async computeSourceEmbeddings(data: string[]): Promise { - return data.map(() => [0.0, 0.0]); - } - } - const dummyEmbeddingConfig: EmbeddingFunctionConfig = { - sourceColumn: "string", - function: new DummyEmbedding(), - }; - - const dummyEmbeddingConfigWithNoDimension: EmbeddingFunctionConfig = { - sourceColumn: "string", - function: new DummyEmbeddingWithNoDimension(), - }; - - describe("convertToTable", function () { - it("will infer data types correctly", async function () { - await checkTableCreation( - // biome-ignore lint/suspicious/noExplicitAny: - async (records) => await (convertToTable)(records), - true, - ); - }); - - it("will allow a schema to be provided", async function () { - await checkTableCreation( - async (records, _, schema) => - // biome-ignore lint/suspicious/noExplicitAny: - await (convertToTable)(records, undefined, { schema }), - false, - ); - }); - - it("will use the field order of any provided schema", async function () { - await checkTableCreation( - async (_, recordsReversed, schema) => - // biome-ignore lint/suspicious/noExplicitAny: - await (convertToTable)(recordsReversed, undefined, { schema }), - false, - ); - }); - - it("will make an empty table", async function () { - await checkTableCreation( - async (_, __, schema) => - // biome-ignore lint/suspicious/noExplicitAny: - await (convertToTable)([], undefined, { schema }), - false, - ); - }); - - it("will apply embeddings", async function () { - const records = sampleRecords(); - const table = await convertToTable(records, dummyEmbeddingConfig); - expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe( - true, - ); - expect( - table.getChild("vector")?.type.children[0].type.toString(), - ).toEqual(new Float16().toString()); - }); - - it("will fail if missing the embedding source column", async function () { - await expect( - convertToTable([{ id: 1 }], dummyEmbeddingConfig), - ).rejects.toThrow("'string' was not present"); - }); - - it("use embeddingDimension if embedding missing from table", async function () { - const schema = new Schema([new Field("string", new Utf8(), false)]); - // Simulate getting an empty Arrow table (minus embedding) from some other source - // In other words, we aren't starting with records - const table = makeEmptyTable(schema); - - // If the embedding specifies the dimension we are fine - await fromTableToBuffer(table, dummyEmbeddingConfig); - - // We can also supply a schema and should be ok - const schemaWithEmbedding = new Schema([ - new Field("string", new Utf8(), false), - new Field( - "vector", - new FixedSizeList(2, new Field("item", new Float16(), false)), - false, - ), - ]); - await fromTableToBuffer( - table, - dummyEmbeddingConfigWithNoDimension, - schemaWithEmbedding, - ); - - // Otherwise we will get an error - await expect( - fromTableToBuffer(table, dummyEmbeddingConfigWithNoDimension), - ).rejects.toThrow("does not specify `embeddingDimension`"); - }); - - it("will apply embeddings to an empty table", async function () { - const schema = new Schema([ - new Field("string", new Utf8(), false), - new Field( - "vector", - new FixedSizeList(2, new Field("item", new Float16(), false)), - false, - ), - ]); - const table = await convertToTable([], dummyEmbeddingConfig, { - schema, - }); - expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe( - true, - ); - expect( - table.getChild("vector")?.type.children[0].type.toString(), - ).toEqual(new Float16().toString()); - }); - - it("will complain if embeddings present but schema missing embedding column", async function () { - const schema = new Schema([new Field("string", new Utf8(), false)]); - await expect( - convertToTable([], dummyEmbeddingConfig, { schema }), - ).rejects.toThrow("column vector was missing"); - }); - - it("will skip embedding application if already applied", async function () { - const records = sampleRecords(); - const table = await convertToTable(records, dummyEmbeddingConfig); - - // fromTableToBuffer will try and apply the embeddings again - // but should skip since the column already has non-null values - const result = await fromTableToBuffer(table, dummyEmbeddingConfig); - expect(result.byteLength).toBeGreaterThan(0); - }); - }); - - describe("makeEmptyTable", function () { - it("will make an empty table", async function () { - await checkTableCreation( - // biome-ignore lint/suspicious/noExplicitAny: - async (_, __, schema) => (makeEmptyTable)(schema), - false, - ); - }); - - it("will make an empty table with a Map field", async function () { - const schema = new Schema([ - new Field( - "attributes", - new Map_( - new Field( - "entries", - new Struct([ - new Field("key", new Utf8(), false), - new Field("value", new Int32(), true), - ]), - false, - ), - true, - ), - ), - ]); - - const table = makeEmptyTable(schema); - - expectValidMapField(table.schema.fields[0]); - - const buffer = await fromTableToBuffer(table); - const roundTripped = tableFromIPC(buffer); - - expectValidMapField(roundTripped.schema.fields[0]); - }); - - it("preserves string schema metadata", function () { - const metadata = new Map([["source", "fixture"]]); - const schema = new Schema( - [new Field("value", new Int32(), true)], - metadata, - ); - - expect(makeEmptyTable(schema).schema.metadata.get("source")).toBe( - "fixture", - ); - }); - - it.each([ - ["non-string keys", new Map([[42, "fixture"]])], - ["non-string values", new Map([["source", 42]])], - [ - "non-string keys and values", - new Map([[42, false]]), ], - ])("rejects schema metadata with %s", function (_, metadataLike) { - const metadata = metadataLike as unknown as Map; - const schema = new Schema( - [new Field("value", new Int32(), true)], - metadata, - ); + new Map([["embedding_functions", JSON.stringify([])]]), + ); - expect(() => makeEmptyTable(schema)).toThrow( - "Expected metadata, if present, to be a Map but it had non-string keys or values", - ); - }); + const data = [ + { id: "doc1", name: "Document 1" }, + { id: "doc2", name: "Document 2" }, + ]; + + const table = await convertToTable(data, undefined, { schema }); + + expect(table.numCols).toBe(3); + expect(table.numRows).toBe(2); + + const buf = await fromTableToBuffer(table); + const retrievedTable = tableFromIPC(buf); + + const rows = []; + for (let i = 0; i < retrievedTable.numRows; i++) { + rows.push(retrievedTable.get(i)); + } + + expect(rows[0].metadata.version).toBe(null); + expect(rows[0].metadata.author).toBe(null); + expect(rows[0].metadata.tags).toBe(null); + expect(rows[0].id).toBe("doc1"); + expect(rows[0].name).toBe("Document 1"); }); - describe("when using two versions of arrow", function () { - it("preserves a dictionary shared by multiple fields", async function () { - const values = ["alpha", "beta", "alpha"]; - const dictionaryVector = vectorFromArray(values); - const batch = new ArrowRecordBatch({ - first: dictionaryVector.data[0], - second: dictionaryVector.data[0], - }); - const table = new ArrowTable([batch]); - - const sanitized = sanitizeTable(table); - expect([...sanitized.getChild("first")!]).toEqual(values); - expect([...sanitized.getChild("second")!]).toEqual(values); - const firstType = sanitized.schema.fields[0].type as { - dictionary: unknown; - }; - const secondType = sanitized.schema.fields[1].type as { - dictionary: unknown; - }; - expect(secondType.dictionary).toBe(firstType.dictionary); - expect(sanitized.batches[0].data.children[1].dictionary).toBe( - sanitized.batches[0].data.children[0].dictionary, - ); - - const buf = await fromDataToBuffer(table); - const actual = currentTableFromIPC(buf); - expect([...actual.getChild("first")!]).toEqual(values); - expect([...actual.getChild("second")!]).toEqual(values); - }); - - it("preserves shared dictionary data from another Arrow version", async function () { - const values = ["alpha", "beta", "alpha"]; - const dictionaryVector = vectorFromArray(values); - const firstBatch = new ArrowRecordBatch({ - label: dictionaryVector.slice(0, 2).data[0], - }); - const secondBatch = new ArrowRecordBatch({ - label: dictionaryVector.slice(2).data[0], - }); - const table = new ArrowTable([firstBatch, secondBatch]); - - const sanitized = sanitizeTable(table); - expect([...sanitized.getChild("label")!]).toEqual(values); - - const dictionaries = sanitized.batches.map( - (batch) => batch.data.children[0].dictionary, - ); - expect(dictionaries[0]).toBeInstanceOf(CurrentVector); - expect(dictionaries[1]).toBe(dictionaries[0]); - - const buf = await fromDataToBuffer(table); - const actual = currentTableFromIPC(buf); - expect([...actual.getChild("label")!]).toEqual(values); - }); - - it("preserves shared chunks in growing dictionaries", async function () { - const type = new Dictionary(new Utf8(), new Int32(), 42, false); - const firstDictionary = vectorFromArray(["alpha", "beta"], new Utf8()); - const secondDictionary = firstDictionary.concat( - vectorFromArray(["gamma"], new Utf8()), - ); - const firstData = arrowMakeData({ - type, - data: Int32Array.from([0, 1]), - dictionary: firstDictionary, - }); - const secondData = arrowMakeData({ - type, - data: Int32Array.from([2]), - dictionary: secondDictionary, - }); - const table = new ArrowTable([ - new ArrowRecordBatch({ label: firstData }), - new ArrowRecordBatch({ label: secondData }), - ]); - - const sanitized = sanitizeTable(table); - const expected = ["alpha", "beta", "gamma"]; - expect([...sanitized.getChild("label")!]).toEqual(expected); - const firstLocalDictionary = - sanitized.batches[0].data.children[0].dictionary!; - const secondLocalDictionary = - sanitized.batches[1].data.children[0].dictionary!; - expect(secondLocalDictionary.data[0]).toBe( - firstLocalDictionary.data[0], - ); - - const buf = await fromTableToBuffer(sanitized); - const actual = currentTableFromIPC(buf); - expect([...actual.getChild("label")!]).toEqual(expected); - }); - - it("can serialize list data from another Arrow version", async function () { - const values = [["anime", "action"], [], null]; - const vector = vectorFromArray( - values, - new List(new Field("item", new Utf8(), true)), - ); - const table = new ArrowTable({ tags: vector }); - - const buf = await fromDataToBuffer(table); - const actual = currentTableFromIPC(buf); - const actualTags = actual.getChild("tags"); - - expect(actualTags?.get(0)?.toJSON()).toEqual(values[0]); - expect(actualTags?.get(1)?.toJSON()).toEqual(values[1]); - expect(actualTags?.get(2)).toBeNull(); - }); - - it("can still import data", async function () { - const schema = new arrow15.Schema([ - new arrow15.Field("id", new arrow15.Int32()), - new arrow15.Field( - "vector", - new arrow15.FixedSizeList( - 1024, - new arrow15.Field("item", new arrow15.Float32(), true), - ), + it("will handle partially missing nested struct fields", async function () { + const schema = new Schema( + [ + new Field("id", new Utf8(), true), + new Field( + "metadata", + new Struct([ + new Field("version", new Int32(), true), + new Field("author", new Utf8(), true), + new Field("created_at", new Utf8(), true), + ]), + true, ), - new arrow15.Field( - "struct", - new arrow15.Struct([ - new arrow15.Field( - "nested", - new arrow15.Dictionary( - new arrow15.Utf8(), - new arrow15.Int32(), - 1, - true, - ), - ), - new arrow15.Field( - "ts_with_tz", - new arrow15.TimestampNanosecond("some_tz"), - ), - new arrow15.Field( - "ts_no_tz", - new arrow15.TimestampNanosecond(null), + ], + new Map([["embedding_functions", JSON.stringify([])]]), + ); + + const data = [ + { id: "doc1", metadata: { version: 1, author: "Alice" } }, + { id: "doc2", metadata: { version: 2 } }, + ]; + + const table = await convertToTable(data, undefined, { schema }); + + expect(table.numCols).toBe(2); + expect(table.numRows).toBe(2); + + const metadataColumn = table.getChild("metadata"); + expect(metadataColumn).toBeDefined(); + expect(metadataColumn?.type.toString()).toBe( + "Struct<{version:Int32, author:Utf8, created_at:Utf8}>", + ); + }); + + it("will handle multiple levels of nested structures", async function () { + const schema = new Schema( + [ + new Field("id", new Utf8(), true), + new Field( + "config", + new Struct([ + new Field("database", new Utf8(), true), + new Field( + "connection", + new Struct([ + new Field("host", new Utf8(), true), + new Field("port", new Int32(), true), + new Field( + "ssl", + new Struct([ + new Field("enabled", new Bool(), true), + new Field("cert_path", new Utf8(), true), + ]), + true, + ), + ]), + true, ), ]), + true, ), - // biome-ignore lint/suspicious/noExplicitAny: skip - ]) as any; - schema.metadataVersion = arrow15.MetadataVersion.V5; - const table = makeArrowTable([], { schema }); + ], + new Map([["embedding_functions", JSON.stringify([])]]), + ); - const buf = await fromTableToBuffer(table); - expect(buf.byteLength).toBeGreaterThan(0); - const actual = tableFromIPC(buf); - const actualSchema = actual.schema; - expect(actualSchema.fields.length).toBe(3); + const data = [ + { + id: "config1", + config: { + database: "postgres", + connection: { host: "localhost" }, + }, + }, + { + id: "config2", + config: { database: "mysql" }, + }, + { + id: "config3", + }, + ]; - // Deep equality gets hung up on some very minor unimportant differences - // between arrow version 13 and 15 which isn't really what we're testing for - // and so we do our own comparison that just checks name/type/nullability - function compareFields(lhs: arrow15.Field, rhs: arrow15.Field) { - expect(lhs.name).toEqual(rhs.name); - expect(lhs.nullable).toEqual(rhs.nullable); - expect(lhs.typeId).toEqual(rhs.typeId); - if ("children" in lhs.type && lhs.type.children !== null) { - const lhsChildren = lhs.type.children as arrow15.Field[]; - lhsChildren.forEach((child: arrow15.Field, idx) => { - compareFields(child, rhs.type.children[idx]); - }); - } - } + const table = await convertToTable(data, undefined, { schema }); + + expect(table.numCols).toBe(2); + expect(table.numRows).toBe(3); + + const configColumn = table.getChild("config"); + expect(configColumn).toBeDefined(); + expect(configColumn?.type.toString()).toBe( + "Struct<{database:Utf8, connection:Struct<{host:Utf8, port:Int32, ssl:Struct<{enabled:Bool, cert_path:Utf8}>}>}>", + ); + }); + + it("will handle missing columns in Arrow table input when using embeddings", async function () { + const incompleteTable = makeArrowTable([ + { domain: "google.com", name: "Google" }, + { domain: "facebook.com", name: "Facebook" }, + ]); + + const schema = new Schema( + [ + new Field("domain", new Utf8(), true), + new Field("name", new Utf8(), true), + new Field("description", new Utf8(), true), + ], + new Map([["embedding_functions", JSON.stringify([])]]), + ); + + const buf = await fromDataToBuffer(incompleteTable, undefined, schema); + + expect(buf.byteLength).toBeGreaterThan(0); + + const retrievedTable = tableFromIPC(buf); + expect(retrievedTable.numCols).toBe(3); + expect(retrievedTable.numRows).toBe(2); + + const descriptionColumn = retrievedTable.getChild("description"); + expect(descriptionColumn).toBeDefined(); + expect(descriptionColumn?.nullCount).toBe(2); + expect(descriptionColumn?.toArray()).toEqual([null, null]); + + expect(retrievedTable.getChild("domain")?.toArray()).toEqual([ + "google.com", + "facebook.com", + ]); + expect(retrievedTable.getChild("name")?.toArray()).toEqual([ + "Google", + "Facebook", + ]); + }); + + it("should correctly retain values in nested struct fields", async function () { + const testData = [ + { + id: "doc1", + vector: [1, 2, 3], + metadata: { + filePath: "/path/to/file1.ts", + startLine: 10, + endLine: 20, + text: "function test() { return true; }", + }, + }, + { + id: "doc2", + vector: [4, 5, 6], + metadata: { + filePath: "/path/to/file2.ts", + startLine: 30, + endLine: 40, + text: "function test2() { return false; }", + }, + }, + ]; + + const table = makeArrowTable(testData); + + const metadataField = table.schema.fields.find( + (f) => f.name === "metadata", + ); + expect(metadataField).toBeDefined(); + // biome-ignore lint/suspicious/noExplicitAny: accessing fields in different Arrow versions + const childNames = metadataField?.type.children.map((c: any) => c.name); + expect(childNames).toEqual(["filePath", "startLine", "endLine", "text"]); + + const buf = await fromTableToBuffer(table); + const retrievedTable = tableFromIPC(buf); + + const rows = []; + for (let i = 0; i < retrievedTable.numRows; i++) { + rows.push(retrievedTable.get(i)); + } + + const firstRow = rows[0]; + expect(firstRow.id).toBe("doc1"); + expect(firstRow.vector.toJSON()).toEqual([1, 2, 3]); + expect(firstRow.metadata.filePath).toBe("/path/to/file1.ts"); + expect(firstRow.metadata.startLine).toBe(10); + expect(firstRow.metadata.endLine).toBe(20); + expect(firstRow.metadata.text).toBe("function test() { return true; }"); + }); + }); + + class DummyEmbedding extends EmbeddingFunction { + toJSON(): Partial { + return {}; + } + + async computeSourceEmbeddings(data: string[]): Promise { + return data.map(() => [0.0, 0.0]); + } + + ndims(): number { + return 2; + } + + embeddingDataType() { + return new Float16(); + } + } + + class DummyEmbeddingWithNoDimension extends EmbeddingFunction { + toJSON(): Partial { + return {}; + } + + embeddingDataType() { + return new Float16(); + } + + async computeSourceEmbeddings(data: string[]): Promise { + return data.map(() => [0.0, 0.0]); + } + } + const dummyEmbeddingConfig: EmbeddingFunctionConfig = { + sourceColumn: "string", + function: new DummyEmbedding(), + }; + + const dummyEmbeddingConfigWithNoDimension: EmbeddingFunctionConfig = { + sourceColumn: "string", + function: new DummyEmbeddingWithNoDimension(), + }; + + describe("convertToTable", function () { + it("will infer data types correctly", async function () { + await checkTableCreation( // biome-ignore lint/suspicious/noExplicitAny: - actualSchema.fields.forEach((field: any, idx: string | number) => { - compareFields(field, actualSchema.fields[idx]); - }); - }); + async (records) => await (convertToTable)(records), + true, + ); }); - describe("converting record batches to buffers", function () { - it("can convert to buffered record batch and back again", async function () { - const records = [ - { text: "dog", vector: [0.1, 0.2] }, - { text: "cat", vector: [0.3, 0.4] }, - ]; - const table = await convertToTable(records); - const batch = table.batches[0]; - - const buffer = await fromRecordBatchToBuffer(batch); - const result = await fromBufferToRecordBatch(buffer); - - expect(JSON.stringify(batch.toArray())).toEqual( - JSON.stringify(result?.toArray()), - ); - }); - - it("converting from buffer returns null if buffer has no record batches", async function () { - const result = await fromBufferToRecordBatch(Buffer.from([0x01, 0x02])); // bad data - expect(result).toEqual(null); - }); + it("will allow a schema to be provided", async function () { + await checkTableCreation( + async (records, _, schema) => + // biome-ignore lint/suspicious/noExplicitAny: + await (convertToTable)(records, undefined, { schema }), + false, + ); }); - describe("boolean null handling", function () { - it("should handle null values in nullable boolean fields", () => { - const { makeArrowTable } = require("../lancedb/arrow"); - const schema = new Schema([new Field("test", new arrow.Bool(), true)]); - - // Test with all null values - const data = [{ test: null }]; - const table = makeArrowTable(data, { schema }); - - expect(table.numRows).toBe(1); - expect(table.schema.names).toEqual(["test"]); - expect(table.getChild("test")!.get(0)).toBeNull(); - }); - - it("should handle mixed null and non-null boolean values", () => { - const { makeArrowTable } = require("../lancedb/arrow"); - const schema = new Schema([new Field("test", new Bool(), true)]); - - // Test with mixed values - const data = [{ test: true }, { test: null }, { test: false }]; - const table = makeArrowTable(data, { schema }); - - expect(table.numRows).toBe(3); - expect(table.getChild("test")!.get(0)).toBe(true); - expect(table.getChild("test")!.get(1)).toBeNull(); - expect(table.getChild("test")!.get(2)).toBe(false); - }); + it("will use the field order of any provided schema", async function () { + await checkTableCreation( + async (_, recordsReversed, schema) => + // biome-ignore lint/suspicious/noExplicitAny: + await (convertToTable)(recordsReversed, undefined, { schema }), + false, + ); }); - // Test for the undefined values bug fix - describe("undefined values handling", () => { - it("should handle mixed undefined and actual values", () => { - const schema = new Schema([ - new Field("text", new Utf8(), true), // nullable - new Field("number", new Int32(), true), // nullable - new Field("bool", new Bool(), true), // nullable - ]); + it("will make an empty table", async function () { + await checkTableCreation( + async (_, __, schema) => + // biome-ignore lint/suspicious/noExplicitAny: + await (convertToTable)([], undefined, { schema }), + false, + ); + }); - const data = [ - { text: undefined, number: 42, bool: true }, - { text: "hello", number: undefined, bool: false }, - { text: "world", number: 123, bool: undefined }, - ]; - const table = makeArrowTable(data, { schema }); + it("will apply embeddings", async function () { + const records = sampleRecords(); + const table = await convertToTable(records, dummyEmbeddingConfig); + expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe( + true, + ); + expect( + table.getChild("vector")?.type.children[0].type.toString(), + ).toEqual(new Float16().toString()); + }); - const result = table.toArray(); - expect(result).toHaveLength(3); - expect(result[0].text).toBe(null); - expect(result[0].number).toBe(42); - expect(result[0].bool).toBe(true); - expect(result[1].text).toBe("hello"); - expect(result[1].number).toBe(null); - expect(result[1].bool).toBe(false); - expect(result[2].text).toBe("world"); - expect(result[2].number).toBe(123); - expect(result[2].bool).toBe(null); + it("will fail if missing the embedding source column", async function () { + await expect( + convertToTable([{ id: 1 }], dummyEmbeddingConfig), + ).rejects.toThrow("'string' was not present"); + }); + + it("use embeddingDimension if embedding missing from table", async function () { + const schema = new Schema([new Field("string", new Utf8(), false)]); + // Simulate getting an empty Arrow table (minus embedding) from some other source + // In other words, we aren't starting with records + const table = makeEmptyTable(schema); + + // If the embedding specifies the dimension we are fine + await fromTableToBuffer(table, dummyEmbeddingConfig); + + // We can also supply a schema and should be ok + const schemaWithEmbedding = new Schema([ + new Field("string", new Utf8(), false), + new Field( + "vector", + new FixedSizeList(2, new Field("item", new Float16(), false)), + false, + ), + ]); + await fromTableToBuffer( + table, + dummyEmbeddingConfigWithNoDimension, + schemaWithEmbedding, + ); + + // Otherwise we will get an error + await expect( + fromTableToBuffer(table, dummyEmbeddingConfigWithNoDimension), + ).rejects.toThrow("does not specify `embeddingDimension`"); + }); + + it("will apply embeddings to an empty table", async function () { + const schema = new Schema([ + new Field("string", new Utf8(), false), + new Field( + "vector", + new FixedSizeList(2, new Field("item", new Float16(), false)), + false, + ), + ]); + const table = await convertToTable([], dummyEmbeddingConfig, { + schema, + }); + expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe( + true, + ); + expect( + table.getChild("vector")?.type.children[0].type.toString(), + ).toEqual(new Float16().toString()); + }); + + it("will complain if embeddings present but schema missing embedding column", async function () { + const schema = new Schema([new Field("string", new Utf8(), false)]); + await expect( + convertToTable([], dummyEmbeddingConfig, { schema }), + ).rejects.toThrow("column vector was missing"); + }); + + it("will skip embedding application if already applied", async function () { + const records = sampleRecords(); + const table = await convertToTable(records, dummyEmbeddingConfig); + + // fromTableToBuffer will try and apply the embeddings again + // but should skip since the column already has non-null values + const result = await fromTableToBuffer(table, dummyEmbeddingConfig); + expect(result.byteLength).toBeGreaterThan(0); + }); + }); + + describe("makeEmptyTable", function () { + it("will make an empty table", async function () { + await checkTableCreation( + // biome-ignore lint/suspicious/noExplicitAny: + async (_, __, schema) => (makeEmptyTable)(schema), + false, + ); + }); + + it("will make an empty table with a Map field", async function () { + const schema = new Schema([ + new Field( + "attributes", + new Map_( + new Field( + "entries", + new Struct([ + new Field("key", new Utf8(), false), + new Field("value", new Int32(), true), + ]), + false, + ), + true, + ), + ), + ]); + + const table = makeEmptyTable(schema); + + expectValidMapField(table.schema.fields[0]); + + const buffer = await fromTableToBuffer(table); + const roundTripped = tableFromIPC(buffer); + + expectValidMapField(roundTripped.schema.fields[0]); + }); + + it("preserves string schema metadata", function () { + const metadata = new Map([["source", "fixture"]]); + const schema = new Schema( + [new Field("value", new Int32(), true)], + metadata, + ); + + expect(makeEmptyTable(schema).schema.metadata.get("source")).toBe( + "fixture", + ); + }); + + it.each([ + ["non-string keys", new Map([[42, "fixture"]])], + ["non-string values", new Map([["source", 42]])], + ["non-string keys and values", new Map([[42, false]])], + ])("rejects schema metadata with %s", function (_, metadataLike) { + const metadata = metadataLike as unknown as Map; + const schema = new Schema( + [new Field("value", new Int32(), true)], + metadata, + ); + + expect(() => makeEmptyTable(schema)).toThrow( + "Expected metadata, if present, to be a Map but it had non-string keys or values", + ); + }); + }); + + describe("when using two versions of arrow", function () { + it("preserves a dictionary shared by multiple fields", async function () { + const values = ["alpha", "beta", "alpha"]; + const dictionaryVector = vectorFromArray(values); + const batch = new ArrowRecordBatch({ + first: dictionaryVector.data[0], + second: dictionaryVector.data[0], + }); + const table = new ArrowTable([batch]); + + const sanitized = sanitizeTable(table); + expect([...sanitized.getChild("first")!]).toEqual(values); + expect([...sanitized.getChild("second")!]).toEqual(values); + const firstType = sanitized.schema.fields[0].type as { + dictionary: unknown; + }; + const secondType = sanitized.schema.fields[1].type as { + dictionary: unknown; + }; + expect(secondType.dictionary).toBe(firstType.dictionary); + expect(sanitized.batches[0].data.children[1].dictionary).toBe( + sanitized.batches[0].data.children[0].dictionary, + ); + + const buf = await fromDataToBuffer(table); + const actual = currentTableFromIPC(buf); + expect([...actual.getChild("first")!]).toEqual(values); + expect([...actual.getChild("second")!]).toEqual(values); + }); + + it("preserves shared dictionary data from another Arrow version", async function () { + const values = ["alpha", "beta", "alpha"]; + const dictionaryVector = vectorFromArray(values); + const firstBatch = new ArrowRecordBatch({ + label: dictionaryVector.slice(0, 2).data[0], + }); + const secondBatch = new ArrowRecordBatch({ + label: dictionaryVector.slice(2).data[0], + }); + const table = new ArrowTable([firstBatch, secondBatch]); + + const sanitized = sanitizeTable(table); + expect([...sanitized.getChild("label")!]).toEqual(values); + + const dictionaries = sanitized.batches.map( + (batch) => batch.data.children[0].dictionary, + ); + expect(dictionaries[0]).toBeInstanceOf(CurrentVector); + expect(dictionaries[1]).toBe(dictionaries[0]); + + const buf = await fromDataToBuffer(table); + const actual = currentTableFromIPC(buf); + expect([...actual.getChild("label")!]).toEqual(values); + }); + + it("preserves shared chunks in growing dictionaries", async function () { + const type = new Dictionary(new Utf8(), new Int32(), 42, false); + const firstDictionary = vectorFromArray(["alpha", "beta"], new Utf8()); + const secondDictionary = firstDictionary.concat( + vectorFromArray(["gamma"], new Utf8()), + ); + const firstData = arrowMakeData({ + type, + data: Int32Array.from([0, 1]), + dictionary: firstDictionary, + }); + const secondData = arrowMakeData({ + type, + data: Int32Array.from([2]), + dictionary: secondDictionary, + }); + const table = new ArrowTable([ + new ArrowRecordBatch({ label: firstData }), + new ArrowRecordBatch({ label: secondData }), + ]); + + const sanitized = sanitizeTable(table); + const expected = ["alpha", "beta", "gamma"]; + expect([...sanitized.getChild("label")!]).toEqual(expected); + const firstLocalDictionary = + sanitized.batches[0].data.children[0].dictionary!; + const secondLocalDictionary = + sanitized.batches[1].data.children[0].dictionary!; + expect(secondLocalDictionary.data[0]).toBe(firstLocalDictionary.data[0]); + + const buf = await fromTableToBuffer(sanitized); + const actual = currentTableFromIPC(buf); + expect([...actual.getChild("label")!]).toEqual(expected); + }); + + it("can serialize list data from another Arrow version", async function () { + const values = [["anime", "action"], [], null]; + const vector = vectorFromArray( + values, + new List(new Field("item", new Utf8(), true)), + ); + const table = new ArrowTable({ tags: vector }); + + const buf = await fromDataToBuffer(table); + const actual = currentTableFromIPC(buf); + const actualTags = actual.getChild("tags"); + + expect(actualTags?.get(0)?.toJSON()).toEqual(values[0]); + expect(actualTags?.get(1)?.toJSON()).toEqual(values[1]); + expect(actualTags?.get(2)).toBeNull(); + }); + + it("can still import data", async function () { + const schema = new arrow15.Schema([ + new arrow15.Field("id", new arrow15.Int32()), + new arrow15.Field( + "vector", + new arrow15.FixedSizeList( + 1024, + new arrow15.Field("item", new arrow15.Float32(), true), + ), + ), + new arrow15.Field( + "struct", + new arrow15.Struct([ + new arrow15.Field( + "nested", + new arrow15.Dictionary( + new arrow15.Utf8(), + new arrow15.Int32(), + 1, + true, + ), + ), + new arrow15.Field( + "ts_with_tz", + new arrow15.TimestampNanosecond("some_tz"), + ), + new arrow15.Field( + "ts_no_tz", + new arrow15.TimestampNanosecond(null), + ), + ]), + ), + // biome-ignore lint/suspicious/noExplicitAny: skip + ]) as any; + schema.metadataVersion = arrow15.MetadataVersion.V5; + const table = makeArrowTable([], { schema }); + + const buf = await fromTableToBuffer(table); + expect(buf.byteLength).toBeGreaterThan(0); + const actual = tableFromIPC(buf); + const actualSchema = actual.schema; + expect(actualSchema.fields.length).toBe(3); + + // Deep equality gets hung up on some very minor unimportant differences + // between arrow version 13 and 15 which isn't really what we're testing for + // and so we do our own comparison that just checks name/type/nullability + function compareFields(lhs: arrow15.Field, rhs: arrow15.Field) { + expect(lhs.name).toEqual(rhs.name); + expect(lhs.nullable).toEqual(rhs.nullable); + expect(lhs.typeId).toEqual(rhs.typeId); + if ("children" in lhs.type && lhs.type.children !== null) { + const lhsChildren = lhs.type.children as arrow15.Field[]; + lhsChildren.forEach((child: arrow15.Field, idx) => { + compareFields(child, rhs.type.children[idx]); + }); + } + } + // biome-ignore lint/suspicious/noExplicitAny: + actualSchema.fields.forEach((field: any, idx: string | number) => { + compareFields(field, actualSchema.fields[idx]); }); }); - }, -); + }); + + describe("converting record batches to buffers", function () { + it("can convert to buffered record batch and back again", async function () { + const records = [ + { text: "dog", vector: [0.1, 0.2] }, + { text: "cat", vector: [0.3, 0.4] }, + ]; + const table = await convertToTable(records); + const batch = table.batches[0]; + + const buffer = await fromRecordBatchToBuffer(batch); + const result = await fromBufferToRecordBatch(buffer); + + expect(JSON.stringify(batch.toArray())).toEqual( + JSON.stringify(result?.toArray()), + ); + }); + + it("converting from buffer returns null if buffer has no record batches", async function () { + const result = await fromBufferToRecordBatch(Buffer.from([0x01, 0x02])); // bad data + expect(result).toEqual(null); + }); + }); + + describe("boolean null handling", function () { + it("should handle null values in nullable boolean fields", () => { + const { makeArrowTable } = require("../lancedb/arrow"); + const schema = new Schema([new Field("test", new arrow.Bool(), true)]); + + // Test with all null values + const data = [{ test: null }]; + const table = makeArrowTable(data, { schema }); + + expect(table.numRows).toBe(1); + expect(table.schema.names).toEqual(["test"]); + expect(table.getChild("test")!.get(0)).toBeNull(); + }); + + it("should handle mixed null and non-null boolean values", () => { + const { makeArrowTable } = require("../lancedb/arrow"); + const schema = new Schema([new Field("test", new Bool(), true)]); + + // Test with mixed values + const data = [{ test: true }, { test: null }, { test: false }]; + const table = makeArrowTable(data, { schema }); + + expect(table.numRows).toBe(3); + expect(table.getChild("test")!.get(0)).toBe(true); + expect(table.getChild("test")!.get(1)).toBeNull(); + expect(table.getChild("test")!.get(2)).toBe(false); + }); + }); + + // Test for the undefined values bug fix + describe("undefined values handling", () => { + it("should handle mixed undefined and actual values", () => { + const schema = new Schema([ + new Field("text", new Utf8(), true), // nullable + new Field("number", new Int32(), true), // nullable + new Field("bool", new Bool(), true), // nullable + ]); + + const data = [ + { text: undefined, number: 42, bool: true }, + { text: "hello", number: undefined, bool: false }, + { text: "world", number: 123, bool: undefined }, + ]; + const table = makeArrowTable(data, { schema }); + + const result = table.toArray(); + expect(result).toHaveLength(3); + expect(result[0].text).toBe(null); + expect(result[0].number).toBe(42); + expect(result[0].bool).toBe(true); + expect(result[1].text).toBe("hello"); + expect(result[1].number).toBe(null); + expect(result[1].bool).toBe(false); + expect(result[2].text).toBe("world"); + expect(result[2].number).toBe(123); + expect(result[2].bool).toBe(null); + }); + }); +}); diff --git a/nodejs/__test__/arrow_versions.ts b/nodejs/__test__/arrow_versions.ts new file mode 100644 index 000000000..bfbbfa5ef --- /dev/null +++ b/nodejs/__test__/arrow_versions.ts @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// Every Apache Arrow release the package accepts as a peer dependency, as +// separately installed copies so the tests can feed objects built by one +// Arrow version into a LanceDB running on another. When a new Arrow major +// is released, add the alias to `package.json` and extend this list; the +// test files pick the change up from here. +import * as arrow15 from "apache-arrow-15"; +import * as arrow16 from "apache-arrow-16"; +import * as arrow17 from "apache-arrow-17"; +import * as arrow18 from "apache-arrow-18"; +import * as arrow19 from "apache-arrow-19"; +import * as arrow20 from "apache-arrow-20"; +import * as arrow21 from "apache-arrow-21"; + +export { arrow15, arrow16, arrow17, arrow18, arrow19, arrow20, arrow21 }; + +export type ApacheArrow = + | typeof arrow15 + | typeof arrow16 + | typeof arrow17 + | typeof arrow18 + | typeof arrow19 + | typeof arrow20 + | typeof arrow21; + +export const arrowVersions: ApacheArrow[] = [ + arrow15, + arrow16, + arrow17, + arrow18, + arrow19, + arrow20, + arrow21, +]; + +/** The newest supported Arrow release, for tests that only need one. */ +export const latestArrow = arrow21; diff --git a/nodejs/__test__/materialized_view.test.ts b/nodejs/__test__/materialized_view.test.ts index 5e5c08745..6739320aa 100644 --- a/nodejs/__test__/materialized_view.test.ts +++ b/nodejs/__test__/materialized_view.test.ts @@ -52,9 +52,19 @@ describe("materialized views", () => { "SELECT * FROM people", ); + for (const [format, fn] of [ + [3, "vector_duplicate_pairs"], + [4, "vector_dedup"], + ]) { + const native = `SELECT * FROM ${fn}('images', 3, 'phash', 4)`; + expect(read(JSON.stringify({ format, query: native })).query).toBe( + native, + ); + } + // A newer writer's layout is reported, never guessed at. for (const newer of [ - `{"format":3,"query":${JSON.stringify(query)}}`, + `{"format":5,"query":${JSON.stringify(query)}}`, '{"kind":"select_v3","source_table":"people"}', ]) { expect(() => read(newer)).toThrow(/cannot refresh/); diff --git a/nodejs/__test__/query.test.ts b/nodejs/__test__/query.test.ts index 5f3e68b16..2cfecf823 100644 --- a/nodejs/__test__/query.test.ts +++ b/nodejs/__test__/query.test.ts @@ -110,6 +110,129 @@ describe("Query outputSchema", () => { }); }); +describe("Query empty results", () => { + let tmpDir: tmp.DirResult; + let table: Table; + + beforeEach(async () => { + tmpDir = tmp.dirSync({ unsafeCleanup: true }); + const db = await connect(tmpDir.name); + const schema = new Schema( + [ + new Field("id", new Int64(), false), + new Field( + "text", + new Utf8(), + true, + new Map([["description", "searchable text"]]), + ), + new Field( + "vector", + new FixedSizeList(2, new Field("item", new Float32())), + true, + ), + ], + new Map([["source", "empty-query-regression"]]), + ); + table = await db.createTable( + "test", + makeArrowTable([{ id: 1n, text: "hello", vector: [1, 2] }], { schema }), + ); + }); + + afterEach(() => { + tmpDir.removeCallback(); + }); + + it.each([ + { + name: "filtered scan", + query: (table: Table) => table.query().where("id < 0"), + fields: ["id", "text", "vector"], + }, + { + name: "zero limit", + query: (table: Table) => table.query().limit(0), + fields: ["id", "text", "vector"], + }, + { + name: "column projection", + query: (table: Table) => + table.query().where("id < 0").select(["text", "id"]), + fields: ["text", "id"], + }, + { + name: "dynamic projection", + query: (table: Table) => + table.query().where("id < 0").select({ doubled: "id * 2" }), + fields: ["doubled"], + }, + { + name: "filtered vector search", + query: (table: Table) => + table.search([0.5, 0.5]).where("id < 0").limit(3), + fields: ["id", "text", "vector", "_distance"], + }, + { + name: "vector projection with row id", + query: (table: Table) => + table + .vectorSearch([0.5, 0.5]) + .where("id < 0") + .select(["id"]) + .withRowId(), + fields: ["id", "_distance", "_rowid"], + }, + { + name: "fast search without an index", + query: (table: Table) => table.search([0.5, 0.5]).fastSearch(), + fields: ["id", "text", "vector", "_rowid", "_distance"], + }, + { + name: "take with no offsets", + query: (table: Table) => table.takeOffsets([]).select(["text"]), + fields: ["text"], + }, + ])("preserves the output schema for $name", async ({ query, fields }) => { + const result = await query(table).toArrow({ maxBatchLength: 1 }); + + expect(result.numRows).toBe(0); + expect(result.schema.fields.map((field) => field.name)).toEqual(fields); + for (const name of fields) { + expect(result.getChild(name)).not.toBeNull(); + expect(result.getChild(name)?.length).toBe(0); + } + }); + + it("preserves types, nullability, and metadata", async () => { + const result = await table.query().where("id < 0").toArrow(); + const schema = await table.schema(); + + expect(result.schema).toEqual(schema); + expect(result.getChild("id")?.type.toString()).toBe("Int64"); + expect(result.schema.fields[0].nullable).toBe(false); + expect(result.schema.fields[1].metadata.get("description")).toBe( + "searchable text", + ); + expect(result.schema.metadata.get("source")).toBe("empty-query-regression"); + }); + + it("preserves the score column for full-text search with no matches", async () => { + await table.createIndex("text", { config: Index.fts() }); + const result = await table + .search("missing", "fts") + .select(["id"]) + .toArrow(); + + expect(result.numRows).toBe(0); + expect(result.schema.fields.map((field) => field.name)).toEqual([ + "id", + "_score", + ]); + expect(result.getChild("_score")?.type.toString()).toBe("Float32"); + }); +}); + describe("Search pagination", () => { let tmpDir: tmp.DirResult; let table: Table; diff --git a/nodejs/__test__/registry.test.ts b/nodejs/__test__/registry.test.ts index 973ad8a25..5c92941d6 100644 --- a/nodejs/__test__/registry.test.ts +++ b/nodejs/__test__/registry.test.ts @@ -3,13 +3,10 @@ import * as apiArrow from "apache-arrow"; -import * as arrow15 from "apache-arrow-15"; -import * as arrow16 from "apache-arrow-16"; -import * as arrow17 from "apache-arrow-17"; -import * as arrow18 from "apache-arrow-18"; - import * as tmp from "tmp"; +import { arrowVersions } from "./arrow_versions"; + import { connect } from "../lancedb"; import { EmbeddingFunction, @@ -18,7 +15,7 @@ import { } from "../lancedb/embedding"; import { getRegistry, register } from "../lancedb/embedding/registry"; -describe.each([arrow15, arrow16, arrow17, arrow18])("LanceSchema", (arrow) => { +describe.each(arrowVersions)("LanceSchema", (arrow) => { test("should preserve input order", async () => { const schema = LanceSchema({ id: new arrow.Int32(), @@ -29,7 +26,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])("LanceSchema", (arrow) => { }); }); -describe.each([arrow15, arrow16, arrow17, arrow18])("Registry", (arrow) => { +describe.each(arrowVersions)("Registry", (arrow) => { let tmpDir: tmp.DirResult; beforeEach(() => { tmpDir = tmp.dirSync({ unsafeCleanup: true }); @@ -240,7 +237,7 @@ describe("Registry.setVar", () => { return data.map(() => [1, 2, 3]); } embeddingDataType() { - return new arrow18.Float32() as apiArrow.Float; + return new apiArrow.Float32(); } protected getSensitiveKeys() { return ["secretKey"]; diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index cb0047f37..ec259b79b 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -14,6 +14,14 @@ import { TlsConfig, connect, } from "../lancedb"; +import { + Table as ArrowTable, + Field, + Int64, + RecordBatch, + Schema, + tableToIPC, +} from "../lancedb/arrow"; import { HeaderProvider, OAuthHeaderProvider, @@ -85,6 +93,161 @@ async function withMockDatabase( } describe("remote connection", () => { + it("rejects nprobes(0) before sending a query", async () => { + const requests: string[] = []; + await withMockDatabase( + (req, res) => { + requests.push(req.url ?? ""); + if (req.url === "/v1/table/test/describe/") { + res + .writeHead(200, { "Content-Type": "application/json" }) + .end(JSON.stringify({ version: 1, schema: { fields: [] } })); + } else { + res.writeHead(404).end(); + } + }, + async (db) => { + const table = await db.openTable("test"); + expect(() => table.vectorSearch([0, 0]).nprobes(0)).toThrow( + "Invalid input, nprobes must be greater than 0", + ); + expect(() => + table.query().nearestTo([0, 0]).fullTextSearch("dog").nprobes(0), + ).toThrow("Invalid input, nprobes must be greater than 0"); + await expect( + table + .vectorSearch(Promise.resolve([0, 0])) + .nprobes(0) + .toArrow(), + ).rejects.toThrow("Invalid input, nprobes must be greater than 0"); + }, + ); + expect(requests).toEqual(["/v1/table/test/describe/"]); + }); + + it.each([false, true])( + "preserves an empty query's schema with an empty batch: %s", + async (withEmptyBatch) => { + const schema = new Schema( + [new Field("doubled", new Int64(), false)], + new Map([["source", "query-output"]]), + ); + const result = new ArrowTable( + schema, + withEmptyBatch ? [new RecordBatch(schema, undefined)] : [], + ); + const response = Buffer.from(tableToIPC(result, "stream")); + let queryRequests = 0; + + await withMockDatabase( + (req, res) => { + if (req.url?.endsWith("/describe/")) { + res.writeHead(200, { "Content-Type": "application/json" }).end( + JSON.stringify({ + name: "items", + version: 1, + schema: { + fields: [ + { name: "id", type: { type: "int64" }, nullable: false }, + ], + }, + }), + ); + } else if (req.url?.endsWith("/query/")) { + queryRequests++; + req.resume(); + req.on("end", () => { + res + .writeHead(200, { + "Content-Type": "application/vnd.apache.arrow.stream", + }) + .end(response); + }); + } else { + res.writeHead(404).end(); + } + }, + async (db) => { + const table = await db.openTable("items"); + const result = await table + .query() + .where("id < 0") + .select({ doubled: "id * 2" }) + .toArrow(); + + expect(result.numRows).toBe(0); + expect(result.schema).toEqual(schema); + expect(result.getChild("doubled")?.length).toBe(0); + }, + ); + + expect(queryRequests).toBe(1); + }, + ); + + it("rejects the external blob opt-in without blocking regular adds", async () => { + let insertRequests = 0; + const describeRequests: string[] = []; + + await withMockDatabase( + (req, res) => { + const requestPath = req.url ?? ""; + if (requestPath.endsWith("/describe/")) { + describeRequests.push(requestPath); + res.writeHead(200, { "Content-Type": "application/json" }).end( + JSON.stringify({ + name: "items", + version: 1, + schema: { + fields: [ + { + name: "id", + type: { type: "int64" }, + nullable: true, + }, + ], + }, + }), + ); + return; + } + if (requestPath.endsWith("/insert/")) { + insertRequests++; + req.resume(); + req.on("end", () => { + res + .writeHead(200, { "Content-Type": "application/json" }) + .end(JSON.stringify({ version: insertRequests + 1 })); + }); + return; + } + res.writeHead(404).end(); + }, + async (db) => { + const table = await db.openTable("items"); + + await expect( + table.add([{ id: 1n }], { + allowExternalBlobOutsideBases: true, + }), + ).rejects.toThrow("only supported on local tables"); + expect(insertRequests).toBe(0); + + await expect(table.add([{ id: 2n }])).resolves.toMatchObject({ + version: 2, + }); + await expect( + table.add([{ id: 3n }], { + allowExternalBlobOutsideBases: false, + }), + ).resolves.toMatchObject({ version: 3 }); + }, + ); + + expect(describeRequests.length).toBeGreaterThan(0); + expect(insertRequests).toBe(2); + }); + it("lists materialized views through the namespace route", async () => { await withMockDatabase( (req, res) => { diff --git a/nodejs/__test__/sanitize.test.ts b/nodejs/__test__/sanitize.test.ts index 587805d53..97856a55b 100644 --- a/nodejs/__test__/sanitize.test.ts +++ b/nodejs/__test__/sanitize.test.ts @@ -2,7 +2,13 @@ // SPDX-FileCopyrightText: Copyright The LanceDB Authors import * as arrow from "../lancedb/arrow"; -import { sanitizeField, sanitizeMap, sanitizeType } from "../lancedb/sanitize"; +import { + sanitizeField, + sanitizeMap, + sanitizeSchema, + sanitizeType, +} from "../lancedb/sanitize"; +import { arrow21 } from "./arrow_versions"; describe("sanitize", function () { describe("sanitizeType function", function () { @@ -59,6 +65,8 @@ describe("sanitize", function () { ["durationmicrosecond", new arrow.DurationMicrosecond()], ["durationmillisecond", new arrow.DurationMillisecond()], ["durationsecond", new arrow.DurationSecond()], + ["largebinary", new arrow.LargeBinary()], + ["largeutf8", new arrow.LargeUtf8()], ] as const; it.each(allTypeNameTestCases)( @@ -89,6 +97,110 @@ describe("sanitize", function () { "Unrecognized type name in schema: invalid_type", ); }); + + // These types only exist in Arrow 20 (IntervalMonthDayNano) and 21 (the + // rest), so the objects come from a second copy of the newest release. The + // library itself may be running on an older Arrow (see the CI matrix), in + // which case sanitizing them has to fail with a clear message instead. + const hostArrow = arrow as unknown as Record; + const hostSupportsNewerTypes = typeof hostArrow.Utf8View === "function"; + const whenSupported = hostSupportsNewerTypes ? describe : describe.skip; + const whenUnsupported = hostSupportsNewerTypes ? describe.skip : describe; + + const newerTypeTestCases = [ + ["Utf8View", () => new arrow21.Utf8View()], + ["BinaryView", () => new arrow21.BinaryView()], + ["IntervalMonthDayNano", () => new arrow21.IntervalMonthDayNano()], + [ + "LargeList", + () => + new arrow21.LargeList(new arrow21.Field("item", new arrow21.Utf8())), + ], + ] as const; + + whenSupported("types added in Arrow 20 and 21", function () { + it.each(newerTypeTestCases)( + "should sanitize a %s type from another copy of Arrow", + function (name, makeForeign) { + const foreign = makeForeign(); + const result = sanitizeType(foreign); + // Interval subclasses share `Type.Interval`, so the sanitized type + // is the generic `Interval` carrying the same unit, as for the + // other units. + const expected = name === "IntervalMonthDayNano" ? "Interval" : name; + expect(result).toBeInstanceOf(hostArrow[expected] as () => unknown); + expect(result.typeId).toBe(foreign.typeId); + expect(result.toString()).toBe(foreign.toString()); + }, + ); + + it.each([ + ["binaryview", "BinaryView"], + ["utf8view", "Utf8View"], + ["intervalmonthdaynano", "Interval"], + ])('should map type name "%s" to %s', function (name, expected) { + expect(sanitizeType(name)).toBeInstanceOf( + hostArrow[expected] as () => unknown, + ); + }); + + it("should sanitize the child field of a LargeList", function () { + const foreign = new arrow21.LargeList( + new arrow21.Field("item", new arrow21.Int32(), false), + ); + const result = sanitizeType(foreign) as arrow.List; + expect(result.children[0]).toBeInstanceOf(arrow.Field); + expect(result.children[0].name).toBe("item"); + expect(result.children[0].nullable).toBe(false); + const child = result.children[0].type; + expect(arrow.DataType.isInt(child)).toBe(true); + expect((child as arrow.Int).bitWidth).toBe(32); + expect((child as arrow.Int).isSigned).toBe(true); + }); + }); + + whenUnsupported("types newer than the installed Arrow", function () { + it.each( + newerTypeTestCases.filter(([name]) => name !== "IntervalMonthDayNano"), + )( + "should explain that %s needs a newer apache-arrow", + function (name, makeForeign) { + expect(() => sanitizeType(makeForeign())).toThrow( + `The Arrow type ${name} requires apache-arrow`, + ); + }, + ); + + it("should sanitize IntervalMonthDayNano to a generic Interval", () => { + // The subclass shares `Type.Interval`, which every supported Arrow + // release has; only the unit is new. + const result = sanitizeType(new arrow21.IntervalMonthDayNano()); + expect(result).toBeInstanceOf(arrow.Interval); + expect((result as arrow.Interval).unit).toBe( + arrow21.IntervalUnit.MONTH_DAY_NANO, + ); + }); + }); + + it("should rebuild a schema from a second copy of the same Arrow version", function () { + // Arrow 21 makes `instanceof Schema` succeed across copies of the library + // but not `instanceof `; the schema must still be rebuilt. + const foreign = new arrow21.Schema([ + new arrow21.Field("x", new arrow21.Float32(), true), + ]); + // Only Arrow 21+ overrides `Symbol.hasInstance` this way. + const crossCopyInstanceof = Object.getOwnPropertySymbols( + arrow.Schema, + ).includes(Symbol.hasInstance); + expect(foreign instanceof arrow.Schema).toBe(crossCopyInstanceof); + const result = sanitizeSchema(foreign); + expect(result).not.toBe(foreign); + const type = result.fields[0].type; + expect(type).toBeInstanceOf(arrow.DataType); + expect(arrow.DataType.isFloat(type)).toBe(true); + expect((type as arrow.Float).precision).toBe(arrow.Precision.SINGLE); + expect(() => arrow.makeBuilder({ type })).not.toThrow(); + }); }); describe("sanitizeField function", function () { diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index ac8904c16..c17b0315a 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -2,14 +2,11 @@ // SPDX-FileCopyrightText: Copyright The LanceDB Authors import * as fs from "fs"; +import { getEventListeners } from "node:events"; +import { pathToFileURL } from "node:url"; import * as path from "path"; import * as tmp from "tmp"; -import * as arrow15 from "apache-arrow-15"; -import * as arrow16 from "apache-arrow-16"; -import * as arrow17 from "apache-arrow-17"; -import * as arrow18 from "apache-arrow-18"; - import { AutoQuery, Connection, @@ -54,19 +51,21 @@ import { instanceOfFullTextQuery, } from "../lancedb/query"; import { LocalTable } from "../lancedb/table"; +import { + type ApacheArrow, + arrow18, + arrowVersions, + latestArrow, +} from "./arrow_versions"; -describe.each([arrow15, arrow16, arrow17, arrow18])( +describe.each(arrowVersions)( "Given a table", // biome-ignore lint/suspicious/noExplicitAny: (arrow: any) => { let tmpDir: tmp.DirResult; let table: Table; - const schema: - | import("apache-arrow-15").Schema - | import("apache-arrow-16").Schema - | import("apache-arrow-17").Schema - | import("apache-arrow-18").Schema = new arrow.Schema([ + const schema: InstanceType = new arrow.Schema([ new arrow.Field("id", new arrow.Float64(), true), ]); @@ -91,6 +90,38 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( await expect(table.countRows()).resolves.toBe(3); }); + it("creates and adds Arrow tables with large string and binary columns", async () => { + const data = new arrow.Table({ + text: arrow.vectorFromArray(["alpha", "beta"], new arrow.LargeUtf8()), + bytes: arrow.vectorFromArray( + [Buffer.from("one"), Buffer.from("two")], + new arrow.LargeBinary(), + ), + }); + const conn = await connect(tmpDir.name); + const largeTable = await conn.createTable("large_columns", data); + await largeTable.add(data); + + const rows = await largeTable.query().toArray(); + expect(rows.map((row) => row.text)).toEqual([ + "alpha", + "beta", + "alpha", + "beta", + ]); + expect(rows.map((row) => Buffer.from(row.bytes).toString())).toEqual([ + "one", + "two", + "one", + "two", + ]); + const fields = (await largeTable.schema()).fields; + expect(fields.map((field) => field.type.typeId)).toEqual([ + arrow.Type.LargeUtf8, + arrow.Type.LargeBinary, + ]); + }); + it("should support a foreign Float64 vector schema end to end", async () => { const conn = await connect(tmpDir.name); const schema = new arrow.Schema([ @@ -416,6 +447,26 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( expect(await table.countRows("id == 10")).toBe(1); }); + it("should let me update float values to NaN and Infinity", async () => { + const db = await connect(tmpDir.name); + const floats = await db.createTable("floats", [ + { id: 1, price: 1.5 }, + { id: 2, price: 2.5 }, + { id: 3, price: 3.5 }, + { id: 4, price: 4.5 }, + ]); + await floats.update({ values: { price: NaN }, where: "id = 1" }); + await floats.update({ values: { price: Infinity }, where: "id = 2" }); + await floats.update({ values: { price: -Infinity }, where: "id = 3" }); + + const rows = await floats.query().toArray(); + const price = (id: number) => rows.find((row) => row.id === id).price; + expect(price(1)).toBeNaN(); + expect(price(2)).toBe(Infinity); + expect(price(3)).toBe(-Infinity); + expect(price(4)).toBe(4.5); + }); + // https://github.com/lancedb/lancedb/issues/1293 test.each([new arrow.Float16(), new arrow.Float32(), new arrow.Float64()])( "can create empty table with non default float type: %s", @@ -688,6 +739,114 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( ); // https://github.com/lancedb/lancedb/issues/1963 +// These types were added in Arrow 21, so they are built with a second copy of +// that release rather than the per-version matrix above, and reading them back +// needs the library's own Arrow to know them too. +const hostArrowSupportsViewTypes = + typeof (arrow as unknown as Record).Utf8View === "function"; +(hostArrowSupportsViewTypes ? it : it.skip)( + "creates and adds tables with LargeList, Utf8View and BinaryView columns", + async () => { + const tmpDir = tmp.dirSync({ unsafeCleanup: true }); + const db = await connect(tmpDir.name); + const makeData = () => + new latestArrow.Table({ + id: latestArrow.vectorFromArray([1, 2], new latestArrow.Int32()), + tags: latestArrow.vectorFromArray( + [["a", "b"], ["c"]], + new latestArrow.LargeList( + new latestArrow.Field("item", new latestArrow.Utf8(), true), + ), + ), + text: latestArrow.vectorFromArray( + ["short", "a string that is longer than the twelve inline bytes"], + new latestArrow.Utf8View(), + ), + bytes: latestArrow.vectorFromArray( + [new Uint8Array([1, 2, 3]), new Uint8Array(20).fill(7)], + new latestArrow.BinaryView(), + ), + }); + const table = await db.createTable("views", makeData()); + await table.add(makeData()); + + const schema = await table.schema(); + // Take the enum from the Arrow 21 copy: the host Arrow may predate + // `Type.LargeList`, and this file still has to type-check there. + expect(schema.fields.map((f) => f.type.typeId)).toEqual([ + latestArrow.Type.Int, + latestArrow.Type.LargeList, + // Lance stores the view types as their offset-based equivalents. + latestArrow.Type.Utf8, + latestArrow.Type.Binary, + ]); + const rows = await table.query().toArrow(); + expect(rows.numRows).toBe(4); + const row = rows.get(1)!.toJSON(); + expect(row.id).toBe(2); + expect(row.tags.toJSON()).toEqual(["c"]); + expect(row.text).toBe( + "a string that is longer than the twelve inline bytes", + ); + expect(Array.from(row.bytes)).toEqual(Array(20).fill(7)); + + // Arrow JS 21 serializes a *sliced* view column with a truncated views + // buffer; the library rebuilds view columns as Utf8/Binary so this works. + const sliced = makeData().slice(1, 2); + expect(sliced.getChild("text")!.data[0].offset).toBe(1); + const slicedTable = await db.createTable("views_sliced", sliced); + await slicedTable.add(sliced); + const slicedRows = await slicedTable.query().toArrow(); + expect(slicedRows.numRows).toBe(2); + for (const i of [0, 1]) { + const r = slicedRows.get(i)!.toJSON(); + expect(r.id).toBe(2); + expect(r.text).toBe( + "a string that is longer than the twelve inline bytes", + ); + expect(Array.from(r.bytes)).toEqual(Array(20).fill(7)); + } + + // Regression: a field with an integer-like name. JavaScript enumerates + // such keys first, so a key-ordered rebuild would pair "1" with the text + // column's values. The batches are built explicitly to pin field order. + const numericSchema = new latestArrow.Schema([ + new latestArrow.Field("text", new latestArrow.Utf8View(), true), + new latestArrow.Field("1", new latestArrow.Utf8View(), true), + new latestArrow.Field("n", new latestArrow.Int32(), true), + ]); + const column = ( + values: unknown[], + type: InstanceType, + ) => latestArrow.vectorFromArray(values, type).data[0]; + const numericBatch = new latestArrow.RecordBatch( + numericSchema, + latestArrow.makeData({ + type: new latestArrow.Struct(numericSchema.fields), + length: 3, + nullCount: 0, + children: [ + column(["t0", "t1", "t2"], new latestArrow.Utf8View()), + column(["one0", "one1", "one2"], new latestArrow.Utf8View()), + column([0, 1, 2], new latestArrow.Int32()), + ], + }), + ); + const numericData = new latestArrow.Table(numericSchema, [numericBatch]); + const numericTable = await db.createTable("views_numeric", numericData); + await numericTable.add(numericData.slice(1)); + const numericRows = (await numericTable.query().toArrow()).toArray(); + expect(numericRows.map((r) => r.toJSON())).toEqual([ + { text: "t0", "1": "one0", n: 0 }, + { text: "t1", "1": "one1", n: 1 }, + { text: "t2", "1": "one2", n: 2 }, + { text: "t1", "1": "one1", n: 1 }, + { text: "t2", "1": "one2", n: 2 }, + ]); + tmpDir.removeCallback(); + }, +); + it("should query documents with LangChain PDF metadata", async () => { const tmpDir = tmp.dirSync({ unsafeCleanup: true }); try { @@ -1052,6 +1211,12 @@ describe("When creating an index", () => { .toArrow(); expect(rst.numRows).toBe(2); + expect(() => tbl.vectorSearch(queryVec).nprobes(0)).toThrow( + "Invalid input, nprobes must be greater than 0", + ); + expect(() => + tbl.query().nearestTo(queryVec).fullTextSearch("dog").nprobes(0), + ).toThrow("Invalid input, nprobes must be greater than 0"); expect(() => tbl.query().nearestTo(queryVec).minimumNprobes(0)).toThrow( "Invalid input, minimum_nprobes must be greater than 0", ); @@ -2429,6 +2594,118 @@ describe("when dealing with blob columns", () => { await expect(table.countRows()).resolves.toBe(0); }); + it("requires an explicit opt-in for blob URIs outside registered bases", async () => { + const { table, uri } = await openExternalBlobTable(); + + await expect(table.add([{ id: 1n, payload: uri }])).rejects.toThrow( + "allow_external_blob_outside_bases", + ); + await expect(table.countRows()).resolves.toBe(0); + + await expect( + table.add([{ id: 2n, payload: uri }], { + allowExternalBlobOutsideBases: false, + }), + ).rejects.toThrow("allow_external_blob_outside_bases"); + await expect(table.countRows()).resolves.toBe(0); + }); + + it.each([ + ["URI string", (uri: string) => uri], + ["URI struct", (uri: string) => ({ uri })], + ])( + "round-trips an external blob from a %s after reopening", + async (_label, blobValue) => { + const { db, table, uri, payload } = await openExternalBlobTable(); + + const result = await table.add([{ id: 1n, payload: blobValue(uri) }], { + allowExternalBlobOutsideBases: true, + }); + expect(result.version).toBeGreaterThan(0); + await expect(table.countRows()).resolves.toBe(1); + + table.close(); + const reopened = await db.openTable("external_blobs"); + const rows = await reopened.query().withRowId().toArray(); + expect(rows).toHaveLength(1); + expect(rows[0].payload.blob_uri).toBe(uri); + const bytes = await reopened.fetchBlobs("payload", [ + rows[0]._rowid as bigint, + ]); + expect(bytes[0]).toEqual(payload); + }, + ); + + it("applies the external blob opt-in to Arrow table input", async () => { + const { table, schema, uri, payload } = await openExternalBlobTable(); + const input = makeArrowTable([{ id: 1n, payload: uri }], { schema }); + + const result = await table.add(input, { + allowExternalBlobOutsideBases: true, + }); + + expect(result.version).toBeGreaterThan(0); + const rows = await table.query().withRowId().toArray(); + expect(rows[0].payload.blob_uri).toBe(uri); + const bytes = await table.fetchBlobs("payload", [rows[0]._rowid as bigint]); + expect(bytes[0]).toEqual(payload); + }); + + it("applies the external blob opt-in only to the current add", async () => { + const { table, uri } = await openExternalBlobTable(); + + await table.add([{ id: 1n, payload: uri }], { + allowExternalBlobOutsideBases: true, + }); + await expect(table.countRows()).resolves.toBe(1); + + await expect(table.add([{ id: 2n, payload: uri }])).rejects.toThrow( + "allow_external_blob_outside_bases", + ); + await expect(table.countRows()).resolves.toBe(1); + }); + + it("supports progress and overwrite with the external blob opt-in", async () => { + const { table, uri, payload } = await openExternalBlobTable(); + await table.add([{ id: 1n, payload: Buffer.from("old") }]); + + let resolveFinal!: (progress: import("../lancedb").WriteProgress) => void; + let rejectFinal!: (error: Error) => void; + const finalProgress = new Promise( + (resolve, reject) => { + resolveFinal = resolve; + rejectFinal = reject; + }, + ); + const timeout = setTimeout( + () => rejectFinal(new Error("timed out waiting for final progress")), + 5_000, + ); + try { + const result = await table.add([{ id: 2n, payload: uri }], { + mode: "overwrite", + allowExternalBlobOutsideBases: true, + progress: (progress) => { + if (progress.done) { + resolveFinal(progress); + } + }, + }); + expect(result.version).toBeGreaterThan(0); + const progress = await finalProgress; + expect(progress.outputRows).toBe(1); + expect(progress.totalRows).toBe(1); + } finally { + clearTimeout(timeout); + } + + const rows = await table.query().withRowId().toArray(); + expect(rows).toHaveLength(1); + expect(rows[0].id).toBe(2n); + const bytes = await table.fetchBlobs("payload", [rows[0]._rowid as bigint]); + expect(bytes[0]).toEqual(payload); + }); + it("discovers blob columns", async () => { const { table } = await openBlobTable(); expect(await table.blobColumns()).toEqual(["image"]); @@ -2468,6 +2745,27 @@ describe("when dealing with blob columns", () => { expect(Buffer.from(await files[1]!.read()).toString()).toBe("beta"); }); + it("adds a LargeBinary Arrow column to a blob table", async () => { + const db = await connect(tmpDir.name); + const schema = new Schema([new Field("id", new Int64()), blob("image")]); + const table = await db.createTable( + "blobs", + [{ id: 1n, image: Buffer.from("alpha") }], + { schema }, + ); + const payload = Buffer.from("beta"); + const data = new arrow18.Table({ + id: arrow18.vectorFromArray([2n], new arrow18.Int64()), + image: arrow18.vectorFromArray([payload], new arrow18.LargeBinary()), + }); + await table.add(data); + + const rows = await table.query().withRowId().toArray(); + const rowId = rows.find((row) => row.id === 2n)!._rowid as bigint; + const [actual] = await table.fetchBlobs("image", [rowId]); + expect(actual).toEqual(payload); + }); + it("reads a half-open range", async () => { const { table, rowIds } = await openBlobTable(); const files = await table.fetchBlobFiles("image", rowIds); @@ -2492,6 +2790,148 @@ describe("when dealing with blob columns", () => { ).rejects.toThrow(/exceeds blob size/); }); + it("reads at most maxBytes and advances the cursor", async () => { + const { table, rowIds } = await openBlobTable(); + const [handle] = await table.fetchBlobFiles("image", rowIds); + expect((await handle!.read(2n)).toString()).toBe("al"); + expect(await handle!.tell()).toBe(2n); + expect((await handle!.read(2n)).toString()).toBe("ph"); + expect((await handle!.read(10n)).toString()).toBe("a"); + expect(await handle!.tell()).toBe(5n); + expect(await handle!.read(1n)).toEqual(Buffer.alloc(0)); + }); + + it("seeks the cursor", async () => { + const { table, rowIds, alpha } = await openBlobTable(); + const [handle] = await table.fetchBlobFiles("image", rowIds); + await handle!.seek(3n); + expect(await handle!.tell()).toBe(3n); + expect((await handle!.read()).toString()).toBe("ha"); + await handle!.seek(0n); + expect(await handle!.read()).toEqual(alpha); + }); + + it("rejects a negative seek position", async () => { + const { table, rowIds } = await openBlobTable(); + const [handle] = await table.fetchBlobFiles("image", rowIds); + await expect(handle!.seek(-1n)).rejects.toThrow(/cannot be negative/); + }); + + it("readRanges returns one buffer per range in order", async () => { + const { table, rowIds } = await openBlobTable(); + const [handle] = await table.fetchBlobFiles("image", rowIds); + const buffers = await handle!.readRanges([ + { start: 3n, end: 5n }, + { start: 0n, end: 2n }, + { start: 3n, end: 5n }, + ]); + expect(buffers.map((b) => b.toString())).toEqual(["ha", "al", "ha"]); + expect(await handle!.tell()).toBe(0n); + }); + + it("fails when a readRanges end is past the blob size", async () => { + const { table, rowIds, alpha } = await openBlobTable(); + const [handle] = await table.fetchBlobFiles("image", rowIds); + await expect( + handle!.readRanges([ + { start: 0n, end: 1n }, + { start: 0n, end: BigInt(alpha.length + 1) }, + ]), + ).rejects.toThrow(/exceeds blob size/); + }); + + it("fails reads after close and closes once", async () => { + const { table, rowIds } = await openBlobTable(); + const [handle] = await table.fetchBlobFiles("image", rowIds); + expect(handle!.isClosed()).toBe(false); + await handle!.close(); + expect(handle!.isClosed()).toBe(true); + await expect(handle!.close()).resolves.toBeUndefined(); + await expect(handle!.read()).rejects.toThrow(/already closed/); + await expect(handle!.read(1n)).rejects.toThrow(/already closed/); + await expect(handle!.readRange(0n, 1n)).rejects.toThrow(/already closed/); + await expect(handle!.readRanges([{ start: 0n, end: 1n }])).rejects.toThrow( + /already closed/, + ); + await expect(handle!.seek(0n)).rejects.toThrow(/already closed/); + await expect(handle!.tell()).rejects.toThrow(/already closed/); + expect(handle!.size()).toBe(5n); + }); + + it("rejects reads with an already aborted signal", async () => { + const { table, rowIds, alpha } = await openBlobTable(); + const [handle] = await table.fetchBlobFiles("image", rowIds); + const signal = AbortSignal.abort(); + const abortError = { name: "AbortError" }; + await expect(handle!.read(undefined, { signal })).rejects.toMatchObject( + abortError, + ); + await expect(handle!.read(2n, { signal })).rejects.toMatchObject( + abortError, + ); + await expect(handle!.readRange(0n, 1n, { signal })).rejects.toMatchObject( + abortError, + ); + await expect( + handle!.readRanges([{ start: 0n, end: 1n }], { signal }), + ).rejects.toMatchObject(abortError); + await expect( + table.fetchBlobs("image", rowIds, { signal }), + ).rejects.toMatchObject(abortError); + await expect( + table.fetchBlobFiles("image", rowIds, { signal }), + ).rejects.toMatchObject(abortError); + expect(await handle!.read()).toEqual(alpha); + }); + + it("rejects with the abort reason", async () => { + const { table, rowIds } = await openBlobTable(); + const [handle] = await table.fetchBlobFiles("image", rowIds); + const reason = new Error("caller gave up"); + await expect( + handle!.readRange(0n, 1n, { signal: AbortSignal.abort(reason) }), + ).rejects.toBe(reason); + }); + + it("reads with a signal and removes its abort listener", async () => { + const { table, rowIds, alpha, beta } = await openBlobTable(); + const controller = new AbortController(); + const { signal } = controller; + const [handle] = await table.fetchBlobFiles("image", rowIds, { signal }); + expect((await handle!.readRange(0n, 2n, { signal })).toString()).toBe("al"); + expect(await table.fetchBlobs("image", rowIds, { signal })).toEqual([ + alpha, + beta, + null, + ]); + expect(getEventListeners(signal, "abort")).toHaveLength(0); + controller.abort(); + expect(await handle!.read()).toEqual(alpha); + }); + + it("cancels an in-flight read", async () => { + const db = await connect(tmpDir.name); + const schema = new Schema([ + new Field("id", new Int64(), true), + blob("payload", { inlineSizeThreshold: 0, dedicatedSizeThreshold: 1 }), + ]); + const payload = Buffer.alloc(32 * 1024 * 1024, 7); + const table = await db.createTable("large_blobs", [{ id: 1n, payload }], { + schema, + }); + const [row] = await table.query().withRowId().toArray(); + const [handle] = await table.fetchBlobFiles("payload", [ + row._rowid as bigint, + ]); + const controller = new AbortController(); + const pending = handle!.readRange(0n, handle!.size(), { + signal: controller.signal, + }); + controller.abort(); + await expect(pending).rejects.toMatchObject({ name: "AbortError" }); + expect(await handle!.readRange(0n, 4n)).toEqual(payload.subarray(0, 4)); + }); + it("rejects fetchBlobs on a non-blob column", async () => { const { table, rowIds } = await openBlobTable(); await expect(table.fetchBlobs("id", rowIds)).rejects.toThrow(/blob/i); @@ -2688,6 +3128,20 @@ describe("when dealing with blob columns", () => { const rowIds = [1, 2, 3].map((id) => rowIdById.get(id)!); return { table, rowIds, alpha, beta }; } + + async function openExternalBlobTable() { + const db = await connect(path.join(tmpDir.name, "db")); + const payload = Buffer.from("external-payload"); + const payloadPath = path.join(tmpDir.name, "payload.bin"); + fs.writeFileSync(payloadPath, payload); + const uri = pathToFileURL(payloadPath).href; + const schema = new Schema([ + new Field("id", new Int64(), true), + blob("payload"), + ]); + const table = await db.createEmptyTable("external_blobs", schema); + return { db, table, schema, uri, payload }; + } }); describe("when dealing with tags", () => { @@ -2860,7 +3314,7 @@ it("passes cleanupOlderThan to the native binding as an absolute timestamp", asy expect(optimize).toHaveBeenCalledWith(cutoff.getTime(), true); }); -describe.each([arrow15, arrow16, arrow17, arrow18])( +describe.each(arrowVersions)( "when optimizing a dataset", // biome-ignore lint/suspicious/noExplicitAny: (arrow: any) => { diff --git a/nodejs/__test__/util.test.ts b/nodejs/__test__/util.test.ts index 802f8ed4d..d7e489f73 100644 --- a/nodejs/__test__/util.test.ts +++ b/nodejs/__test__/util.test.ts @@ -14,6 +14,9 @@ test.each([ [new ArrayBuffer(8), "X'0000000000000000'"], [Buffer.from("hello"), "X'68656c6c6f'"], ["Hello 'world'", "'Hello ''world'''"], + [NaN, "CAST('NaN' AS DOUBLE)"], + [Infinity, "CAST('Infinity' AS DOUBLE)"], + [-Infinity, "CAST('-Infinity' AS DOUBLE)"], ])("toSQL(%p) === %p", (value, expected) => { expect(toSQL(value)).toBe(expected); }); diff --git a/nodejs/lancedb/arrow.ts b/nodejs/lancedb/arrow.ts index df875a808..3daee7d65 100644 --- a/nodejs/lancedb/arrow.ts +++ b/nodejs/lancedb/arrow.ts @@ -66,11 +66,20 @@ export type SchemaLike = export type FieldLike = | Field | { - type: string; + type: string | DataTypeLike; name: string; nullable: boolean; metadata?: Map; }; +/** + * A `DataType` from any copy or version of apache-arrow. + * + * Arrow 21 brands its classes with `unique symbol` properties, so a type + * object from a second copy of the library no longer satisfies the `DataType` + * type of this one even though it is structurally identical. Inputs that only + * need to be sanitized accept this looser shape instead. + */ +export type DataTypeLike = DataType | { readonly typeId: number }; /** * Create an Arrow field backed by LanceDB's JSON extension type. @@ -1059,6 +1068,71 @@ export async function fromRecordsToStreamBuffer( return Buffer.from(await writer.toUint8Array()); } +// `Type.Utf8View` / `Type.BinaryView` as numbers: the enum members only exist +// in Arrow 21+, and this module compiles against every supported release. +const UTF8_VIEW_TYPE_ID = 24; +const BINARY_VIEW_TYPE_ID = 23; + +/** + * Copy a Utf8View / BinaryView `Data` into a single `Data` of `type`. + */ +function materializeViewData(child: ArrowData, type: DataType): ArrowData { + const builder = makeBuilder({ type, nullValues: [null] }); + for (const value of new Vector([child])) { + builder.append(value); + } + return builder.finish().flush(); +} + +/** + * Rebuild any top-level Utf8View / BinaryView column as Utf8 / Binary. + * + * Lance stores the view types as their offset-based equivalents anyway, so + * nothing is lost. Doing it here also sidesteps an Arrow JS 21 bug: its IPC + * writer emits a truncated views buffer for a *sliced* view array, which the + * Rust reader rejects with "Need at least N bytes in buffers[0]". + * + * The record batches are rebuilt positionally rather than through a + * `Record`: JavaScript enumerates integer-like keys first, so + * a field named e.g. `"1"` would otherwise be paired with the wrong column. + * + * Tables without view columns are returned as-is. + */ +function materializeViewColumns(table: ArrowTable): ArrowTable { + const replacements = new Map(); + table.schema.fields.forEach((field, i) => { + if (field.type.typeId === UTF8_VIEW_TYPE_ID) { + replacements.set(i, new Utf8()); + } else if (field.type.typeId === BINARY_VIEW_TYPE_ID) { + replacements.set(i, new Binary()); + } + }); + if (replacements.size === 0) { + return table; + } + const fields = table.schema.fields.map((field, i) => { + const type = replacements.get(i); + return type === undefined + ? field + : new Field(field.name, type, field.nullable, field.metadata); + }); + const schema = new Schema(fields, table.schema.metadata); + const batches = table.batches.map((batch) => { + const children = batch.data.children.map((child, i) => { + const type = replacements.get(i); + return type === undefined ? child : materializeViewData(child, type); + }); + const data = makeData({ + type: new Struct(fields), + length: batch.numRows, + nullCount: 0, + children, + }); + return new RecordBatch(schema, data); + }); + return new ArrowTable(schema, batches); +} + /** * Serialize an Arrow Table into a buffer using the Arrow IPC File serialization * @@ -1075,7 +1149,9 @@ export async function fromTableToBuffer( if (schema !== undefined && schema !== null) { schema = sanitizeSchema(schema); } - const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema); + const tableWithEmbeddings = materializeViewColumns( + await applyEmbeddings(table, embeddings, schema), + ); validateBlobSchema(tableWithEmbeddings.schema); const writer = RecordBatchFileWriter.writeAll(tableWithEmbeddings); return Buffer.from(await writer.toUint8Array()); @@ -1146,7 +1222,8 @@ export async function fromRecordBatchToBuffer( export async function fromRecordBatchToStreamBuffer( batch: RecordBatch, ): Promise { - const writer = RecordBatchStreamWriter.writeAll([batch]); + const table = materializeViewColumns(new ArrowTable([batch])); + const writer = RecordBatchStreamWriter.writeAll(table); return Buffer.from(await writer.toUint8Array()); } @@ -1163,7 +1240,9 @@ export async function fromTableToStreamBuffer( embeddings?: EmbeddingFunctionConfig, schema?: SchemaLike, ): Promise { - const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema); + const tableWithEmbeddings = materializeViewColumns( + await applyEmbeddings(table, embeddings, schema), + ); const writer = RecordBatchStreamWriter.writeAll(tableWithEmbeddings); return Buffer.from(await writer.toUint8Array()); } diff --git a/nodejs/lancedb/blob.ts b/nodejs/lancedb/blob.ts index 244faac09..e6c780ca9 100644 --- a/nodejs/lancedb/blob.ts +++ b/nodejs/lancedb/blob.ts @@ -141,13 +141,16 @@ export class BlobFile { } /** - * Reads from the cursor to the end and advances the cursor. + * Reads from the cursor and advances the cursor. * - * A second call returns an empty buffer. {@link BlobFile.readRange} does - * not move the cursor. + * Reads to the end when `maxBytes` is omitted, or at most `maxBytes` bytes + * otherwise. Returns an empty buffer at the end of the blob. + * {@link BlobFile.readRange} does not move the cursor. */ - read(): Promise { - return this.inner.read(); + read(maxBytes?: bigint, options?: BlobReadOptions): Promise { + return runWithSignal(options?.signal, (signal) => + this.inner.read(maxBytes, signal), + ); } /** @@ -155,11 +158,96 @@ export class BlobFile { * * Fails when `end` is past the blob size. Does not move the cursor. */ - readRange(start: bigint, end: bigint): Promise { - return this.inner.readRange(start, end); + readRange( + start: bigint, + end: bigint, + options?: BlobReadOptions, + ): Promise { + return runWithSignal(options?.signal, (signal) => + this.inner.readRange(start, end, signal), + ); + } + + /** + * Reads several half-open byte ranges. Returns one buffer per range, in + * the order given. + * + * Fails when any `end` is past the blob size. Does not move the cursor. + */ + readRanges( + ranges: BlobRange[], + options?: BlobReadOptions, + ): Promise { + return runWithSignal(options?.signal, (signal) => + this.inner.readRanges(ranges, signal), + ); + } + + /** Moves the cursor to `position`, in bytes from the start of the blob. */ + seek(position: bigint): Promise { + return this.inner.seek(position); + } + + /** Returns the cursor position in bytes. */ + tell(): Promise { + return this.inner.tell(); + } + + /** + * Releases the handle. Reads after `close()` fail. Calling it again does + * nothing. + */ + close(): Promise { + return this.inner.close(); + } + + /** Returns true after {@link BlobFile.close}. */ + isClosed(): boolean { + return this.inner.isClosed(); } } +/** Options for blob reads. */ +export type BlobReadOptions = { + /** + * Cancels the read. The call rejects with `signal.reason`, and the native + * read stops, including in-flight requests to a remote table. + */ + signal?: AbortSignal; +}; + +/** @ignore */ +export async function runWithSignal( + signal: AbortSignal | undefined, + run: (signal: AbortSignal | undefined) => Promise, +): Promise { + if (signal === undefined) { + return run(undefined); + } + signal.throwIfAborted(); + // napi-rs replaces `onabort` on the signal it receives, so pass a private + // signal and forward the caller's abort to it. + const controller = new AbortController(); + const onAbort = () => controller.abort(); + signal.addEventListener("abort", onAbort, { once: true }); + try { + return await run(controller.signal); + } catch (err) { + if (signal.aborted) { + throw signal.reason; + } + throw err; + } finally { + signal.removeEventListener("abort", onAbort); + } +} + +/** A half-open byte range `[start, end)` for {@link BlobFile.readRanges}. */ +export type BlobRange = { + start: bigint; + end: bigint; +}; + export function coerceBlobValue(value: unknown): BlobInput | null { if (value == null) { return null; diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index ed6889232..9d4e5608b 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -84,7 +84,7 @@ export { } from "./arrow"; export { blob, isBlobField, BlobFile } from "./blob"; -export type { BlobOptions } from "./blob"; +export type { BlobOptions, BlobRange, BlobReadOptions } from "./blob"; export { Connection, @@ -195,6 +195,7 @@ export * as rerankers from "./rerankers"; export { SchemaLike, TableLike, + DataTypeLike, FieldLike, RecordBatchLike, DataLike, diff --git a/nodejs/lancedb/materialized_view.ts b/nodejs/lancedb/materialized_view.ts index 51f8cfead..a59a3a5f6 100644 --- a/nodejs/lancedb/materialized_view.ts +++ b/nodejs/lancedb/materialized_view.ts @@ -9,9 +9,10 @@ export const DEFINITION_META_KEY = "mv.definition"; /** * The newest stored layout this version reads: `{"format": N, "query": ""}`, - * format 2 being a query with `GROUP BY`. + * format 2 being a query with `GROUP BY`, format 3 native pairs and format 4 + * native dedup results. */ -export const DEFINITION_FORMAT = 2; +export const DEFINITION_FORMAT = 4; /** * The query that defines a materialized view, as stored: diff --git a/nodejs/lancedb/query.ts b/nodejs/lancedb/query.ts index 75a787fcd..f370e1b31 100644 --- a/nodejs/lancedb/query.ts +++ b/nodejs/lancedb/query.ts @@ -41,26 +41,6 @@ export async function* RecordBatchIterator( } } -class RecordBatchIterable< - NativeQueryType extends NativeQuery | NativeVectorQuery | NativeTakeQuery, -> implements AsyncIterable -{ - private inner: NativeQueryType; - private options?: QueryExecutionOptions; - - constructor(inner: NativeQueryType, options?: QueryExecutionOptions) { - this.inner = inner; - this.options = options; - } - - // biome-ignore lint/suspicious/noExplicitAny: skip - [Symbol.asyncIterator](): AsyncIterator, any, undefined> { - return RecordBatchIterator( - this.inner.execute(this.options?.maxBatchLength, this.options?.timeoutMs), - ); - } -} - /** * Options that control the behavior of a particular query execution */ @@ -271,13 +251,20 @@ export class QueryBase< return RecordBatchIterator(this.nativeExecute()); } - /** Collect the results as an Arrow @see {@link ArrowTable}. */ + /** + * Collect the results as an Arrow @see {@link ArrowTable}. + * + * Empty results retain the query's output schema. + */ async toArrow(options?: Partial): Promise { const batches = []; - const inner = await this.getInner(); - for await (const batch of new RecordBatchIterable(inner, options)) { + const iterator = this.nativeExecute(options); + for await (const batch of RecordBatchIterator(iterator)) { batches.push(batch); } + if (batches.length === 0) { + return tableFromIPC((await iterator).schema()); + } return new ArrowTable(batches); } @@ -532,6 +519,8 @@ export class VectorQuery extends StandardQueryBase { /** * Set the number of partitions to search (probe) * + * The number of probes must be greater than 0. + * * This argument is only used when the vector column has an IVF PQ index. * If there is no index then this value is ignored. * diff --git a/nodejs/lancedb/sanitize.ts b/nodejs/lancedb/sanitize.ts index 3e5d583c4..6ba4cc881 100644 --- a/nodejs/lancedb/sanitize.ts +++ b/nodejs/lancedb/sanitize.ts @@ -9,6 +9,7 @@ // comes from the exact same library instance. This is not always the case // and so we must sanitize the input to ensure that it is compatible. +import * as arrowModule from "apache-arrow"; import { BufferType, Data, Vector } from "apache-arrow"; import type { IntBitWidth, TKeys, TimeBitWidth } from "apache-arrow/type"; import { @@ -43,6 +44,8 @@ import { Interval, IntervalDayTime, IntervalYearMonth, + LargeBinary, + LargeUtf8, List, Map_, Null, @@ -74,6 +77,66 @@ import { Utf8, } from "./arrow"; +/** + * Whether `value` was created by *this* copy of apache-arrow's `ctor`. + * + * Arrow 21 overrides `Symbol.hasInstance` on `Schema`, `Field`, `Table`, + * `RecordBatch`, `Vector` and `Data` so that `instanceof` also matches objects + * built by a different copy of the library. Those are exactly the objects + * this module has to rebuild: the `DataType` instances nested inside them are + * deliberately *not* given that treatment upstream, so Arrow's own builders and + * writers reject them. Walk the real prototype chain instead of trusting + * `instanceof`. + */ +function isOwnInstance( + value: unknown, + // biome-ignore lint/suspicious/noExplicitAny: matches any class constructor + ctor: abstract new (...args: any[]) => T, +): value is T { + return ( + typeof value === "object" && + value !== null && + Object.prototype.isPrototypeOf.call(ctor.prototype, value) + ); +} + +/** + * Types added to Arrow JS after release 18, keyed by the `Type` enum value + * they use. The library compiles and runs against every Arrow release in the + * peer range, so these are looked up by name at runtime instead of imported: + * an older Arrow simply does not have the class. + */ +type NewerTypeName = + | "IntervalMonthDayNano" + | "LargeList" + | "BinaryView" + | "Utf8View"; +const NEWER_TYPES: ReadonlyArray<{ + name: NewerTypeName; + typeId: number; + since: number; +}> = [ + { name: "IntervalMonthDayNano", typeId: -31, since: 20 }, + { name: "LargeList", typeId: 21, since: 21 }, + { name: "BinaryView", typeId: 23, since: 21 }, + { name: "Utf8View", typeId: 24, since: 21 }, +]; + +// biome-ignore lint/suspicious/noExplicitAny: constructor arity varies by type +type DataTypeConstructor = new (...args: any[]) => DataType; + +function newerTypeConstructor(name: NewerTypeName): DataTypeConstructor { + const ctor = (arrowModule as unknown as Record)[name]; + if (typeof ctor !== "function") { + const since = NEWER_TYPES.find((t) => t.name === name)?.since; + throw new Error( + `The Arrow type ${name} requires apache-arrow ${since} or newer, ` + + "but the installed apache-arrow does not provide it", + ); + } + return ctor as DataTypeConstructor; +} + type SanitizationContext = { types: WeakMap; vectors: WeakMap; @@ -210,19 +273,37 @@ export function sanitizeList(typeLike: object) { return sanitizeListWithContext(typeLike, createSanitizationContext()); } +function sanitizeListChildWithContext( + typeLike: object, + context: SanitizationContext, + typeName: string, +): Field { + if (!("children" in typeLike) || !Array.isArray(typeLike.children)) { + throw Error( + `Expected a ${typeName} type to have an array-like \`children\` property`, + ); + } + if (typeLike.children.length !== 1) { + throw Error(`Expected a ${typeName} type to have exactly one child`); + } + return sanitizeFieldWithContext(typeLike.children[0], context); +} + function sanitizeListWithContext( typeLike: object, context: SanitizationContext, ) { - if (!("children" in typeLike) || !Array.isArray(typeLike.children)) { - throw Error( - "Expected a List type to have an array-like `children` property", - ); - } - if (typeLike.children.length !== 1) { - throw Error("Expected a List type to have exactly one child"); - } - return new List(sanitizeFieldWithContext(typeLike.children[0], context)); + return new List(sanitizeListChildWithContext(typeLike, context, "List")); +} + +function sanitizeLargeListWithContext( + typeLike: object, + context: SanitizationContext, +) { + const LargeList = newerTypeConstructor("LargeList"); + return new LargeList( + sanitizeListChildWithContext(typeLike, context, "LargeList"), + ); } export function sanitizeStruct(typeLike: object) { @@ -449,11 +530,30 @@ function sanitizeTypeWithContext( return type; } +function sanitizeNewerTypeById( + typeLike: object, + typeId: number, + context: SanitizationContext, +): DataType | undefined { + const entry = NEWER_TYPES.find((t) => t.typeId === typeId); + if (entry === undefined) { + return undefined; + } + if (entry.name === "LargeList") { + return sanitizeLargeListWithContext(typeLike, context); + } + return new (newerTypeConstructor(entry.name))(); +} + function sanitizeTypeById( typeLike: object, typeId: Type, context: SanitizationContext, ): DataType { + const newer = sanitizeNewerTypeById(typeLike, typeId, context); + if (newer !== undefined) { + return newer; + } switch (typeId) { case Type.NONE: throw Error("Received a Type with a typeId of NONE"); @@ -465,8 +565,12 @@ function sanitizeTypeById( return sanitizeFloat(typeLike); case Type.Binary: return new Binary(); + case Type.LargeBinary: + return new LargeBinary(); case Type.Utf8: return new Utf8(); + case Type.LargeUtf8: + return new LargeUtf8(); case Type.Bool: return new Bool(); case Type.Decimal: @@ -566,7 +670,7 @@ function sanitizeFieldWithContext( fieldLike: unknown, context: SanitizationContext, ): Field { - if (fieldLike instanceof Field) { + if (isOwnInstance(fieldLike, Field)) { return fieldLike; } if (typeof fieldLike !== "object" || fieldLike === null) { @@ -620,7 +724,7 @@ function sanitizeSchemaWithContext( schemaLike: SchemaLike, context: SanitizationContext, ): Schema { - if (schemaLike instanceof Schema) { + if (isOwnInstance(schemaLike, Schema)) { return schemaLike; } if (typeof schemaLike !== "object" || schemaLike === null) { @@ -647,7 +751,7 @@ function sanitizeSchemaWithContext( } export function sanitizeTable(tableLike: TableLike): Table { - if (tableLike instanceof Table) { + if (isOwnInstance(tableLike, Table)) { return tableLike; } if (typeof tableLike !== "object" || tableLike === null) { @@ -675,7 +779,7 @@ function sanitizeRecordBatch( batchLike: RecordBatchLike, context: SanitizationContext, ): RecordBatch { - if (batchLike instanceof RecordBatch) { + if (isOwnInstance(batchLike, RecordBatch)) { return batchLike; } if (typeof batchLike !== "object" || batchLike === null) { @@ -704,11 +808,17 @@ type DictionaryDataLike = DataLike & { dictionary?: DictionaryVectorLike; }; +// Arrow 21 added `variadicBuffers` to `Data` for the Utf8View / BinaryView +// types. Older Arrow versions do not have the property, so it is optional. +type VariadicDataLike = DataLike & { + variadicBuffers?: ReadonlyArray; +}; + function sanitizeData( dataLike: DataLike, context: SanitizationContext, ): Data { - if (dataLike instanceof Data) { + if (isOwnInstance(dataLike, Data)) { return dataLike; } const cachedData = context.data.get(dataLike); @@ -726,7 +836,12 @@ function sanitizeData( context.vectors.set(dictionaryLike, dictionary); } } - const data = new Data( + // Arrow 21 added an eighth constructor parameter (`variadicBuffers`); older + // releases ignore the extra argument but their typings do not declare it. + const DataCtor = Data as unknown as new ( + ...args: unknown[] + ) => Data; + const data = new DataCtor( sanitizeTypeWithContext(dataLike.type, context), dataLike.offset, dataLike.length, @@ -739,6 +854,7 @@ function sanitizeData( }, dataLike.children.map((child) => sanitizeData(child, context)), dictionary, + (dataLike as VariadicDataLike).variadicBuffers, ); context.data.set(dataLike, data); return data; @@ -747,7 +863,11 @@ function sanitizeData( const constructorsByTypeName = { null: () => new Null(), binary: () => new Binary(), + largebinary: () => new LargeBinary(), + binaryview: () => new (newerTypeConstructor("BinaryView"))(), utf8: () => new Utf8(), + largeutf8: () => new LargeUtf8(), + utf8view: () => new (newerTypeConstructor("Utf8View"))(), bool: () => new Bool(), int8: () => new Int8(), int16: () => new Int16(), @@ -768,6 +888,8 @@ const constructorsByTypeName = { timesecond: () => new TimeSecond(), intervaldaytime: () => new IntervalDayTime(), intervalyearmonth: () => new IntervalYearMonth(), + intervalmonthdaynano: () => + new (newerTypeConstructor("IntervalMonthDayNano"))(), durationnanosecond: () => new DurationNanosecond(), durationmicrosecond: () => new DurationMicrosecond(), durationmillisecond: () => new DurationMillisecond(), diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index a098322a9..d4c5db509 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -17,7 +17,7 @@ import { tableFromIPC, } from "./arrow"; -import { BlobFile } from "./blob"; +import { BlobFile, BlobReadOptions, runWithSignal } from "./blob"; import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry"; import { IndexOptions } from "./indices"; import { Job } from "./job"; @@ -124,6 +124,17 @@ export interface AddDataOptions { * ``` */ progress: (progress: WriteProgress) => void; + + /** + * Whether blob URIs outside registered external bases may be written. + * + * Defaults to `false`. This option is supported only for local/native + * tables; remote tables return an error when it is enabled. An enabled write + * stores the absolute URI reference without registering a base or copying + * the external object into the database. The object must remain accessible + * when the blob is read later. + */ + allowExternalBlobOutsideBases?: boolean; } export interface UpdateOptions { @@ -526,10 +537,15 @@ export abstract class Table { * Reads the table's current checkout. IDs from another version can fail after * compaction unless stable row ids are enabled. Results keep input order and * duplicates. Null blobs are `null`. Empty blobs are empty buffers. + * Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes. + * The client splits requests automatically and reads an individual larger + * blob through the Range route. This method still materializes all bytes in + * memory; use {@link Table.fetchBlobFiles} for large values. */ abstract fetchBlobs( column: string, rowIds: readonly (bigint | number)[], + options?: BlobReadOptions, ): Promise<(Buffer | null)[]>; /** @@ -542,6 +558,7 @@ export abstract class Table { abstract fetchBlobFiles( column: string, rowIds: readonly (bigint | number)[], + options?: BlobReadOptions, ): Promise<(BlobFile | null)[]>; /** @@ -1091,7 +1108,12 @@ export class LocalTable extends Table { } } : undefined; - return await this.inner.add(buffer, mode, progress); + return await this.inner.add( + buffer, + mode, + progress, + options?.allowExternalBlobOutsideBases, + ); } async update( @@ -1228,8 +1250,11 @@ export class LocalTable extends Table { async fetchBlobs( column: string, rowIds: readonly (bigint | number)[], + options?: BlobReadOptions, ): Promise<(Buffer | null)[]> { - const values = await this.inner.fetchBlobs(column, rowIdsToBigInts(rowIds)); + const values = await runWithSignal(options?.signal, (signal) => + this.inner.fetchBlobs(column, rowIdsToBigInts(rowIds), signal), + ); // N-API Option maps missing values to undefined. Collapse those to null. return values.map((value) => value ?? null); } @@ -1237,10 +1262,10 @@ export class LocalTable extends Table { async fetchBlobFiles( column: string, rowIds: readonly (bigint | number)[], + options?: BlobReadOptions, ): Promise<(BlobFile | null)[]> { - const files = await this.inner.fetchBlobFiles( - column, - rowIdsToBigInts(rowIds), + const files = await runWithSignal(options?.signal, (signal) => + this.inner.fetchBlobFiles(column, rowIdsToBigInts(rowIds), signal), ); // N-API Option maps missing values to undefined. Collapse those to null. return files.map((file) => diff --git a/nodejs/lancedb/util.ts b/nodejs/lancedb/util.ts index 9219f3d1d..5127a1c7f 100644 --- a/nodejs/lancedb/util.ts +++ b/nodejs/lancedb/util.ts @@ -15,6 +15,14 @@ export function toSQL(value: IntoSql): string { if (typeof value === "string") { return `'${value.replace(/'/g, "''")}'`; } else if (typeof value === "number") { + // toString() gives "NaN" / "Infinity", which SQL reads as column names. + if (Number.isNaN(value)) { + return "CAST('NaN' AS DOUBLE)"; + } else if (!Number.isFinite(value)) { + return value > 0 + ? "CAST('Infinity' AS DOUBLE)" + : "CAST('-Infinity' AS DOUBLE)"; + } return value.toString(); } else if (typeof value === "boolean") { return value ? "TRUE" : "FALSE"; diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 0e9d131b0..051bdb98f 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.40.0-beta.11", + "version": "0.40.0-beta.12", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 39002b700..755035ee2 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.40.0-beta.11", + "version": "0.40.0-beta.12", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 232796091..d31737229 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.40.0-beta.11", + "version": "0.40.0-beta.12", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index cc9c003e0..618df080d 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.40.0-beta.11", + "version": "0.40.0-beta.12", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 2f0356da2..87628c0c3 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.40.0-beta.11", + "version": "0.40.0-beta.12", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 2f96d1019..65c7b500c 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.40.0-beta.11", + "version": "0.40.0-beta.12", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index c941d597f..ad6a81acc 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.40.0-beta.11", + "version": "0.40.0-beta.12", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index bd37d97e2..da5dc77b2 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.40.0-beta.11", + "version": "0.40.0-beta.12", "main": "dist/index.js", "exports": { ".": "./dist/index.js", @@ -49,10 +49,14 @@ "@types/jest": "^29.1.2", "@types/node": "22.7.4", "@types/tmp": "^0.2.6", + "apache-arrow": "21.2.0", "apache-arrow-15": "npm:apache-arrow@15.0.0", "apache-arrow-16": "npm:apache-arrow@16.0.0", "apache-arrow-17": "npm:apache-arrow@17.0.0", "apache-arrow-18": "npm:apache-arrow@18.0.0", + "apache-arrow-19": "npm:apache-arrow@19.0.1", + "apache-arrow-20": "npm:apache-arrow@20.0.0", + "apache-arrow-21": "npm:apache-arrow@21.2.0", "eslint": "^8.57.0", "jest": "^29.7.0", "shx": "^0.3.4", @@ -102,7 +106,7 @@ }, "peerDependencies": { "@types/node": ">=22", - "apache-arrow": ">=15.0.0 <=18.1.0" + "apache-arrow": ">=15.0.0 <22.0.0" }, "peerDependenciesMeta": { "@types/node": { diff --git a/nodejs/pnpm-lock.yaml b/nodejs/pnpm-lock.yaml index a03838999..4c19305c8 100644 --- a/nodejs/pnpm-lock.yaml +++ b/nodejs/pnpm-lock.yaml @@ -14,9 +14,6 @@ importers: '@opentelemetry/api': specifier: ^1.9.0 version: 1.9.1 - apache-arrow: - specifier: '>=15.0.0 <=18.1.0' - version: 18.1.0 reflect-metadata: specifier: ^0.2.2 version: 0.2.2 @@ -54,6 +51,9 @@ importers: '@types/tmp': specifier: ^0.2.6 version: 0.2.6 + apache-arrow: + specifier: 21.2.0 + version: 21.2.0 apache-arrow-15: specifier: npm:apache-arrow@15.0.0 version: apache-arrow@15.0.0 @@ -66,6 +66,15 @@ importers: apache-arrow-18: specifier: npm:apache-arrow@18.0.0 version: apache-arrow@18.0.0 + apache-arrow-19: + specifier: npm:apache-arrow@19.0.1 + version: apache-arrow@19.0.1 + apache-arrow-20: + specifier: npm:apache-arrow@20.0.0 + version: apache-arrow@20.0.0 + apache-arrow-21: + specifier: npm:apache-arrow@21.2.0 + version: apache-arrow@21.2.0 eslint: specifier: ^8.57.0 version: 8.57.1 @@ -1703,6 +1712,9 @@ packages: '@types/node@22.7.4': resolution: {integrity: sha512-y+NPi1rFzDs1NdQHHToqeiX2TIS79SWEAw9GYhkkx8bD0ChpfqC+n2j5OXOCpzfojBEBt6DnEnnG9MY0zk1XLg==} + '@types/node@25.9.8': + resolution: {integrity: sha512-VfMrScDmMhUJQmd5hArdQnFvK0OIeD36uN2Va1FcpYaLB/BgkgM8Ulc50XtYISPMz7APJ30+N0T5EM0jlcdfRw==} + '@types/stack-utils@2.0.3': resolution: {integrity: sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==} @@ -1840,8 +1852,16 @@ packages: resolution: {integrity: sha512-gFlPaqN9osetbB83zC29AbbZqGiCuFH1vyyPseJ+B7SIbfBtESV62mMT/CkiIt77W6ykC/nTWFzTXFs0Uldg4g==} hasBin: true - apache-arrow@18.1.0: - resolution: {integrity: sha512-v/ShMp57iBnBp4lDgV8Jx3d3Q5/Hac25FWmQ98eMahUiHPXcvwIMKJD0hBIgclm/FCG+LwPkAKtkRO1O/W0YGg==} + apache-arrow@19.0.1: + resolution: {integrity: sha512-APmMLzS4qbTivLrPdQXexGM4JRr+0g62QDaobzEvip/FdQIrv2qLy0mD5Qdmw4buydtVJgbFeKR8f59I6PPGDg==} + hasBin: true + + apache-arrow@20.0.0: + resolution: {integrity: sha512-JUeK0jFRUd7rbmrhhzR3O2KXjLaZ4YYYFOptyUfxOsMIoZCPi6bZR58gVi/xi3HTBMPseXm9PXyQ2V916930pA==} + hasBin: true + + apache-arrow@21.2.0: + resolution: {integrity: sha512-Hxe6Agq26gQOM954qpzYSllJBPJl+e16U5CkfuMUhLrNba+5nKkttIVlflaovN6oaTratqMGAO8H5u/aNhmHWQ==} hasBin: true argparse@1.0.10: @@ -2033,6 +2053,15 @@ packages: resolution: {integrity: sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==} engines: {node: '>=4.0.0'} + command-line-args@6.0.2: + resolution: {integrity: sha512-AIjYVxrV9X752LmPDLbVYv8aMCuHPSLZJXEo2qo/xJfv+NYhaZ4sMSF01rM+gHPaMgvPM0l5D/F+Qx+i2WfSmQ==} + engines: {node: '>=12.20'} + peerDependencies: + '@75lb/nature': latest + peerDependenciesMeta: + '@75lb/nature': + optional: true + command-line-usage@7.0.4: resolution: {integrity: sha512-85UdvzTNx/+s5CkSgBm/0hzP80RFHAa7PsfeADE5ezZF3uHz3/Tqj9gIKGT9PTtpycc3Ua64T0oVulGfKxzfqg==} engines: {node: '>=12.20.0'} @@ -2281,6 +2310,15 @@ packages: resolution: {integrity: sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==} engines: {node: '>=4.0.0'} + find-replace@5.0.2: + resolution: {integrity: sha512-Y45BAiE3mz2QsrN2fb5QEtO4qb44NcS7en/0y9PEVsg351HsLeVclP8QPMH79Le9sH3rs5RSwJu99W0WPZO43Q==} + engines: {node: '>=14'} + peerDependencies: + '@75lb/nature': latest + peerDependenciesMeta: + '@75lb/nature': + optional: true + find-up@4.1.0: resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==} engines: {node: '>=8'} @@ -2302,6 +2340,9 @@ packages: flatbuffers@24.12.23: resolution: {integrity: sha512-dLVCAISd5mhls514keQzmEG6QHmUUsNuWsb4tFafIUwvvgDjXhtfAYSKOzt5SWOy+qByV5pbsDZ+Vb7HUOBEdA==} + flatbuffers@25.9.23: + resolution: {integrity: sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==} + flatted@3.4.2: resolution: {integrity: sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==} @@ -3345,6 +3386,9 @@ packages: undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + undici-types@7.24.6: + resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + unist-util-is@6.0.1: resolution: {integrity: sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==} @@ -5505,6 +5549,10 @@ snapshots: dependencies: undici-types: 6.19.8 + '@types/node@25.9.8': + dependencies: + undici-types: 7.24.6 + '@types/stack-utils@2.0.3': {} '@types/tmp@0.2.6': {} @@ -5694,17 +5742,40 @@ snapshots: json-bignum: 0.0.3 tslib: 2.8.1 - apache-arrow@18.1.0: + apache-arrow@19.0.1: dependencies: '@swc/helpers': 0.5.21 '@types/command-line-args': 5.2.3 '@types/command-line-usage': 5.0.4 '@types/node': 20.19.41 - command-line-args: 5.2.1 + command-line-args: 6.0.2 command-line-usage: 7.0.4 flatbuffers: 24.12.23 json-bignum: 0.0.3 tslib: 2.8.1 + transitivePeerDependencies: + - '@75lb/nature' + + apache-arrow@20.0.0: + dependencies: + '@swc/helpers': 0.5.21 + '@types/command-line-args': 5.2.3 + '@types/command-line-usage': 5.0.4 + '@types/node': 20.19.41 + command-line-args: 6.0.2 + command-line-usage: 7.0.4 + flatbuffers: 25.9.23 + json-bignum: 0.0.3 + tslib: 2.8.1 + transitivePeerDependencies: + - '@75lb/nature' + + apache-arrow@21.2.0: + dependencies: + '@types/node': 25.9.8 + flatbuffers: 25.9.23 + json-with-bigint: 3.5.8 + tslib: 2.8.1 argparse@1.0.10: dependencies: @@ -5906,6 +5977,13 @@ snapshots: lodash.camelcase: 4.3.0 typical: 4.0.0 + command-line-args@6.0.2: + dependencies: + array-back: 6.2.3 + find-replace: 5.0.2 + lodash.camelcase: 4.3.0 + typical: 7.3.0 + command-line-usage@7.0.4: dependencies: array-back: 6.2.3 @@ -6178,6 +6256,8 @@ snapshots: dependencies: array-back: 3.1.0 + find-replace@5.0.2: {} + find-up@4.1.0: dependencies: locate-path: 5.0.0 @@ -6201,6 +6281,8 @@ snapshots: flatbuffers@24.12.23: {} + flatbuffers@25.9.23: {} + flatted@3.4.2: {} follow-redirects@1.16.0: {} @@ -7405,6 +7487,8 @@ snapshots: undici-types@6.21.0: {} + undici-types@7.24.6: {} + unist-util-is@6.0.1: dependencies: '@types/unist': 3.0.3 diff --git a/nodejs/src/blob.rs b/nodejs/src/blob.rs index 230a11bae..59cf61af6 100644 --- a/nodejs/src/blob.rs +++ b/nodejs/src/blob.rs @@ -1,25 +1,45 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors +use std::future::Future; use std::ops::Range; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use arrow_array::{Array, LargeBinaryArray}; +use futures::future::abortable; use lancedb::blob::BlobFile as LanceBlobFile; +use napi::Env; use napi::bindgen_prelude::*; use napi_derive::napi; use crate::error::convert_error; +#[napi(object)] +pub struct BlobRange { + pub start: BigInt, + pub end: BigInt, +} + #[napi] pub struct BlobFile { inner: Arc, + closed: AtomicBool, } impl BlobFile { pub(crate) fn new(inner: LanceBlobFile) -> Self { Self { inner: Arc::new(inner), + closed: AtomicBool::new(false), + } + } + + fn ensure_open(&self) -> napi::Result<()> { + if self.closed.load(Ordering::Acquire) { + Err(napi::Error::from_reason("blob file is already closed")) + } else { + Ok(()) } } } @@ -32,21 +52,124 @@ impl BlobFile { } #[napi] - pub async fn read(&self) -> napi::Result { - let bytes = self.inner.read().await.map_err(|err| convert_error(&err))?; - Ok(Buffer::from(bytes.as_ref())) + pub fn read<'env>( + &self, + env: &'env Env, + max_bytes: Option, + signal: Option, + ) -> napi::Result> { + self.ensure_open()?; + let max_bytes = max_bytes + .map(|max_bytes| { + usize::try_from(parse_u64(max_bytes, "maxBytes")?) + .map_err(|_| napi::Error::from_reason("maxBytes is too large")) + }) + .transpose()?; + let inner = self.inner.clone(); + spawn_abortable(env, signal, async move { + let bytes = match max_bytes { + None => inner.read().await, + Some(max_bytes) => inner.read_up_to(max_bytes).await, + } + .map_err(|err| convert_error(&err))?; + Ok(Buffer::from(bytes.as_ref())) + }) } #[napi] - pub async fn read_range(&self, start: BigInt, end: BigInt) -> napi::Result { + pub fn read_range<'env>( + &self, + env: &'env Env, + start: BigInt, + end: BigInt, + signal: Option, + ) -> napi::Result> { + self.ensure_open()?; let range = bigint_range(start, end)?; - let bytes = self - .inner - .read_range(range) - .await - .map_err(|err| convert_error(&err))?; - Ok(Buffer::from(bytes.as_ref())) + let inner = self.inner.clone(); + spawn_abortable(env, signal, async move { + let bytes = inner + .read_range(range) + .await + .map_err(|err| convert_error(&err))?; + Ok(Buffer::from(bytes.as_ref())) + }) } + + #[napi] + pub fn read_ranges<'env>( + &self, + env: &'env Env, + ranges: Vec, + signal: Option, + ) -> napi::Result>> { + self.ensure_open()?; + let ranges = ranges + .into_iter() + .map(|range| bigint_range(range.start, range.end)) + .collect::>>()?; + let inner = self.inner.clone(); + spawn_abortable(env, signal, async move { + let buffers = inner + .read_ranges(&ranges) + .await + .map_err(|err| convert_error(&err))?; + Ok(buffers + .iter() + .map(|bytes| Buffer::from(bytes.as_ref())) + .collect()) + }) + } + + #[napi] + pub async fn seek(&self, position: BigInt) -> napi::Result<()> { + self.ensure_open()?; + let position = parse_u64(position, "position")?; + self.inner + .seek(position) + .await + .map_err(|err| convert_error(&err)) + } + + #[napi] + pub async fn tell(&self) -> napi::Result { + self.ensure_open()?; + let position = self.inner.tell().await.map_err(|err| convert_error(&err))?; + Ok(BigInt::from(position)) + } + + #[napi] + pub async fn close(&self) -> napi::Result<()> { + if self.closed.swap(true, Ordering::AcqRel) { + return Ok(()); + } + self.inner.close().await.map_err(|err| convert_error(&err)) + } + + #[napi] + pub fn is_closed(&self) -> bool { + self.closed.load(Ordering::Acquire) + } +} + +/// Runs `fut` on the napi runtime. Aborting `signal` drops the future, which +/// cancels in-flight reads. +pub fn spawn_abortable<'env, T>( + env: &'env Env, + signal: Option, + fut: impl Future> + Send + 'static, +) -> napi::Result> +where + T: ToNapiValue + Send + 'static, +{ + let (fut, handle) = abortable(fut); + if let Some(signal) = signal { + signal.on_abort(move || handle.abort()); + } + env.spawn_future(async move { + fut.await + .unwrap_or_else(|_| Err(napi::Error::from_reason("blob read aborted"))) + }) } fn bigint_range(start: BigInt, end: BigInt) -> napi::Result> { diff --git a/nodejs/src/iterator.rs b/nodejs/src/iterator.rs index 8d65dfc3f..96d08c765 100644 --- a/nodejs/src/iterator.rs +++ b/nodejs/src/iterator.rs @@ -7,6 +7,8 @@ use lancedb::ipc::batches_to_ipc_file; use napi::bindgen_prelude::*; use napi_derive::napi; +use crate::util::schema_to_buffer; + /** Typescript-style Async Iterator over RecordBatches */ #[napi] pub struct RecordBatchIterator { @@ -19,6 +21,12 @@ impl RecordBatchIterator { Self { inner } } + /// Return the executed stream's output schema as an Arrow IPC file. + #[napi] + pub fn schema(&self) -> napi::Result { + schema_to_buffer(&self.inner.schema()) + } + #[napi(catch_unwind)] pub async unsafe fn next(&mut self) -> napi::Result> { if let Some(rst) = self.inner.next().await { diff --git a/nodejs/src/query.rs b/nodejs/src/query.rs index f9fe91d35..712569b2f 100644 --- a/nodejs/src/query.rs +++ b/nodejs/src/query.rs @@ -289,8 +289,13 @@ impl VectorQuery { } #[napi] - pub fn nprobes(&mut self, nprobe: u32) { - self.inner = self.inner.clone().nprobes(nprobe as usize); + pub fn nprobes(&mut self, nprobe: u32) -> napi::Result<()> { + self.inner = self + .inner + .clone() + .nprobes(nprobe as usize) + .default_error()?; + Ok(()) } #[napi] @@ -658,6 +663,7 @@ impl JsFullTextQuery { FtsQuery::Phrase(_) => "phrase".to_string(), FtsQuery::Boost(_) => "boost".to_string(), FtsQuery::MultiMatch(_) => "multi_match".to_string(), + FtsQuery::CombinedFields(_) => "combined_fields".to_string(), FtsQuery::Boolean(_) => "boolean".to_string(), } } diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index fa74c20dc..f5c3b2987 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -11,11 +11,12 @@ use lancedb::table::{ FieldMetadataUpdate as LanceFieldMetadataUpdate, FtsToken as LanceDbFtsToken, NewColumnTransform, OptimizeAction, OptimizeOptions, Ref, Table as LanceDbTable, }; +use napi::Env; use napi::bindgen_prelude::*; use napi::threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode}; use napi_derive::napi; -use crate::blob::{BlobFile, copy_blob_buffers, parse_row_ids}; +use crate::blob::{BlobFile, copy_blob_buffers, parse_row_ids, spawn_abortable}; use crate::error::NapiErrorExt; use crate::index::Index; use crate::merge::NativeMergeInsertBuilder; @@ -74,13 +75,14 @@ impl Table { #[napi( catch_unwind, - ts_args_type = "buf: Buffer, mode: string, progressCallback?: (progress: WriteProgressInfo) => void" + ts_args_type = "buf: Buffer, mode: string, progressCallback?: (progress: WriteProgressInfo) => void, allowExternalBlobOutsideBases?: boolean" )] pub async fn add( &self, buf: Buffer, mode: String, progress_callback: Option, + allow_external_blob_outside_bases: Option, ) -> napi::Result { let batches = ipc_file_to_batches(buf.to_vec()) .map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?; @@ -105,6 +107,9 @@ impl Table { return Err(napi::Error::from_reason(format!("Invalid mode: {}", mode))); }; + op = op + .allow_external_blob_outside_bases(allow_external_blob_outside_bases.unwrap_or(false)); + if let Some(tsfn) = progress_callback { op = op.progress(move |p| { // NonBlocking: dispatch onto the JS event loop without @@ -336,36 +341,44 @@ impl Table { } #[napi(catch_unwind)] - pub async fn fetch_blobs( + pub fn fetch_blobs<'env>( &self, + env: &'env Env, column: String, row_ids: Vec, - ) -> napi::Result>> { + signal: Option, + ) -> napi::Result>>> { let row_ids = parse_row_ids(row_ids)?; - let array = self - .inner_ref()? - .fetch_blobs(column.as_str(), &row_ids) - .await - .default_error()?; - Ok(copy_blob_buffers(array)) + let table = self.inner_ref()?.clone(); + spawn_abortable(env, signal, async move { + let array = table + .fetch_blobs(column.as_str(), &row_ids) + .await + .default_error()?; + Ok(copy_blob_buffers(array)) + }) } #[napi(catch_unwind)] - pub async fn fetch_blob_files( + pub fn fetch_blob_files<'env>( &self, + env: &'env Env, column: String, row_ids: Vec, - ) -> napi::Result>> { + signal: Option, + ) -> napi::Result>>> { let row_ids = parse_row_ids(row_ids)?; - let files = self - .inner_ref()? - .fetch_blob_files(column.as_str(), &row_ids) - .await - .default_error()?; - Ok(files - .into_iter() - .map(|file| file.map(BlobFile::new)) - .collect()) + let table = self.inner_ref()?.clone(); + spawn_abortable(env, signal, async move { + let files = table + .fetch_blob_files(column.as_str(), &row_ids) + .await + .default_error()?; + Ok(files + .into_iter() + .map(|file| file.map(BlobFile::new)) + .collect()) + }) } #[napi(catch_unwind)] diff --git a/nodejs/tsconfig.json b/nodejs/tsconfig.json index 9b83da5ec..094fe5003 100644 --- a/nodejs/tsconfig.json +++ b/nodejs/tsconfig.json @@ -10,7 +10,10 @@ "resolveJsonModule": true, "emitDecoratorMetadata": true, "experimentalDecorators": true, - "moduleResolution": "Node" + "moduleResolution": "Node", + // flatbuffers 25 (via apache-arrow 21) ships declarations that need + // TypeScript 5.7+ generic typed arrays; skip checking third-party .d.ts. + "skipLibCheck": true }, "exclude": ["./dist/*", "./examples/*"], "typedocOptions": { diff --git a/python/Cargo.toml b/python/Cargo.toml index 17ff3f4fc..9ba96b55f 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.40.0-beta.11" +version = "0.40.0-beta.12" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index d34b541a8..d3a678359 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -46,6 +46,7 @@ from .materialized_view import ( AsyncMaterializedView, MaterializedView, MaterializedViewDefinition, + vector_dedup as vector_dedup, ) from .view import ViewDescription as ViewDescription from .table import AsyncTable, Table diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 1f26ad6f4..176e57ac3 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -149,12 +149,20 @@ class Connection(object): limit: Optional[int], ) -> list[str]: ... # Deprecated: Use list_tables instead async def open_job(self, job_id: str) -> Job: ... - async def create_function_async(self, request_json: str) -> Job: ... - async def get_function(self, name: str, version: str) -> str: ... - async def list_functions(self) -> List[str]: ... - async def drop_function(self, name: str, version: str) -> bool: ... + async def create_function_async( + self, request_json: str, namespace_path: Optional[List[str]] = None + ) -> Job: ... + async def get_function( + self, name: str, version: str, namespace_path: Optional[List[str]] = None + ) -> str: ... + async def list_functions( + self, namespace_path: Optional[List[str]] = None + ) -> List[str]: ... + async def drop_function( + self, name: str, version: str, namespace_path: Optional[List[str]] = None + ) -> bool: ... async def drop_function_async( - self, name: str, version: str + self, name: str, version: str, namespace_path: Optional[List[str]] = None ) -> Tuple[bool, Job]: ... async def create_secret( self, name: str, value: str, namespace_path: Optional[List[str]] = None @@ -247,6 +255,8 @@ class Connection(object): filter: Optional[str] = None, limit: Optional[int] = None, with_no_data: bool = False, + *, + vector_source_json: Optional[str] = None, ) -> Table: ... async def create_materialized_view_async( self, @@ -256,6 +266,8 @@ class Connection(object): filter: Optional[str] = None, limit: Optional[int] = None, with_no_data: bool = False, + *, + vector_source_json: Optional[str] = None, ) -> Job: ... async def list_materialized_views(self) -> List[str]: ... async def drop_materialized_view( diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index 02de75460..1af3c1b4a 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -56,6 +56,8 @@ from .sql import QueryDescription from .materialized_view import ( AsyncMaterializedView, MaterializedView, + MaterializedViewSource, + VectorDedupSource, SelectArg, normalize_select, ) @@ -120,7 +122,48 @@ def _view_description( class DBConnection(EnforceOverrides): - """An active LanceDB connection interface.""" + """An active LanceDB connection interface. + + Use [close][lancedb.db.DBConnection.close] to release the connection's + underlying resources, or use the connection as a context manager to close it + automatically when leaving the block, including when an exception is raised. + + Examples + -------- + >>> import lancedb + >>> with lancedb.connect("memory://") as db: + ... assert db.is_open() + >>> db.is_open() + False + """ + + def __enter__(self) -> DBConnection: + return self + + def __exit__(self, *_) -> None: + self.close() + + @abstractmethod + def is_open(self) -> bool: + """Return True if the connection is open.""" + pass + + @abstractmethod + def close(self) -> None: + """Close the connection, releasing any underlying resources. + + It is safe to call this method multiple times. Database operations on a + closed connection raise ``RuntimeError: Connection is closed``. + + Examples + -------- + >>> import lancedb + >>> db = lancedb.connect("memory://") + >>> db.close() + >>> db.is_open() + False + """ + pass def list_namespaces( self, @@ -366,9 +409,10 @@ class DBConnection(EnforceOverrides): - [LanceModel][lancedb.pydantic.LanceModel] mode: str; default "create" The mode to use when creating the table. - Can be either "create" or "overwrite". + Can be "create", "overwrite", or "exist_ok". By default, if the table already exists, an exception is raised. If you want to overwrite the table, use mode="overwrite". + To open an existing table without adding data, use mode="exist_ok". exist_ok: bool, default False If a table by the same name already exists, then raise an exception if exist_ok=False. If exist_ok=True, then open the existing table; @@ -555,7 +599,7 @@ class DBConnection(EnforceOverrides): def create_materialized_view( self, name: str, - source: str, + source: MaterializedViewSource, *, select: SelectArg = None, where: Optional[str] = None, @@ -569,17 +613,21 @@ class DBConnection(EnforceOverrides): table. The view is a normal table: it can be queried, indexed and searched, and it appears in ``table_names``. - The source table must have stable row ids (create it with the + Ordinary source tables must have stable row ids (create them with the ``new_table_enable_stable_row_ids`` storage option): they keep the view's provenance valid across source compactions, and cannot be - enabled after a table exists. + enabled after a table exists. Native dedup sources also support physical + row IDs interpreted against their pinned snapshot. Parameters ---------- name: str The name of the view. - source: str - The name of the source table, in this database. + source: str or VectorDedupSource + The name of the source table, or an indexed dedup source declared + with [vector_dedup][lancedb.vector_dedup]. Dedup sources capture a + fixed snapshot and preserve the source table. They accept no + additional select, where or limit options. select: list or dict, optional The view's columns: column names, ``(alias, SQL expression)`` pairs, or a dict of the same. Omitting it selects every source @@ -602,7 +650,7 @@ class DBConnection(EnforceOverrides): def create_materialized_view_async( self, name: str, - source: str, + source: MaterializedViewSource, *, select: SelectArg = None, where: Optional[str] = None, @@ -778,6 +826,7 @@ class DBConnection(EnforceOverrides): definition: UdfDefinition, *, secrets: Optional[Sequence[EnvVarSecret]] = None, + namespace_path: Optional[List[str]] = None, ) -> FunctionVersion: """Build and register a scalar Python UDF, then return its version. @@ -794,6 +843,9 @@ class DBConnection(EnforceOverrides): Function needs, each naming a Secret and the environment variable its value arrives in. The Function's source is unchanged by this; it reads the variable the way it already did. + namespace_path : list of str, optional + The namespace holding the Function. None or an empty list is the + root namespace. Examples -------- @@ -809,13 +861,16 @@ class DBConnection(EnforceOverrides): ) ``` """ - return self.create_function_async(definition, secrets=secrets).wait() + return self.create_function_async( + definition, secrets=secrets, namespace_path=namespace_path + ).wait() def create_function_async( self, definition: UdfDefinition, *, secrets: Optional[Sequence[EnvVarSecret]] = None, + namespace_path: Optional[List[str]] = None, ) -> Job[FunctionVersion]: """Submit a scalar Python UDF for building and registration. @@ -827,17 +882,26 @@ class DBConnection(EnforceOverrides): "Function catalog operations are not supported for this connection type" ) - def get_function(self, name: str, *, version: str) -> FunctionVersion: + def get_function( + self, + name: str, + *, + version: str, + namespace_path: Optional[List[str]] = None, + ) -> FunctionVersion: """Open one exact immutable Function version from the remote catalog.""" raise NotImplementedError( "Function catalog operations are not supported for this connection type" ) - def list_functions(self) -> List[FunctionVersion]: - """List every published immutable Function version. + def list_functions( + self, *, namespace_path: Optional[List[str]] = None + ) -> List[FunctionVersion]: + """List every published immutable Function version in a namespace. - Results are ordered by Function name then version. Local connections - raise ``NotImplementedError``. + Functions in child namespaces are not included. Results are ordered by + Function name then version. Local connections raise + ``NotImplementedError``. Examples -------- @@ -851,7 +915,13 @@ class DBConnection(EnforceOverrides): "Function catalog operations are not supported for this connection type" ) - def drop_function(self, name: str, *, version: str) -> bool: + def drop_function( + self, + name: str, + *, + version: str, + namespace_path: Optional[List[str]] = None, + ) -> bool: """Drop a Function name and the object it was bound to. The requested version must exist in the currently named object. Returns @@ -864,7 +934,13 @@ class DBConnection(EnforceOverrides): "Function catalog operations are not supported for this connection type" ) - def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]: + def drop_function_async( + self, + name: str, + *, + version: str, + namespace_path: Optional[List[str]] = None, + ) -> Tuple[bool, Job]: """Drop a Function name and return its cleanup job. The name is unbound before this returns; the object's content may still @@ -1241,6 +1317,14 @@ class LanceDBConnection(DBConnection): def __repr__(self) -> str: return f"{self.__class__.__name__}(uri={self._conn.uri!r})" + @override + def is_open(self) -> bool: + return self._conn.is_open() + + @override + def close(self) -> None: + self._conn.close() + @override def serialize(self) -> str: import json @@ -1463,8 +1547,6 @@ class LanceDBConnection(DBConnection): """ if namespace_path is None: namespace_path = [] - if mode.lower() not in ["create", "overwrite"]: - raise ValueError("mode must be either 'create' or 'overwrite'") validate_table_name(name) tbl = LanceTable.create( @@ -1558,7 +1640,7 @@ class LanceDBConnection(DBConnection): def create_materialized_view( self, name: str, - source: str, + source: MaterializedViewSource, *, select: SelectArg = None, where: Optional[str] = None, @@ -1603,7 +1685,7 @@ class LanceDBConnection(DBConnection): def create_materialized_view_async( self, name: str, - source: str, + source: MaterializedViewSource, *, select: SelectArg = None, where: Optional[str] = None, @@ -1802,25 +1884,62 @@ class LanceDBConnection(DBConnection): definition: UdfDefinition, *, secrets: Optional[Sequence[EnvVarSecret]] = None, + namespace_path: Optional[List[str]] = None, ) -> Job[FunctionVersion]: - job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets)) + job = LOOP.run( + self._conn.create_function_async( + definition, secrets=secrets, namespace_path=namespace_path + ) + ) return Job(job) @override - def get_function(self, name: str, *, version: str) -> FunctionVersion: - return LOOP.run(self._conn.get_function(name, version=version)) + def get_function( + self, + name: str, + *, + version: str, + namespace_path: Optional[List[str]] = None, + ) -> FunctionVersion: + return LOOP.run( + self._conn.get_function( + name, version=version, namespace_path=namespace_path + ) + ) @override - def list_functions(self) -> List[FunctionVersion]: - return LOOP.run(self._conn.list_functions()) + def list_functions( + self, *, namespace_path: Optional[List[str]] = None + ) -> List[FunctionVersion]: + return LOOP.run(self._conn.list_functions(namespace_path=namespace_path)) @override - def drop_function(self, name: str, *, version: str) -> bool: - return LOOP.run(self._conn.drop_function(name, version=version)) + def drop_function( + self, + name: str, + *, + version: str, + namespace_path: Optional[List[str]] = None, + ) -> bool: + return LOOP.run( + self._conn.drop_function( + name, version=version, namespace_path=namespace_path + ) + ) @override - def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]: - dropped, job = LOOP.run(self._conn.drop_function_async(name, version=version)) + def drop_function_async( + self, + name: str, + *, + version: str, + namespace_path: Optional[List[str]] = None, + ) -> Tuple[bool, Job]: + dropped, job = LOOP.run( + self._conn.drop_function_async( + name, version=version, namespace_path=namespace_path + ) + ) return dropped, Job(job) @override @@ -2230,11 +2349,12 @@ class AsyncConnection(object): - pyarrow.Schema - [LanceModel][lancedb.pydantic.LanceModel] - mode: Literal["create", "overwrite"]; default "create" + mode: Literal["create", "overwrite", "exist_ok"]; default "create" The mode to use when creating the table. - Can be either "create" or "overwrite". + Can be "create", "overwrite", or "exist_ok". By default, if the table already exists, an exception is raised. If you want to overwrite the table, use mode="overwrite". + To open an existing table without adding data, use mode="exist_ok". exist_ok: bool, default False If a table by the same name already exists, then raise an exception if exist_ok=False. If exist_ok=True, then open the existing table; @@ -2498,10 +2618,31 @@ class AsyncConnection(object): await tbl.checkout(version) return tbl + async def _materialized_view_source( + self, + source: MaterializedViewSource, + select: SelectArg, + where: Optional[str], + limit: Optional[int], + ) -> Tuple[str, Dict[str, str]]: + if not isinstance(source, VectorDedupSource): + return source, {} + if select is not None or where is not None or limit is not None: + raise ValueError( + "vector_dedup cannot be combined with select, where or limit" + ) + version = source.dataset_version + if version is None: + table = await self.open_table(source.source) + version = await table.version() + return source.source, { + "vector_source_json": source._native_source_json(version) + } + async def create_materialized_view( self, name: str, - source: str, + source: MaterializedViewSource, *, select: SelectArg = None, where: Optional[str] = None, @@ -2512,6 +2653,9 @@ class AsyncConnection(object): See [DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view]. """ + source, native_options = await self._materialized_view_source( + source, select, where, limit + ) inner = await self._inner.create_materialized_view( name, source, @@ -2519,13 +2663,14 @@ class AsyncConnection(object): filter=where, limit=limit, with_no_data=with_no_data, + **native_options, ) return AsyncMaterializedView(AsyncTable(inner)) async def create_materialized_view_async( self, name: str, - source: str, + source: MaterializedViewSource, *, select: SelectArg = None, where: Optional[str] = None, @@ -2536,6 +2681,9 @@ class AsyncConnection(object): Wait for the returned job before opening or querying the view. """ + source, native_options = await self._materialized_view_source( + source, select, where, limit + ) inner = await self._inner.create_materialized_view_async( name, source, @@ -2543,6 +2691,7 @@ class AsyncConnection(object): filter=where, limit=limit, with_no_data=with_no_data, + **native_options, ) return AsyncJob(inner) @@ -2752,6 +2901,7 @@ class AsyncConnection(object): definition: UdfDefinition, *, secrets: Optional[Sequence[EnvVarSecret]] = None, + namespace_path: Optional[List[str]] = None, ) -> AsyncJob[FunctionVersion]: """Submit a scalar Python UDF for building and registration. @@ -2759,36 +2909,60 @@ class AsyncConnection(object): artifact. Waiting on the job returns the immutable Function version. ``secrets`` is a sequence of [EnvVarSecret][lancedb.secrets.EnvVarSecret], each naming a Secret and - the environment variable its value arrives in. Local connections raise - ``NotImplementedError``. + the environment variable its value arrives in. ``namespace_path`` is + the namespace the Function is registered in; None or an empty list is + the root. Local connections raise ``NotImplementedError``. """ if not isinstance(definition, UdfDefinition): raise TypeError("create_function_async requires a @udf definition") request = definition.bind_secrets(secrets) - inner = await self._inner.create_function_async(request.to_canonical_json()) + inner = await self._inner.create_function_async( + request.to_canonical_json(), namespace_path + ) return _typed_job(inner, FunctionVersion.from_json) - async def get_function(self, name: str, *, version: str) -> FunctionVersion: + async def get_function( + self, + name: str, + *, + version: str, + namespace_path: Optional[List[str]] = None, + ) -> FunctionVersion: """Open one exact immutable Function version from the remote catalog.""" - return FunctionVersion.from_json(await self._inner.get_function(name, version)) + return FunctionVersion.from_json( + await self._inner.get_function(name, version, namespace_path) + ) - async def list_functions(self) -> List[FunctionVersion]: - """List every published immutable Function version. + async def list_functions( + self, *, namespace_path: Optional[List[str]] = None + ) -> List[FunctionVersion]: + """List every published immutable Function version in a namespace. - Results are ordered by Function name then version. Local connections - raise ``NotImplementedError``. + Functions in child namespaces are not included. Results are ordered by + Function name then version. Local connections raise + ``NotImplementedError``. """ return [ FunctionVersion.from_json(value) - for value in await self._inner.list_functions() + for value in await self._inner.list_functions(namespace_path) ] - async def drop_function(self, name: str, *, version: str) -> bool: + async def drop_function( + self, + name: str, + *, + version: str, + namespace_path: Optional[List[str]] = None, + ) -> bool: """Drop a Function name and the object it was bound to.""" - return await self._inner.drop_function(name, version) + return await self._inner.drop_function(name, version, namespace_path) async def drop_function_async( - self, name: str, *, version: str + self, + name: str, + *, + version: str, + namespace_path: Optional[List[str]] = None, ) -> Tuple[bool, AsyncJob]: """Drop a Function name and return its cleanup job. @@ -2796,7 +2970,9 @@ class AsyncConnection(object): be being deleted. Await :meth:`AsyncJob.wait` to wait for that to finish. """ - dropped, job = await self._inner.drop_function_async(name, version) + dropped, job = await self._inner.drop_function_async( + name, version, namespace_path + ) return dropped, AsyncJob(job) async def create_secret( diff --git a/python/python/lancedb/embeddings/base.py b/python/python/lancedb/embeddings/base.py index 149b9c089..fbe43860d 100644 --- a/python/python/lancedb/embeddings/base.py +++ b/python/python/lancedb/embeddings/base.py @@ -182,7 +182,11 @@ class EmbeddingFunction(BaseModel, ABC): def VectorField(self, **kwargs): """ Creates a pydantic Field that can automatically annotate - the target vector column for this embedding function + the target vector column for this embedding function. + + The field can be annotated as ``list[float]`` for compatibility with + static type checkers. LanceDB will infer the fixed vector dimension from + this embedding function. """ return Field(json_schema_extra={"vector_column_for": self}, **kwargs) diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index 5f26b590e..c170c7375 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -356,9 +356,16 @@ _ObjectVersion = Annotated[ class FunctionVersion(_RemoteValue): - """A pinned object revision, independent of its executable image digest.""" + """A pinned object revision, independent of its executable image digest. + + ``name`` and ``namespace_path`` are the Function's parts, as a + [SecretReference][lancedb.functions.SecretReference] carries a Secret's; + they are what [get_function][lancedb.db.DBConnection.get_function] takes. + An empty ``namespace_path`` is the root namespace. + """ name: str + namespace_path: tuple[str, ...] = () object_id: str location: str version: _ObjectVersion @@ -444,6 +451,7 @@ class FunctionVersion(_RemoteValue): return FunctionApplication( function=FunctionVersionRef( name=self.name, + namespace_path=self.namespace_path, object_id=self.object_id, location=self.location, version=self.version, @@ -502,6 +510,7 @@ class FunctionRegistrationRequest(_RemoteValue): class FunctionVersionRef(_OpenRemoteValue): name: str + namespace_path: tuple[str, ...] = () object_id: str location: str version: _ObjectVersion diff --git a/python/python/lancedb/materialized_view.py b/python/python/lancedb/materialized_view.py index a836210cb..b4d455870 100644 --- a/python/python/lancedb/materialized_view.py +++ b/python/python/lancedb/materialized_view.py @@ -7,6 +7,7 @@ maintained by refresh. See ``DBConnection.create_materialized_view``.""" from __future__ import annotations import json +import math from dataclasses import dataclass from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Tuple, Union @@ -20,6 +21,7 @@ if TYPE_CHECKING: from .table import AsyncTable, LanceTable DEFINITION_META_KEY = b"mv.definition" +_MAX_FLOAT32 = float.fromhex("0x1.fffffep+127") SelectArg = Union[ str, @@ -29,9 +31,104 @@ SelectArg = Union[ ] -DEFINITION_FORMAT = 2 +@dataclass(frozen=True) +class VectorDedupSource: + """A declarative source of retained original rows from an indexed snapshot. + + Construct with [vector_dedup][lancedb.vector_dedup] and pass it as the + ``source`` of ``create_materialized_view`` or + ``create_materialized_view_async``. An omitted ``dataset_version`` is + captured once when creation is submitted, not when this object is made. + Refreshes keep that captured version. The source table is never modified. + """ + + source: str + column: str + distance_threshold: float + dataset_version: Optional[int] = None + + def __post_init__(self): + for name in ("source", "column"): + value = getattr(self, name) + if not isinstance(value, str) or not value: + raise ValueError(f"{name} must be a non-empty string") + if self.dataset_version is not None and ( + type(self.dataset_version) is not int + or not 0 < self.dataset_version < 2**64 + ): + raise ValueError("dataset_version must be a positive uint64 integer") + value = self.distance_threshold + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TypeError("distance_threshold must be a number") + try: + value = float(value) + except OverflowError as exc: + raise ValueError("distance_threshold must fit a finite float32") from exc + if not math.isfinite(value) or abs(value) > _MAX_FLOAT32: + raise ValueError("distance_threshold must fit a finite float32") + object.__setattr__(self, "distance_threshold", value) + + def _native_source_json(self, version: int) -> str: + return json.dumps( + { + "kind": "Dedup", + "dataset_version": version, + "column": self.column, + "distance_threshold": repr(self.distance_threshold), + }, + allow_nan=False, + ) + + +def vector_dedup( + source: str, + *, + column: str, + distance_threshold: float, + dataset_version: Optional[int] = None, +) -> VectorDedupSource: + """Declare an indexed dedup source for a materialized view. + + The default policy keeps the smallest unassigned snapshot row ID and removes + only its direct qualifying neighbors: an A-B-C chain without an A-C edge + retains A and C. Rows with no pair, including null vectors, survive. + Only same-segment/same-IVF-partition pairs are considered; quantized distances + use reconstructed index vectors. The index must cover the selected snapshot. + + ``source`` is an exact table name in the connection's root namespace, just + as for ``create_materialized_view``. ``dataset_version=None`` captures the + source's version when creation is submitted. Pass an explicit positive + version to reproduce a previous snapshot. The result contains original + source columns and rows. Do not combine this source with ``select``, + ``where`` or ``limit``; query the resulting table for further transforms. + + Creation follows the usual MV contract: by default it waits for the result. + Use ``with_no_data=True`` to declare only, or + ``create_materialized_view_async`` to get a job handle. On Cloud/Enterprise, + execution uses server-side MV registry jobs; local connections run in process. + + Examples + -------- + Given a connected ``db`` and an indexed ``images.phash`` column:: + + from lancedb import vector_dedup + + view = db.create_materialized_view( + "images_clean", + vector_dedup("images", column="phash", distance_threshold=4), + ) + cleaned = view.table + """ + return VectorDedupSource(source, column, distance_threshold, dataset_version) + + +MaterializedViewSource = Union[str, VectorDedupSource] + + +DEFINITION_FORMAT = 4 """The newest stored layout this version reads: ``{"format": N, "query": ""}``, -format 2 being a query with ``GROUP BY``. A ``kind`` key beside it is for +format 2 being a query with ``GROUP BY``, format 3 native pairs and format 4 +native dedup results. A ``kind`` key beside it is for readers older than the format number.""" diff --git a/python/python/lancedb/namespace.py b/python/python/lancedb/namespace.py index fe38a47c7..16be63e2d 100644 --- a/python/python/lancedb/namespace.py +++ b/python/python/lancedb/namespace.py @@ -68,6 +68,7 @@ from lance_namespace import ( from lancedb.materialized_view import ( AsyncMaterializedView, MaterializedView, + MaterializedViewSource, SelectArg, ) from lancedb.table import AsyncTable, LanceTable, Table @@ -488,6 +489,14 @@ class LanceNamespaceDBConnection(DBConnection): ) self._uri = self._inner.uri + @override + def is_open(self) -> bool: + return self._inner.is_open() + + @override + def close(self) -> None: + self._inner.close() + @override def serialize(self) -> str: import json @@ -632,7 +641,7 @@ class LanceNamespaceDBConnection(DBConnection): def create_materialized_view( self, name: str, - source: str, + source: MaterializedViewSource, *, select: "SelectArg" = None, where: Optional[str] = None, @@ -662,7 +671,7 @@ class LanceNamespaceDBConnection(DBConnection): def create_materialized_view_async( self, name: str, - source: str, + source: MaterializedViewSource, *, select: "SelectArg" = None, where: Optional[str] = None, @@ -1242,7 +1251,7 @@ class AsyncLanceNamespaceDBConnection: async def create_materialized_view( self, name: str, - source: str, + source: MaterializedViewSource, *, select: "SelectArg" = None, where: Optional[str] = None, @@ -1265,7 +1274,7 @@ class AsyncLanceNamespaceDBConnection: async def create_materialized_view_async( self, name: str, - source: str, + source: MaterializedViewSource, *, select: "SelectArg" = None, where: Optional[str] = None, diff --git a/python/python/lancedb/pydantic.py b/python/python/lancedb/pydantic.py index 528d865d7..313c514c3 100644 --- a/python/python/lancedb/pydantic.py +++ b/python/python/lancedb/pydantic.py @@ -88,6 +88,17 @@ def Vector( ... pa.field("url", pa.utf8(), False), ... pa.field("embeddings", pa.list_(pa.float32(), 768)) ... ]) + + Notes + ----- + ``Vector`` creates a type dynamically, so calls such as ``Vector(768)`` are + not valid static type annotations. For an embedding field, use the standard + ``list[float]`` annotation when running mypy; ``VectorField`` supplies the + fixed dimension to LanceDB:: + + class MyModel(LanceModel): + text: str = embeddings.SourceField() + vector: list[float] = embeddings.VectorField() """ # TODO: make a public parameterized type. @@ -313,6 +324,10 @@ def _unwrap_optional_annotation(annotation: Any) -> Any | None: def _pydantic_to_arrow_type(field: FieldInfo) -> pa.DataType: """Convert a Pydantic FieldInfo to Arrow DataType""" + embedding_vector_type = _embedding_vector_to_arrow_type(field) + if embedding_vector_type is not None: + return embedding_vector_type + unwrapped = _unwrap_optional_annotation(field.annotation) if unwrapped is not None: return _pydantic_type_to_arrow_type(unwrapped, field) @@ -326,8 +341,32 @@ def _pydantic_to_arrow_type(field: FieldInfo) -> pa.DataType: return _pydantic_type_to_arrow_type(field.annotation, field) +def _embedding_vector_to_arrow_type(field: FieldInfo) -> pa.DataType | None: + """Infer a fixed-size vector type from ``VectorField`` metadata.""" + if not _is_embedding_vector_annotation(field): + return None + + function = get_extras(field, "vector_column_for") + return pa.list_(pa.float32(), function.ndims()) + + +def _is_embedding_vector_annotation(field: FieldInfo) -> bool: + if get_extras(field, "vector_column_for") is None: + return False + + annotation = _unwrap_optional_annotation(field.annotation) + if annotation is None: + annotation = field.annotation + + origin = getattr(annotation, "__origin__", None) + args = getattr(annotation, "__args__", ()) + return origin is list and args == (float,) + + def is_nullable(field: FieldInfo) -> bool: """Check if a Pydantic FieldInfo is nullable.""" + if _is_embedding_vector_annotation(field): + return True if _unwrap_optional_annotation(field.annotation) is not None: return True if isinstance(field.annotation, (_GenericAlias, GenericAlias)): diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py index 7da2c67a3..c8f699c28 100644 --- a/python/python/lancedb/query.py +++ b/python/python/lancedb/query.py @@ -40,7 +40,7 @@ from .rerankers.base import Reranker from .rerankers.rrf import RRFReranker from .rerankers.util import check_reranker_result from .schema import is_blob_like_field, schema_has_blob_field -from .util import flatten_columns, get_uri_scheme +from .util import _validate_query_vector, flatten_columns, get_uri_scheme from . import _wal_hybrid # WAL-PK-FUSION: delete. from ._blob import ( BLOB_MODE_TO_HANDLING, @@ -961,6 +961,9 @@ class LanceQueryBuilder(ABC): fast_search: bool Skip flat search of unindexed data. """ + if query_type != "fts": + _validate_query_vector(query) + if ordering_field_name is not None: import warnings @@ -1675,6 +1678,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): str_query: Optional[str] = None, fast_search: bool = None, ): + _validate_query_vector(query) super().__init__(table) self._query = query self._distance_type = None @@ -1749,13 +1753,15 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): Parameters ---------- nprobes: int - The number of probes to use. + The number of probes to use. Must be greater than 0. Returns ------- LanceVectorQueryBuilder The LanceQueryBuilder object. """ + if nprobes <= 0: + raise ValueError("Invalid input, nprobes must be greater than 0") self._minimum_nprobes = nprobes self._maximum_nprobes = nprobes return self @@ -2352,7 +2358,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): norm=self._norm, fts_query=self._fts_query._query, reranker=self._reranker, - limit=self._limit, + limit=self._limit or DEFAULT_HYBRID_LIMIT, with_row_ids=True, offset=self._offset, ) @@ -2551,13 +2557,15 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): Parameters ---------- nprobes: int - The number of probes to use. + The number of probes to use. Must be greater than 0. Returns ------- LanceHybridQueryBuilder The LanceHybridQueryBuilder object. """ + if nprobes <= 0: + raise ValueError("Invalid input, nprobes must be greater than 0") self._minimum_nprobes = nprobes self._maximum_nprobes = nprobes return self @@ -2804,13 +2812,13 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): ) # Apply common configurations - if self._limit: - # The final offset/limit window is sliced out of the combined, - # reranked results, so each sub-query must fetch enough rows to - # cover the skipped prefix as well as the window itself. - sub_query_limit = self._limit + (self._offset or 0) - self._vector_query.limit(sub_query_limit) - self._fts_query.limit(sub_query_limit) + # The final offset/limit window is sliced out of the combined, + # reranked results, so each sub-query must fetch enough rows to + # cover the skipped prefix as well as the window itself. + limit = self._limit or DEFAULT_HYBRID_LIMIT + sub_query_limit = limit + (self._offset or 0) + self._vector_query.limit(sub_query_limit) + self._fts_query.limit(sub_query_limit) # WAL-PK-FUSION: without the fallback, select `self._columns` as is. self._pk_fusion = None columns = self._columns @@ -3721,6 +3729,8 @@ class AsyncVectorQueryBase: """ Set the number of partitions to search (probe) + The number of probes must be greater than 0. + This argument is only used when the vector column has an IVF-based index. If there is no index then this value is ignored. diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index 69c5a62b0..52323daef 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -39,7 +39,7 @@ from ..functions import FunctionVersion, UdfDefinition from ..job import AsyncJob, Job from ..sql import Query as SqlQuery from ..sql import QueryDescription -from ..materialized_view import MaterializedView, SelectArg +from ..materialized_view import MaterializedView, MaterializedViewSource, SelectArg from ..secrets import EnvVarSecret, SecretInfo from ..view import ViewDescription @@ -713,7 +713,7 @@ class RemoteDBConnection(DBConnection): def create_materialized_view( self, name: str, - source: str, + source: MaterializedViewSource, *, select: SelectArg = None, where: Optional[str] = None, @@ -745,7 +745,7 @@ class RemoteDBConnection(DBConnection): def create_materialized_view_async( self, name: str, - source: str, + source: MaterializedViewSource, *, select: SelectArg = None, where: Optional[str] = None, @@ -862,25 +862,62 @@ class RemoteDBConnection(DBConnection): definition: UdfDefinition, *, secrets: Optional[Sequence[EnvVarSecret]] = None, + namespace_path: Optional[List[str]] = None, ) -> Job[FunctionVersion]: - job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets)) + job = LOOP.run( + self._conn.create_function_async( + definition, secrets=secrets, namespace_path=namespace_path + ) + ) return Job(job) @override - def get_function(self, name: str, *, version: str) -> FunctionVersion: - return LOOP.run(self._conn.get_function(name, version=version)) + def get_function( + self, + name: str, + *, + version: str, + namespace_path: Optional[List[str]] = None, + ) -> FunctionVersion: + return LOOP.run( + self._conn.get_function( + name, version=version, namespace_path=namespace_path + ) + ) @override - def list_functions(self) -> List[FunctionVersion]: - return LOOP.run(self._conn.list_functions()) + def list_functions( + self, *, namespace_path: Optional[List[str]] = None + ) -> List[FunctionVersion]: + return LOOP.run(self._conn.list_functions(namespace_path=namespace_path)) @override - def drop_function(self, name: str, *, version: str) -> bool: - return LOOP.run(self._conn.drop_function(name, version=version)) + def drop_function( + self, + name: str, + *, + version: str, + namespace_path: Optional[List[str]] = None, + ) -> bool: + return LOOP.run( + self._conn.drop_function( + name, version=version, namespace_path=namespace_path + ) + ) @override - def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]: - dropped, job = LOOP.run(self._conn.drop_function_async(name, version=version)) + def drop_function_async( + self, + name: str, + *, + version: str, + namespace_path: Optional[List[str]] = None, + ) -> Tuple[bool, Job]: + dropped, job = LOOP.run( + self._conn.drop_function_async( + name, version=version, namespace_path=namespace_path + ) + ) return dropped, Job(job) @override @@ -1017,6 +1054,10 @@ class RemoteDBConnection(DBConnection): """ return LOOP.run(self._conn.namespace_client()) - async def close(self): - """Close the connection to the database.""" + @override + def is_open(self) -> bool: + return self._conn.is_open() + + @override + def close(self) -> None: self._conn.close() diff --git a/python/python/lancedb/rerankers/cross_encoder.py b/python/python/lancedb/rerankers/cross_encoder.py index 30ad9d382..7b16a6704 100644 --- a/python/python/lancedb/rerankers/cross_encoder.py +++ b/python/python/lancedb/rerankers/cross_encoder.py @@ -31,6 +31,12 @@ class CrossEncoderReranker(Reranker): trust_remote_code : bool, default True If True, will trust the remote code to be safe. If False, will not trust the remote code and will not run it + batch_size : int, default 32 + The number of query-passage pairs scored per forward pass. + **kwargs + Additional keyword arguments passed to + ``sentence_transformers.CrossEncoder``, for example ``max_length`` or + ``model_kwargs={"torch_dtype": "bfloat16"}``. """ def __init__( @@ -40,6 +46,8 @@ class CrossEncoderReranker(Reranker): device: Union[str, None] = None, return_score="relevance", trust_remote_code: bool = True, + batch_size: int = 32, + **kwargs, ): super().__init__(return_score) torch = attempt_import_or_raise("torch") @@ -47,6 +55,8 @@ class CrossEncoderReranker(Reranker): self.column = column self.device = device self.trust_remote_code = trust_remote_code + self.batch_size = batch_size + self.kwargs = kwargs if self.device is None: self.device = "cuda" if torch.cuda.is_available() else "cpu" @@ -61,6 +71,7 @@ class CrossEncoderReranker(Reranker): self.model_name, device=self.device, trust_remote_code=self.trust_remote_code, + **self.kwargs, ) return cross_encoder @@ -71,7 +82,7 @@ class CrossEncoderReranker(Reranker): return result_set passages = result_set[self.column].to_pylist() cross_inp = [[query, passage] for passage in passages] - cross_scores = self.model.predict(cross_inp) + cross_scores = self.model.predict(cross_inp, batch_size=self.batch_size) result_set = result_set.append_column( "_relevance_score", pa.array(cross_scores, type=pa.float32()) ) diff --git a/python/python/lancedb/rerankers/rrf.py b/python/python/lancedb/rerankers/rrf.py index 6d5b9a2ba..9c835a3ea 100644 --- a/python/python/lancedb/rerankers/rrf.py +++ b/python/python/lancedb/rerankers/rrf.py @@ -107,8 +107,24 @@ class RRFReranker(Reranker): `search().with_row_id(True)`" ) - combined = pa.concat_tables(vector_results, **self._concat_tables_args) - empty_table = pa.Table.from_arrays([], names=[]) - reranked = self.rerank_hybrid(query, combined, empty_table) + # RRF ranks each result list on its own, so a document's rank is its + # position within the list it came from, not within the concatenation. + rrf_score_map = defaultdict(float) + for result in vector_results: + for i, result_id in enumerate(result["_rowid"].to_pylist(), 1): + rrf_score_map[result_id] += 1 / (i + self.K) - return reranked + combined = pa.concat_tables(vector_results, **self._concat_tables_args) + combined = self._deduplicate(combined) + relevance_scores = [ + rrf_score_map[row_id] for row_id in combined["_rowid"].to_pylist() + ] + combined = combined.append_column( + "_relevance_score", pa.array(relevance_scores, type=pa.float32()) + ) + combined = combined.sort_by([("_relevance_score", "descending")]) + + if self.score == "relevance": + combined = self._keep_relevance_score(combined) + + return combined diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 0dc927b29..9d951f559 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -134,6 +134,34 @@ _MODEL_BACKED_TOKENIZER_ERRORS = ( ) +def _optimize_cleanup_since_ms( + cleanup_older_than: Optional[timedelta], retrain: bool +) -> Optional[int]: + # Called directly by both the sync and async optimize so stacklevel=3 + # names the user's call site rather than the background event loop. + cleanup_since_ms: Optional[int] = None + if cleanup_older_than is not None: + cleanup_since_ms = round(cleanup_older_than.total_seconds() * 1000) + if cleanup_since_ms <= 0: + warnings.warn( + "optimize(cleanup_older_than=0) removes every version except " + "the latest. Any concurrent reader or writer still using an " + "older version will fail. Use a longer cleanup_older_than " + "unless no other process is working on this table.", + UserWarning, + stacklevel=3, + ) + + if retrain: + warnings.warn( + "The 'retrain' parameter is deprecated and will be removed in a " + "future version.", + DeprecationWarning, + stacklevel=3, + ) + return cleanup_since_ms + + def _add_unique_note(exception: BaseException, note: str) -> None: existing_notes = getattr(exception, "__notes__", ()) or () message = ( @@ -233,6 +261,101 @@ IndexConfigType = Union[ KNOWN_METRICS = {"l2", "cosine", "dot", "hamming"} +def _blob_value_to_storage(value: Any) -> Optional[dict]: + """Keep a Python blob's inline data or external URI before Arrow infers it.""" + if value is None: + return None + if isinstance(value, (bytes, bytearray, memoryview)): + return {"data": bytes(value)} + if isinstance(value, str): + if not value: + raise ValueError("Blob uri cannot be empty") + return {"uri": value} + if isinstance(value, dict): + unknown = value.keys() - {"data", "uri", "position", "size"} + if unknown: + raise ValueError(f"Unknown blob fields: {sorted(unknown)}") + return value + + try: + from lance.blob import Blob + except ModuleNotFoundError as err: + if err.name not in ("lance", "lance.blob"): + raise + else: + if isinstance(value, Blob): + return { + "data": value.data, + "uri": value.uri, + "position": value.position, + "size": value.size, + } + raise TypeError(f"Unsupported blob value: {type(value).__name__}") + + +def _blob_input_to_arrow(data: Any, schema: Optional[pa.Schema]) -> Optional[pa.Table]: + """Convert Python rows with blob fields before Arrow loses their value types.""" + if schema is None: + return None + + if isinstance(data, list) and data and isinstance(data[0], dict): + names = { + field.name + for field in schema + if is_blob_v2_field(field) + and any(isinstance(row, dict) and field.name in row for row in data) + } + if not names: + return None + values = {name: [row.get(name) for row in data] for name in names} + rows = [{**row, **{name: None for name in names}} for row in data] + table = pa.Table.from_pylist(rows) + elif _check_for_pandas(data) and isinstance(data, pd.DataFrame): + names = { + field.name + for field in schema + if is_blob_v2_field(field) and field.name in data.columns + } + if not names: + return None + values = { + name: [ + None + if value is pd.NA or isinstance(value, float) and np.isnan(value) + else value + for value in data[name] + ] + for name in names + } + table = pa.Table.from_pandas( + data.assign(**{name: None for name in names}), preserve_index=False + ).replace_schema_metadata(None) + else: + return None + + for field in schema: + if field.name not in names: + continue + storage_type = ( + field.type.storage_type + if isinstance(field.type, pa.ExtensionType) + else field.type + ) + storage = pa.array( + [_blob_value_to_storage(value) for value in values[field.name]], + type=storage_type, + ) + column = ( + pa.ExtensionArray.from_storage(field.type, storage) + if isinstance(field.type, pa.ExtensionType) + else storage + ) + table = table.set_column( + table.schema.get_field_index(field.name), field, column + ) + return table + + def _into_pyarrow_reader( data, schema: Optional[pa.Schema] = None ) -> pa.RecordBatchReader: @@ -275,9 +398,14 @@ def _into_pyarrow_reader( return pa.Table.from_batches(data).to_reader() else: data = _serialize_json_values(data, schema) - return pa.Table.from_pylist(data).to_reader() + table = _blob_input_to_arrow(data, schema) + return ( + table if table is not None else pa.Table.from_pylist(data) + ).to_reader() elif _check_for_pandas(data) and isinstance(data, pd.DataFrame): - table = pa.Table.from_pandas(data, preserve_index=False) + table = _blob_input_to_arrow(data, schema) + if table is None: + table = pa.Table.from_pandas(data, preserve_index=False) # Do not serialize Pandas metadata meta = table.schema.metadata if table.schema.metadata is not None else {} meta = {k: v for k, v in meta.items() if k != b"pandas"} @@ -1935,8 +2063,10 @@ class Table(ABC): ``_rowid`` values stay valid after compaction when the table has stable row ids. - Convenience for small payloads. For large values use - :meth:`fetch_blob_files`. + Remote servers limit each request to 1024 row IDs and 64 MiB of blob + bytes. The client splits requests automatically and reads an individual + larger blob through the Range route. This method still materializes all + bytes in memory; for large values use :meth:`fetch_blob_files`. """ @abstractmethod @@ -2217,6 +2347,13 @@ class Table(ABC): All files belonging to versions older than this will be removed. Set to 0 days to remove all versions except the latest. The latest version is never removed. + + .. warning:: + + Setting this to 0 deletes the data files of every older + version, so any other reader or writer still using an older + version of the table will fail. Only set it to 0 if no other + process is working on this dataset. delete_unverified: bool, default False Files leftover from a failed transaction may appear to be part of an in-progress operation (e.g. appending new data) and these files will not @@ -2480,6 +2617,10 @@ class Table(ABC): to nullable. Currently, you cannot change a nullable column to non-nullable. + Each alteration must specify at least one of "rename", "data_type", + or "nullable". The legacy key "name" is also accepted for renaming. + Unknown keys raise ValueError before any alterations are applied. + Returns ------- AlterColumnsResult @@ -4195,9 +4336,9 @@ class LanceTable(Table): ) if storage_options is None: storage_options = {} - storage_options["new_table_enable_v2_manifest_paths"] = ( + storage_options["new_table_enable_v2_manifest_paths"] = str( enable_v2_manifest_paths - ) + ).lower() self._table = LOOP.run( self._conn._conn.create_table( @@ -4423,6 +4564,13 @@ class LanceTable(Table): All files belonging to versions older than this will be removed. Set to 0 days to remove all versions except the latest. The latest version is never removed. + + .. warning:: + + Setting this to 0 deletes the data files of every older + version, so any other reader or writer still using an older + version of the table will fail. Only set it to 0 if no other + process is working on this dataset. delete_unverified: bool, default False Files leftover from a failed transaction may appear to be part of an in-progress operation (e.g. appending new data) and these files will not @@ -4447,10 +4595,9 @@ class LanceTable(Table): modification operations. """ LOOP.run( - self._table.optimize( - cleanup_older_than=cleanup_older_than, - delete_unverified=delete_unverified, - retrain=retrain, + self._table._do_optimize( + _optimize_cleanup_since_ms(cleanup_older_than, retrain), + delete_unverified, ) ) @@ -4623,7 +4770,7 @@ class LanceTable(Table): [LanceTable.uses_v2_manifest_paths][lancedb.table.LanceTable.uses_v2_manifest_paths] to check if the table is already using the new path style. """ - LOOP.run(self._table.migrate_v2_manifest_paths()) + LOOP.run(self._table.migrate_manifest_paths_v2()) @deprecation.deprecated( deprecated_in="0.33.1", @@ -5848,6 +5995,9 @@ class AsyncTable: """ schema = await self.schema() data = _serialize_json_values(data, schema) + blob_table = _blob_input_to_arrow(data, schema) + if blob_table is not None: + data = blob_table if on_bad_vectors is None: on_bad_vectors = "error" if fill_value is None: @@ -5961,7 +6111,17 @@ class AsyncTable: @overload async def search( self, - query: Optional[str] = None, + query: None = None, + vector_column_name: Optional[str] = None, + query_type: QueryType = "auto", + ordering_field_name: Optional[str] = None, + fts_columns: Optional[Union[str, List[str]]] = None, + ) -> AsyncQuery: ... + + @overload + async def search( + self, + query: str, vector_column_name: Optional[str] = None, query_type: Literal["auto"] = ..., ordering_field_name: Optional[str] = None, @@ -5971,7 +6131,7 @@ class AsyncTable: @overload async def search( self, - query: Optional[str] = None, + query: str, vector_column_name: Optional[str] = None, query_type: Literal["hybrid"] = ..., ordering_field_name: Optional[str] = None, @@ -5981,7 +6141,7 @@ class AsyncTable: @overload async def search( self, - query: Optional[Union[VEC, "PIL.Image.Image", Tuple]] = None, + query: Union[VEC, "PIL.Image.Image", Tuple], vector_column_name: Optional[str] = None, query_type: Literal["auto"] = ..., ordering_field_name: Optional[str] = None, @@ -5991,7 +6151,7 @@ class AsyncTable: @overload async def search( self, - query: Optional[str] = None, + query: str, vector_column_name: Optional[str] = None, query_type: Literal["fts"] = ..., ordering_field_name: Optional[str] = None, @@ -6001,9 +6161,7 @@ class AsyncTable: @overload async def search( self, - query: Optional[ - Union[VEC, str, "PIL.Image.Image", Tuple, FullTextQuery] - ] = None, + query: Union[VEC, str, "PIL.Image.Image", Tuple, FullTextQuery], vector_column_name: Optional[str] = None, query_type: Literal["vector"] = ..., ordering_field_name: Optional[str] = None, @@ -6019,7 +6177,7 @@ class AsyncTable: query_type: QueryType = "auto", ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, - ) -> Union[AsyncHybridQuery, AsyncFTSQuery, AsyncVectorQuery]: + ) -> Union[AsyncQuery, AsyncHybridQuery, AsyncFTSQuery, AsyncVectorQuery]: """Create a search query to find the nearest neighbors of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/) and [full-text search](https://lancedb.com/docs/search/full-text-search/). @@ -6034,8 +6192,9 @@ class AsyncTable: - *default None*. Acceptable types are: list, np.ndarray, PIL.Image.Image - - If None then the select/where/limit clauses are applied to filter - the table + - If None then a plain [AsyncQuery][lancedb.query.AsyncQuery] is + returned, equivalent to calling [query][lancedb.table.AsyncTable.query]. + The select/where/limit clauses are applied to filter the table. vector_column_name: str, optional The name of the vector column to search. @@ -6063,10 +6222,13 @@ class AsyncTable: Returns ------- - LanceQueryBuilder + AsyncQuery, AsyncHybridQuery, AsyncFTSQuery, or AsyncVectorQuery A query builder object representing the query. """ + if query is None: + return self.query() + def is_embedding(query): return isinstance(query, (list, np.ndarray, pa.Array, pa.ChunkedArray)) @@ -6716,6 +6878,10 @@ class AsyncTable: to nullable. Currently, you cannot change a nullable column to non-nullable. + Each alteration must specify at least one of "rename", "data_type", + or "nullable". The legacy key "name" is also accepted for renaming. + Unknown keys raise ValueError before any alterations are applied. + Returns ------- AlterColumnsResult @@ -6958,6 +7124,13 @@ class AsyncTable: All files belonging to versions older than this will be removed. Set to 0 days to remove all versions except the latest. The latest version is never removed. + + .. warning:: + + Setting this to 0 deletes the data files of every older + version, so any other reader or writer still using an older + version of the table will fail. Only set it to 0 if no other + process is working on this dataset. delete_unverified: bool, default False Files leftover from a failed transaction may appear to be part of an in-progress operation (e.g. appending new data) and these files will not @@ -6981,20 +7154,14 @@ class AsyncTable: you have added or modified 100,000 or more records or run more than 20 data modification operations. """ - cleanup_since_ms: Optional[int] = None - if cleanup_older_than is not None: - cleanup_since_ms = round(cleanup_older_than.total_seconds() * 1000) - - if retrain: - import warnings - - warnings.warn( - "The 'retrain' parameter is deprecated and will be removed in a " - "future version.", - DeprecationWarning, - stacklevel=2, - ) + return await self._do_optimize( + _optimize_cleanup_since_ms(cleanup_older_than, retrain), + delete_unverified, + ) + async def _do_optimize( + self, cleanup_since_ms: Optional[int], delete_unverified: bool + ) -> OptimizeStats: return await self._inner.optimize( cleanup_since_ms=cleanup_since_ms, delete_unverified=delete_unverified, diff --git a/python/python/lancedb/util.py b/python/python/lancedb/util.py index dbc52bff6..3a6a8a102 100644 --- a/python/python/lancedb/util.py +++ b/python/python/lancedb/util.py @@ -5,6 +5,7 @@ import binascii import functools import importlib +import math import os import pathlib import warnings @@ -279,6 +280,14 @@ def infer_vector_column_dim(data_type: pa.DataType) -> Optional[int]: return None +def _validate_query_vector(query: Any) -> None: + """Reject empty vector inputs before vector-column inference or execution.""" + if (isinstance(query, list) and not query) or ( + isinstance(query, np.ndarray) and query.size == 0 + ): + raise ValueError("Query vector must not be empty") + + def _query_vector_dim(query: Optional[Any]) -> Optional[int]: if query is None: return None @@ -300,6 +309,9 @@ def infer_vector_column_name( query: Optional[Any], # inferred later in query builder vector_column_name: Optional[str], ): + if query_type != "fts": + _validate_query_vector(query) + if vector_column_name is not None: return vector_column_name @@ -351,6 +363,13 @@ def _(value: int): @value_to_sql.register(float) def _(value: float): + # str() gives "nan" / "inf", which SQL would read as column names. + if math.isnan(value): + return "CAST('NaN' AS DOUBLE)" + if math.isinf(value): + return ( + "CAST('Infinity' AS DOUBLE)" if value > 0 else "CAST('-Infinity' AS DOUBLE)" + ) return str(value) diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index d1abdde21..919149f3e 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -7,10 +7,11 @@ import sys import textwrap import lance +import pandas as pd import pyarrow as pa import pyarrow.compute as pc import pytest -from lance.blob import BlobType as LanceBlobType +from lance.blob import Blob, BlobType as LanceBlobType import lancedb from lancedb._blob import ( @@ -1532,3 +1533,129 @@ def test_add_external_uri_string_round_trips_with_flag(tmp_path): hits = table.search().to_arrow() blobs = table.fetch_blobs("image", hits) assert blobs[0].as_py() == payload + + +@pytest.mark.parametrize("as_pandas", [False, True]) +def test_add_bytes_and_uri_in_one_batch_preserves_external_uri(tmp_path, as_pandas): + payload = b"external-payload" + blob_path = tmp_path / "payload.bin" + blob_path.write_bytes(payload) + rows = [ + {"id": 1, "image": b"inline"}, + {"id": 2, "image": blob_path.as_uri()}, + ] + db = lancedb.connect(tmp_path / "db") + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + table = db.create_table("mixed", schema=schema) + + table.add( + pd.DataFrame(rows) if as_pandas else rows, + allow_external_blob_outside_bases=True, + ) + + by_id = _row_ids_by_id(table) + assert table.fetch_blobs("image", [by_id[1], by_id[2]]).to_pylist() == [ + b"inline", + payload, + ] + + +@pytest.mark.parametrize("as_pandas", [False, True]) +def test_add_mixed_python_blob_values_preserves_external_uris(tmp_path, as_pandas): + payload = b"external-payload" + blob_path = tmp_path / "payload.bin" + blob_path.write_bytes(payload) + uri = blob_path.as_uri() + rows = [ + {"id": 1, "image": b"inline"}, + {"id": 2, "image": uri}, + {"id": 3, "image": {"data": b"from-dict"}}, + {"id": 4, "image": Blob.from_bytes(b"from-blob")}, + {"id": 5, "image": Blob.from_uri(uri)}, + {"id": 6, "image": {"uri": uri}}, + {"id": 7, "image": None}, + ] + db = lancedb.connect(tmp_path / "db") + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + table = db.create_table("mixed", schema=schema) + + table.add( + pd.DataFrame(rows) if as_pandas else rows, + allow_external_blob_outside_bases=True, + ) + + by_id = _row_ids_by_id(table) + assert table.fetch_blobs("image", [by_id[i] for i in range(1, 8)]).to_pylist() == [ + b"inline", + payload, + b"from-dict", + b"from-blob", + payload, + payload, + None, + ] + + +@pytest.mark.parametrize("as_pandas", [False, True]) +def test_merge_insert_mixed_python_blobs_validates_external_uri(tmp_path, as_pandas): + blob_path = tmp_path / "payload.bin" + blob_path.write_bytes(b"external-payload") + table = _blob_table("merge_mixed_uri", [{"id": 1, "image": b"before"}]) + rows = [ + {"id": 1, "image": b"updated"}, + {"id": 2, "image": blob_path.as_uri()}, + ] + + with pytest.raises(ValueError, match="outside registered external bases"): + ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute(pd.DataFrame(rows) if as_pandas else rows) + ) + assert table.count_rows() == 1 + + +@pytest.mark.parametrize("as_pandas", [False, True]) +def test_merge_insert_python_blob_dicts_and_objects(as_pandas): + table = _blob_table("merge_python_blobs", [{"id": 1, "image": b"before"}]) + rows = [ + {"id": 1, "image": b"updated"}, + {"id": 2, "image": {"data": b"from-dict"}}, + {"id": 3, "image": Blob.from_bytes(b"from-blob")}, + ] + + result = ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute(pd.DataFrame(rows) if as_pandas else rows) + ) + + assert result.num_updated_rows == 1 + assert result.num_inserted_rows == 2 + by_id = _row_ids_by_id(table) + assert table.fetch_blobs("image", [by_id[i] for i in range(1, 4)]).to_pylist() == [ + b"updated", + b"from-dict", + b"from-blob", + ] + + +def test_create_table_with_python_blob_dict_and_object(): + db = lancedb.connect("memory:///") + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + table = db.create_table( + "create_python_blobs", + data=[ + {"id": 1, "image": {"data": b"from-dict"}}, + {"id": 2, "image": Blob.from_bytes(b"from-blob")}, + ], + schema=schema, + ) + + by_id = _row_ids_by_id(table) + assert table.fetch_blobs("image", [by_id[1], by_id[2]]).to_pylist() == [ + b"from-dict", + b"from-blob", + ] diff --git a/python/python/tests/test_db.py b/python/python/tests/test_db.py index aeb9feeb8..ebd86bdc3 100644 --- a/python/python/tests/test_db.py +++ b/python/python/tests/test_db.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright The LanceDB Authors +import contextlib import inspect import re import sys @@ -350,6 +351,40 @@ async def test_create_table_from_iterator_async(mem_db_async: lancedb.AsyncConne assert await table.count_rows() == 10 +def test_create_exist_ok_mode(tmp_db: lancedb.DBConnection): + db = tmp_db + data = [{"vector": [1.1, 1.2]}, {"vector": [0.2, 1.8]}] + table = db.create_table("t1", data, mode="exist_ok") + assert table.count_rows() == 2 + original = table.to_arrow() + + reopened = db.create_table("t1", [{"vector": [9.0, 8.0]}], mode="exist_ok") + assert reopened.to_arrow().equals(original) + + +def test_create_empty_exist_ok_mode(tmp_db: lancedb.DBConnection): + schema = pa.schema([pa.field("id", pa.int64())]) + table = tmp_db.create_table("empty", schema=schema, mode="exist_ok") + assert table.schema == schema + assert table.count_rows() == 0 + table.add([{"id": 7}]) + + reopened = tmp_db.create_table("empty", schema=schema, mode="exist_ok") + assert reopened.to_arrow().to_pylist() == [{"id": 7}] + + bad_schema = pa.schema([pa.field("other", pa.int64())]) + with pytest.raises(ValueError): + tmp_db.create_table("empty", schema=bad_schema, mode="exist_ok") + + +def test_create_table_invalid_mode(tmp_db: lancedb.DBConnection): + db = tmp_db + data = [{"vector": [1.1, 1.2]}, {"vector": [0.2, 1.8]}] + with pytest.raises(ValueError, match="Invalid mode bogus"): + db.create_table("tb", data, mode="bogus") + assert "tb" not in db + + def test_create_exist_ok(tmp_db: lancedb.DBConnection): from conftest import pandas_string_type @@ -405,6 +440,49 @@ async def test_connect(tmp_path): assert str(db) == f"ListingDatabase(uri={tmp_path}, read_consistency_interval=5s)" +@pytest.fixture(params=["memory", "local", "namespace", "from_inner"]) +def sync_db(request, tmp_path): + if request.param == "memory": + return lancedb.connect("memory://") + if request.param == "namespace": + return lancedb.connect_namespace("dir", {"root": str(tmp_path)}) + db = lancedb.connect(tmp_path) + if request.param == "from_inner": + return lancedb.db.LanceDBConnection.from_inner(db._inner, None) + return db + + +def test_sync_close(sync_db): + assert sync_db.is_open() + assert sync_db.close() is None + assert not sync_db.is_open() + assert sync_db.close() is None + + with pytest.warns(DeprecationWarning, match="table_names"): + with pytest.raises(RuntimeError, match="Connection is closed"): + sync_db.table_names() + with pytest.raises(RuntimeError, match="Connection is closed"): + sync_db.list_tables() + with pytest.raises(RuntimeError, match="Connection is closed"): + sync_db.open_table("test") + + +@pytest.mark.parametrize("raise_error", [False, True]) +def test_sync_context_manager(sync_db, raise_error): + with contextlib.ExitStack() as stack: + if raise_error: + stack.enter_context(pytest.raises(ValueError, match="test error")) + with sync_db as db: + assert db is sync_db + assert db.is_open() + if raise_error: + raise ValueError("test error") + + assert not sync_db.is_open() + with pytest.raises(RuntimeError, match="Connection is closed"): + sync_db.list_tables() + + @pytest.mark.asyncio async def test_close(mem_db_async: lancedb.AsyncConnection): assert mem_db_async.is_open() @@ -495,6 +573,27 @@ async def test_create_exist_ok_async(tmp_db_async: lancedb.AsyncConnection): # await db.create_table("test", schema=bad_schema, exist_ok=True) +@pytest.mark.parametrize("enable_v2_manifest_paths", [False, True]) +@pytest.mark.parametrize("empty", [False, True]) +def test_create_table_deprecated_v2_manifest_paths( + tmp_path, enable_v2_manifest_paths, empty +): + db = lancedb.connect(tmp_path) + data = None if empty else [{"id": 1}] + + with pytest.warns(DeprecationWarning, match="enable_v2_manifest_paths"): + table = db.create_table( + "test", + data=data, + schema=pa.schema([("id", pa.int64())]), + enable_v2_manifest_paths=enable_v2_manifest_paths, + ) + + assert table.uses_v2_manifest_paths() == enable_v2_manifest_paths + assert table.to_arrow().to_pylist() == ([] if empty else data) + assert db.open_table("test").uses_v2_manifest_paths() == enable_v2_manifest_paths + + @pytest.mark.asyncio async def test_create_table_v2_manifest_paths_async(tmp_path): db_with_v2_paths = await lancedb.connect_async( diff --git a/python/python/tests/test_first_class_function_slice1.py b/python/python/tests/test_first_class_function_slice1.py index 1abd42e56..b08dadf36 100644 --- a/python/python/tests/test_first_class_function_slice1.py +++ b/python/python/tests/test_first_class_function_slice1.py @@ -162,6 +162,29 @@ def test_function_version_binds_named_columns_as_one_immutable_application(): ] == [("text", "column", "documents.body")] +def test_function_version_names_its_namespace_as_a_separate_field(): + """A Function is named by its parts, as a Secret is; a root one states no + namespace.""" + value = job_result("remote_function_job.json") + root = FunctionVersion.from_json(json.dumps(value)) + assert root.namespace_path == () + assert "namespace_path" not in json.loads(root.to_canonical_json()) + + nested = FunctionVersion(**{**value, "namespace_path": ["analytics", "features"]}) + assert nested.name == "embed" + assert nested.namespace_path == ("analytics", "features") + assert json.loads(nested.to_canonical_json())["namespace_path"] == [ + "analytics", + "features", + ] + application = nested(text=col("documents.body")) + assert application.function.name == "embed" + assert application.function.namespace_path == ("analytics", "features") + assert json.loads(application.to_canonical_json())["function"][ + "namespace_path" + ] == ["analytics", "features"] + + def test_function_version_binding_validates_names_and_direct_columns(): version = FunctionVersion.from_json( json.dumps(job_result("remote_function_job.json")) diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index 1cd8d4cf5..4610170f6 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -1539,8 +1539,10 @@ def _mock_remote_function_catalog(): else (None, None) ) if function_action[1] == "create": + *namespace_path, name = function_action[0].split("$") state["version"] = { - "name": "normalize_score", + "name": name, + **({"namespace_path": namespace_path} if namespace_path else {}), "version": FUNCTION_VERSION, "object_id": "fixture", "location": "memory:///fixture", @@ -1565,10 +1567,18 @@ def _mock_remote_function_catalog(): "job_state": "DONE", "result": state["version"], } - elif self.path == "/v1/function/normalize_score/describe": + # A namespaced Function's id is its namespace path and name joined + # by `$`, so the name is the last segment of it. + elif ( + function_action[1] == "describe" + and function_action[0].split("$")[-1] == "normalize_score" + ): assert body == {"version": FUNCTION_VERSION} response = state["version"] - elif self.path == "/v1/function/normalize_score/drop": + elif ( + function_action[1] == "drop" + and function_action[0].split("$")[-1] == "normalize_score" + ): assert body == {"version": FUNCTION_VERSION} response = {"dropped": True} elif secret_action[1] in ("create", "alter"): @@ -1602,7 +1612,8 @@ def _mock_remote_function_catalog(): assert query["page_token"] == "next" self._write_response(200, {"secrets": [{"name": "hf-prod"}]}) return - if url.path != "/v1/namespace/$/function/list": + parts = url.path.strip("/").split("/") + if parts[:2] != ["v1", "namespace"] or parts[3:] != ["function", "list"]: self._write_response(404, {"error": "not found"}) return assert query["include_definition"] == "true" @@ -1840,3 +1851,102 @@ async def test_async_remote_drop_function_sends_exact_version(): {"version": FUNCTION_VERSION}, ) ] + + +def test_remote_function_verbs_address_a_namespace_in_the_path(): + """A Function's namespace is part of the identifier each route addresses; + no body carries it, so a namespaced request differs only in its path.""" + namespace = ["analytics", "features"] + with _mock_remote_function_catalog() as (host, state): + db = lancedb.connect( + "db://dev", + api_key="fake", + host_override=host, + client_config={"retry_config": {"retries": 0}}, + ) + created = db.create_function(normalize_score, namespace_path=namespace) + # The returned parts are what the other verbs take, so a created + # Function reopens from its own fields. + assert created.name == "normalize_score" + assert created.namespace_path == tuple(namespace) + reopened = db.get_function( + created.name, + version=created.version, + namespace_path=list(created.namespace_path), + ) + assert reopened == created + assert reopened(value=lancedb.col("score")).function.namespace_path == tuple( + namespace + ) + db.list_functions(namespace_path=namespace) + assert ( + db.drop_function( + "normalize_score", version=FUNCTION_VERSION, namespace_path=namespace + ) + is True + ) + dropped, _ = db.drop_function_async( + "normalize_score", version=FUNCTION_VERSION, namespace_path=namespace + ) + assert dropped is True + + assert [path for path, _ in state["requests"]] == [ + "/v1/function/analytics$features$normalize_score/create", + "/v1/jobs/describe", + "/v1/function/analytics$features$normalize_score/describe", + "/v1/namespace/analytics$features/function/list", + "/v1/namespace/analytics$features/function/list", + "/v1/function/analytics$features$normalize_score/drop", + "/v1/function/analytics$features$normalize_score/drop", + ] + + +@pytest.mark.asyncio +async def test_async_remote_function_verbs_address_a_namespace_in_the_path(): + namespace = ["analytics", "features"] + with _mock_remote_function_catalog() as (host, state): + db = await lancedb.connect_async( + "db://dev", + api_key="fake", + host_override=host, + client_config={"retry_config": {"retries": 0}}, + ) + registration = await db.create_function_async( + normalize_score, namespace_path=namespace + ) + await registration.wait() + await db.get_function( + "normalize_score", version=FUNCTION_VERSION, namespace_path=namespace + ) + await db.list_functions(namespace_path=namespace) + assert ( + await db.drop_function( + "normalize_score", version=FUNCTION_VERSION, namespace_path=namespace + ) + is True + ) + + assert [path for path, _ in state["requests"]] == [ + "/v1/function/analytics$features$normalize_score/create", + "/v1/jobs/describe", + "/v1/function/analytics$features$normalize_score/describe", + "/v1/namespace/analytics$features/function/list", + "/v1/namespace/analytics$features/function/list", + "/v1/function/analytics$features$normalize_score/drop", + ] + + +def test_a_function_namespace_path_must_be_a_list_of_segments(): + """A bare string is refused rather than read as a sequence of one-character + segments, and nothing reaches the service.""" + with _mock_remote_function_catalog() as (host, state): + db = lancedb.connect( + "db://dev", + api_key="fake", + host_override=host, + client_config={"retry_config": {"retries": 0}}, + ) + with pytest.raises(TypeError): + db.list_functions(namespace_path="analytics") + + assert state["requests"] == [] diff --git a/python/python/tests/test_hybrid_query.py b/python/python/tests/test_hybrid_query.py index 61b8001cf..c5f58308d 100644 --- a/python/python/tests/test_hybrid_query.py +++ b/python/python/tests/test_hybrid_query.py @@ -315,6 +315,33 @@ def test_hybrid_query_offset(sync_table: Table): assert offset_result["_rowid"].to_pylist() == full["_rowid"].to_pylist()[2:] +def test_hybrid_query_default_limit(sync_table: Table): + # The vector and FTS legs match disjoint rows, so fusing their results + # yields more rows than the default limit. + new_rows = [] + for i in range(20): + new_rows.append({"text": "close_vec", "vector": [0.1, 0.1]}) + new_rows.append({"text": "dog", "vector": [50.0 + i, 50.0 + i]}) + sync_table.add(new_rows) + + def query(): + return ( + sync_table.search(query_type="hybrid") + .vector([0.1, 0.1]) + .text("dog") + .with_row_id(True) + ) + + # Like the async hybrid query, the default limit is 10. + result = query().to_arrow() + assert len(result) == 10 + + # The offset window is taken after the default limit is applied. + full = query().limit(15).to_arrow() + offset_result = query().offset(5).to_arrow() + assert offset_result["_rowid"].to_pylist() == full["_rowid"].to_pylist()[5:] + + def test_hybrid_query_minimum_nprobes_zero_raises(sync_table: Table): # minimum_nprobes(0) must raise the same validation error a plain vector # query raises, not silently no-op because 0 is falsy. diff --git a/python/python/tests/test_lsm_write_spec.py b/python/python/tests/test_lsm_write_spec.py index d43cb7532..ac767ea31 100644 --- a/python/python/tests/test_lsm_write_spec.py +++ b/python/python/tests/test_lsm_write_spec.py @@ -52,9 +52,9 @@ def test_set_lsm_write_spec_validates(tmp_path): with pytest.raises(Exception, match="num_buckets"): table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 1025)) - # Happy path then mutation rejected. + # Happy path, then a set over the installed spec is refused. table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 4)) - with pytest.raises(Exception, match="mutation"): + with pytest.raises(Exception, match="already set"): table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 8)) @@ -176,18 +176,18 @@ def test_get_lsm_write_spec(tmp_path): table.unset_lsm_write_spec() assert table.get_lsm_write_spec() is None - # Identity round-trips (column recovered from the schema). Leaving the - # maintained set to be inferred picks up the index on the table, so the - # spec reads back naming it rather than as "infer". + # Identity round-trips (column recovered from the schema). An unnamed + # maintained set reads back as None: what is stored is the intent to + # maintain every index, not the set it resolves to now. table.set_lsm_write_spec(LsmWriteSpec.identity("id")) spec = table.get_lsm_write_spec() assert spec.spec_type == "identity" assert spec.column == "id" - assert spec.maintained_indexes == [idx_name] + assert spec.maintained_indexes is None table.unset_lsm_write_spec() # Unsharded round-trips (no routing column). Opting out is distinct from - # the inferred default. + # the unnamed default. table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([])) spec = table.get_lsm_write_spec() assert spec.spec_type == "unsharded" diff --git a/python/python/tests/test_materialized_views.py b/python/python/tests/test_materialized_views.py index c030ac1df..1ee21fbac 100644 --- a/python/python/tests/test_materialized_views.py +++ b/python/python/tests/test_materialized_views.py @@ -520,10 +520,166 @@ def test_stored_queries_and_legacy_layouts_are_read(): "SELECT * FROM people" ) + for fmt, function in [(3, "vector_duplicate_pairs"), (4, "vector_dedup")]: + native = f"SELECT * FROM {function}('images', 3, 'phash', 4)" + assert read({"format": fmt, "query": native}).query == native + # A newer writer's layout is reported, never guessed at. for newer in ( - {"format": 3, "query": query}, + {"format": 5, "query": query}, {"kind": "select_v3", "source_table": "people"}, ): with pytest.raises(NotImplementedError, match="cannot refresh"): read(newer) + + +def indexed_dedup_source(db, name="images"): + import pyarrow as pa + from lancedb.index import IvfFlat + + vectors = pa.FixedSizeListArray.from_arrays( + pa.array( + [v for i in range(256) for v in (float(i if i < 3 else i * 10), 0.0)], + type=pa.float32(), + ), + 2, + ) + table = db.create_table(name, pa.table({"id": range(256), "vector": vectors})) + table.create_index("vector", config=IvfFlat(num_partitions=1)) + return table + + +@pytest.mark.parametrize("stable", [False, True]) +def test_vector_dedup_source_captures_snapshot_and_preserves_source(tmp_path, stable): + db = lancedb.connect( + tmp_path, + storage_options={"new_table_enable_stable_row_ids": str(stable).lower()}, + ) + source = indexed_dedup_source(db, "images.with.dot") + version = source.version + expression = lancedb.vector_dedup( + source.name, column="vector", distance_threshold=1 + ) + view = db.create_materialized_view("clean", expression) + assert "vector_dedup(" in view.definition.query + assert f", {version}, 'vector', 1.0)" in view.definition.query + assert view.table.count_rows() == 255 + assert [r["id"] for r in view.table.search().where("id < 3").to_list()] == [0, 2] + assert source.version == version + assert source.count_rows() == 256 + + source.delete("id = 0") + view.refresh_async().wait() + assert view.table.search().where("id = 0").to_list() == [ + {"id": 0, "vector": [0.0, 0.0]} + ] + assert expression.dataset_version is None # reusable immutable declaration + + +def test_vector_dedup_create_job_and_explicit_snapshot(tmp_path): + db = lancedb.connect(tmp_path) + source = indexed_dedup_source(db) + version = source.version + source.delete("id = 0") + job = db.create_materialized_view_async( + "clean", + lancedb.vector_dedup( + "images", column="vector", distance_threshold=1, dataset_version=version + ), + ) + assert job.wait() is None + view = db.open_materialized_view("clean") + assert view.table.count_rows() == 255 + assert len(view.table.search().where("id = 0").to_list()) == 1 + + +@pytest.mark.asyncio +async def test_async_vector_dedup_declaration_and_creation(tmp_path): + sync_db = lancedb.connect(tmp_path) + source = indexed_dedup_source(sync_db) + db = await lancedb.connect_async(tmp_path) + expression = lancedb.vector_dedup("images", column="vector", distance_threshold=1) + view = await db.create_materialized_view("clean", expression, with_no_data=True) + assert await view.table.count_rows() == 0 + # The snapshot is captured at declaration even if refresh comes later. + source.delete("id = 0") + job = await view.refresh_async() + await job.wait() + assert await view.table.count_rows() == 255 + another = await db.create_materialized_view_async( + "later", expression, with_no_data=True + ) + await another.wait() + later = await db.open_materialized_view("later") + assert (await later.definition()).query != (await view.definition()).query + assert await later.table.count_rows() == 0 + + +@pytest.mark.parametrize( + "options", [{"select": ["id"]}, {"select": []}, {"where": "id > 0"}, {"limit": 1}] +) +def test_vector_dedup_rejects_extra_clauses_before_opening_source(tmp_path, options): + db = lancedb.connect(tmp_path) + expression = lancedb.vector_dedup("missing", column="vector", distance_threshold=1) + with pytest.raises(ValueError, match="cannot be combined"): + db.create_materialized_view("bad", expression, **options) + assert db.list_tables().tables == [] + + +@pytest.mark.parametrize("version", [0, -1, True, 1.5, "2", 2**64]) +def test_vector_dedup_rejects_invalid_snapshot(version): + with pytest.raises(ValueError, match="dataset_version"): + lancedb.vector_dedup( + "images", column="vector", distance_threshold=1, dataset_version=version + ) + + +@pytest.mark.parametrize( + "threshold", [float("nan"), float("inf"), -float("inf"), 1e100, 10**1000, True, "4"] +) +def test_vector_dedup_rejects_invalid_threshold(threshold): + with pytest.raises((ValueError, TypeError), match="distance_threshold"): + lancedb.vector_dedup("images", column="vector", distance_threshold=threshold) + + +def test_vector_dedup_descriptor_is_frozen(): + from dataclasses import FrozenInstanceError + + expression = lancedb.vector_dedup("images", column="vector", distance_threshold=1) + with pytest.raises(FrozenInstanceError): + expression.column = "other" + + +@pytest.mark.parametrize("with_no_data", [False, True]) +def test_remote_vector_dedup_uses_mv_endpoint_without_sql_client(with_no_data): + with mock_remote_materialized_view_create() as (host, requests): + db = lancedb.connect( + "db://dev", + api_key="fake", + host_override=host, + sql_host_override="grpc://127.0.0.1:1", + client_config={"retry_config": {"retries": 0}}, + ) + job = db.create_materialized_view_async( + "clean", + lancedb.vector_dedup( + "images.with'dot", + column="vector'field", + distance_threshold=4, + dataset_version=7, + ), + with_no_data=with_no_data, + ) + assert job.id == "mv-create-123" + assert requests == [ + ( + "/v1/materialized_view/clean/create", + { + "query": ( + "SELECT * FROM vector_dedup('`images.with''dot`', " + "7, 'vector''field', 4.0)" + ), + "with_no_data": with_no_data, + }, + ) + ] diff --git a/python/python/tests/test_pydantic.py b/python/python/tests/test_pydantic.py index ca03940f2..81674d735 100644 --- a/python/python/tests/test_pydantic.py +++ b/python/python/tests/test_pydantic.py @@ -9,6 +9,7 @@ from typing import List, Optional, Tuple import pyarrow as pa import pydantic import pytest +from lancedb.conftest import MockTextEmbeddingFunction from lancedb.pydantic import ( LanceModel, MultiVector, @@ -425,6 +426,25 @@ def test_bare_vector_raises_clear_error(): exec("class TestModel(LanceModel):\n vector: Vector", namespace) +def test_embedding_vector_list_annotation(): + embedding = MockTextEmbeddingFunction.create() + + class StaticTypingModel(LanceModel): + text: str = embedding.SourceField() + vector: list[float] = embedding.VectorField() + + schema = pydantic_to_schema(StaticTypingModel) + assert schema == pa.schema( + [ + pa.field("text", pa.utf8(), False), + pa.field("vector", pa.list_(pa.float32(), embedding.ndims()), True), + ] + ) + + model = StaticTypingModel(text="hello", vector=[0.0] * embedding.ndims()) + assert model.vector == [0.0] * embedding.ndims() + + def test_fixed_size_list_field(): class TestModel(pydantic.BaseModel): vec: Vector(16) diff --git a/python/python/tests/test_query.py b/python/python/tests/test_query.py index c2d9760ae..75e163fa0 100644 --- a/python/python/tests/test_query.py +++ b/python/python/tests/test_query.py @@ -24,6 +24,7 @@ from lancedb.pydantic import LanceModel, Vector from lancedb.query import ( AsyncFTSQuery, AsyncHybridQuery, + AsyncQuery, AsyncQueryBase, AsyncVectorQuery, ColumnOrdering, @@ -1153,6 +1154,19 @@ def test_query_builder_with_filter(table): assert all(np.array(rs[0]["vector"]) == [3, 4]) +@pytest.mark.parametrize("query_type", ["vector", "hybrid"]) +@pytest.mark.parametrize("nprobes", [0, -1]) +def test_nprobes_nonpositive_sync(table, query_type, nprobes): + if query_type == "hybrid": + query = table.search(query_type="hybrid").vector([0, 0]).text("a") + else: + query = table.search([0, 0]) + with pytest.raises( + ValueError, match="^Invalid input, nprobes must be greater than 0$" + ): + query.nprobes(nprobes) + + def test_invalid_nprobes_sync(table): with pytest.raises(ValueError, match="minimum_nprobes must be greater than 0"): LanceVectorQueryBuilder(table, [0, 0], "vector").minimum_nprobes(0).to_list() @@ -1184,6 +1198,18 @@ def test_multiple_nprobes_calls_works_sync(table): ).minimum_nprobes(20).to_list() +@pytest.mark.asyncio +@pytest.mark.parametrize("hybrid", [False, True]) +async def test_nprobes_zero_async(table_async: AsyncTable, hybrid): + query = table_async.query().nearest_to([0, 0]) + if hybrid: + query = query.nearest_to_text("dog") + with pytest.raises( + ValueError, match="^Invalid input, nprobes must be greater than 0$" + ): + query.nprobes(0) + + @pytest.mark.asyncio async def test_invalid_nprobes_async(table_async: AsyncTable): with pytest.raises(ValueError, match="minimum_nprobes must be greater than 0"): @@ -1458,6 +1484,27 @@ async def test_query_to_polars_async(table_async: AsyncTable): assert df.shape == (0, num_columns) +@pytest.mark.asyncio +@pytest.mark.parametrize("search_kwargs", [{}, {"query": None}]) +@pytest.mark.parametrize("with_vector", [False, True]) +async def test_async_search_without_query(mem_db_async, search_kwargs, with_vector): + data = [{"id": i} for i in range(10)] + if with_vector: + for row in data: + row["vector"] = [float(row["id"]), 1.0] + table = await mem_db_async.create_table("test", data) + + query = await table.search(**search_kwargs) + assert isinstance(query, AsyncQuery) + assert await query.limit(3).to_arrow() == await table.query().limit(3).to_arrow() + + query = await table.search(**search_kwargs) + assert await query.where("id >= 8").select(["id"]).limit(3).to_list() == [ + {"id": 8}, + {"id": 9}, + ] + + @pytest.mark.asyncio async def test_none_query(table_async: AsyncTable): with pytest.raises(ValueError): @@ -2270,6 +2317,12 @@ def test_ensure_vector_query_nested_empty_list(): ensure_vector_query([[]]) +@pytest.mark.parametrize("query", [[], np.array([], dtype=np.float32)]) +def test_vector_query_builder_empty_vector(table, query): + with pytest.raises(ValueError, match="^Query vector must not be empty$"): + LanceVectorQueryBuilder(table, query, "vector") + + def test_fast_search(tmp_path): db = lancedb.connect(tmp_path) diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 1f2522fe8..573aee956 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -16,8 +16,9 @@ import uuid from packaging.version import Version import lancedb +import numpy as np from lancedb.conftest import MockTextEmbeddingFunction -from lancedb.query import ColumnOrdering +from lancedb.query import AsyncQuery, ColumnOrdering from lancedb.remote import ClientConfig from lancedb.remote.errors import HttpError, RetryError import pytest @@ -119,6 +120,77 @@ async def test_async_remote_db(): assert table_names == [] +@pytest.mark.parametrize( + "alteration, match", + [ + ({"path": "id"}, "One of rename, nullable or data_type"), + ( + {"path": "id", "nulable": False}, # spellchecker:disable-line + "Unknown column alteration key 'nulable'", # spellchecker:disable-line + ), + ( + { + "path": "id", + "rename": "new_id", + "nulable": False, # spellchecker:disable-line + }, + "Unknown column alteration key 'nulable'", # spellchecker:disable-line + ), + ], +) +def test_remote_alter_columns_rejects_invalid_before_request(alteration, match): + requests = [] + + def handler(request): + requests.append(request.path) + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(b'{"version": 1, "schema": {"fields": []}}') + + with mock_lancedb_connection(handler) as db: + table = db.open_table("test") + requests.clear() + + with pytest.raises(ValueError, match=match): + table.alter_columns(alteration) + + assert requests == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "alteration, match", + [ + ({"path": "id"}, "One of rename, nullable or data_type"), + ( + {"path": "id", "nulable": False}, # spellchecker:disable-line + "Unknown column alteration key 'nulable'", # spellchecker:disable-line + ), + ], +) +async def test_async_remote_alter_columns_rejects_invalid_before_request( + alteration, match +): + requests = [] + + def handler(request): + requests.append(request.path) + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(b'{"version": 1, "schema": {"fields": []}}') + + async with mock_lancedb_connection_async(handler) as db: + table = await db.open_table("test") + requests.clear() + + with pytest.raises(ValueError, match=match): + await table.alter_columns(alteration) + + assert requests == [] + + @pytest.mark.asyncio async def test_async_checkout(): def handler(request): @@ -1521,6 +1593,60 @@ def test_query_sync_empty_query(): assert data == expected +@pytest.mark.parametrize("query", [[], np.array([], dtype=np.float32)]) +@pytest.mark.parametrize("vector_column_name", [None, "vector"]) +@pytest.mark.parametrize("query_type", ["auto", "vector"]) +def test_query_sync_empty_vector(query, vector_column_name, query_type): + def handler(body): + pytest.fail("An empty query vector must be rejected before sending a query") + + with query_test_table(handler) as table: + with pytest.raises(ValueError, match="^Query vector must not be empty$"): + table.search( + query, vector_column_name=vector_column_name, query_type=query_type + ).limit(3).to_arrow() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("search_kwargs", [{}, {"query": None}]) +async def test_async_search_without_query(search_kwargs): + expected = pa.table({"id": [7, 8, 9]}) + + def handler(request): + if request.path == "/v1/table/test/describe/": + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(b'{"version": 1, "schema": {"fields": []}}') + elif request.path == "/v1/table/test/query/": + body = json.loads( + request.rfile.read(int(request.headers["Content-Length"])) + ) + assert body == { + "k": 3, + "filter": "id >= 7", + "vector": [], + "columns": ["id"], + "prefilter": True, + "version": None, + } + request.send_response(200) + request.send_header("Content-Type", "application/vnd.apache.arrow.file") + request.end_headers() + with pa.ipc.new_file(request.wfile, schema=expected.schema) as writer: + writer.write_table(expected) + else: + request.send_response(404) + request.end_headers() + + async with mock_lancedb_connection_async(handler) as db: + table = await db.open_table("test") + query = await table.search(**search_kwargs) + assert isinstance(query, AsyncQuery) + result = await query.where("id >= 7").select(["id"]).limit(3).to_arrow() + assert result == expected + + def test_query_sync_maximal(): def handler(body): assert body == { @@ -1581,6 +1707,53 @@ def test_query_sync_maximal(): ) +@pytest.mark.parametrize("hybrid", [False, True]) +def test_query_sync_nprobes_zero(hybrid): + query_requests = [] + + def handler(body): + query_requests.append(body) + return pa.table({"id": []}) + + with query_test_table(handler) as table: + if hybrid: + query = table.search(query_type="hybrid").vector([1, 2, 3]).text("dog") + else: + query = table.search([1, 2, 3]) + with pytest.raises( + ValueError, match="^Invalid input, nprobes must be greater than 0$" + ): + query.nprobes(0) + + assert query_requests == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("hybrid", [False, True]) +async def test_query_async_nprobes_zero(hybrid): + requests = [] + + def handler(request): + requests.append(request.path) + if request.path == "/v1/table/test/describe/": + send_json(request, {"version": 1, "schema": {"fields": []}}) + else: + request.send_response(404) + request.end_headers() + + async with mock_lancedb_connection_async(handler) as db: + table = await db.open_table("test") + query = table.query().nearest_to([1, 2, 3]) + if hybrid: + query = query.nearest_to_text("dog") + with pytest.raises( + ValueError, match="^Invalid input, nprobes must be greater than 0$" + ): + query.nprobes(0) + + assert requests == ["/v1/table/test/describe/"] + + def test_query_sync_nprobes(): def handler(body): assert body == { @@ -2145,15 +2318,46 @@ async def test_header_provider_overrides_static_headers(): def test_close(): - """Test that close() works without AttributeError.""" - import asyncio - def handler(req): req.send_response(200) req.end_headers() with mock_lancedb_connection(handler) as db: - asyncio.run(db.close()) + assert db.close() is None + assert not db.is_open() + assert db.close() is None + + with pytest.warns(DeprecationWarning, match="table_names"): + with pytest.raises(RuntimeError, match="Connection is closed"): + db.table_names() + with pytest.raises(RuntimeError, match="Connection is closed"): + db.list_tables() + with pytest.raises(RuntimeError, match="Connection is closed"): + db.open_table("test") + + +@pytest.mark.parametrize("raise_error", [False, True]) +def test_sync_context_manager(raise_error): + def handler(req): + req.send_response(200) + req.send_header("Content-Type", "application/json") + req.end_headers() + req.wfile.write(b'{"tables": []}') + + with mock_lancedb_connection(handler) as db: + with contextlib.ExitStack() as stack: + if raise_error: + stack.enter_context(pytest.raises(ValueError, match="test error")) + with db as conn: + assert conn is db + assert conn.is_open() + assert conn.list_tables().tables == [] + if raise_error: + raise ValueError("test error") + + assert not db.is_open() + with pytest.raises(RuntimeError, match="Connection is closed"): + db.list_tables() @pytest.mark.parametrize("exception", [KeyboardInterrupt, SystemExit, GeneratorExit]) @@ -2391,6 +2595,27 @@ def blob_query_response_table(): ) +def blob_descriptor_take_table(): + """What `fetch_blob_files` reads for rows 10, 20, 30, and 40.""" + queried = blob_query_response_table() + image_field = queried.schema.field("image") + images = pa.StructArray.from_arrays( + [ + pa.array([0, 0, 0, 0], type=pa.uint8()), + pa.array([0, 0, 0, 0], type=pa.uint64()), + pa.array([5, 0, 5, 0], type=pa.uint64()), + pa.array([0, 0, 0, 0], type=pa.uint32()), + pa.array(["", "", "", ""], type=pa.string()), + ], + fields=image_field.type, + mask=pa.array([False, True, False, False]), + ) + return pa.Table.from_arrays( + [images, pa.array([10, 20, 30, 40], type=pa.uint64())], + schema=pa.schema([image_field, pa.field("_rowid", pa.uint64())]), + ) + + @contextlib.contextmanager def blob_remote_table(*, server_version=Version("0.5.0")): def handler(request): @@ -2403,11 +2628,16 @@ def blob_remote_table(*, server_version=Version("0.5.0")): elif request.path.startswith("/v1/table/test/blob/image/"): path = request.path.partition("?")[0] row_id = int(path.split("/")[-2]) - payload = {10: b"alpha", 20: None, 30: b"gamma"}[row_id] + payload = {10: b"alpha", 20: None, 30: b"gamma", 40: b""}[row_id] if payload is None: request.send_response(204) request.end_headers() return + if not payload: + request.send_response(416) + request.send_header("Content-Range", "bytes */0") + request.end_headers() + return byte_range = request.headers["Range"].removeprefix("bytes=") start_text, end_text = byte_range.split("-", maxsplit=1) start = int(start_text) @@ -2416,14 +2646,22 @@ def blob_remote_table(*, server_version=Version("0.5.0")): request.send_response(206) request.send_header("Content-Range", f"bytes {start}-{end}/{len(payload)}") request.send_header("Content-Length", str(len(chunk))) + request.send_header( + "x-lancedb-version", str(BLOB_DESCRIBE_RESPONSE["version"]) + ) request.end_headers() request.wfile.write(chunk) elif request.path == "/v1/table/test/query/": content_len = int(request.headers.get("Content-Length", 0)) body = json.loads(request.rfile.read(content_len)) - assert body["columns"] == ["id", "image"] assert body["with_row_id"] is True - response_table = blob_query_response_table() + if body["columns"] == ["image"]: + # fetch_blob_files sizes its handles from a descriptor take. + assert body["filter"].startswith("_rowid IN") + response_table = blob_descriptor_take_table() + else: + assert body["columns"] == ["id", "image"] + response_table = blob_query_response_table() request.send_response(200) request.send_header("Content-Type", "application/vnd.apache.arrow.file") request.end_headers() @@ -2459,17 +2697,21 @@ def test_remote_blob_columns_and_fetch(): def test_remote_blob_files_are_lazy_seekable_handles(): with blob_remote_table() as table: - files = table.fetch_blob_files("image", [10, 20, 30]) + files = table.fetch_blob_files("image", [10, 20, 30, 40]) - assert len(files) == 3 - alpha, null_row, gamma = files + assert len(files) == 4 + alpha, null_row, gamma, empty = files assert null_row is None assert alpha is not None assert gamma is not None + assert empty is not None assert alpha.size() == 5 assert alpha.read_range(1, 3) == b"lph" gamma.seek(2) assert gamma.read() == b"mma" + assert empty.size() == 0 + assert empty.read() == b"" + assert empty.read_range(0, 0) == b"" alpha.close() assert alpha.closed with pytest.raises(RuntimeError, match="already closed"): diff --git a/python/python/tests/test_rerankers.py b/python/python/tests/test_rerankers.py index 33c0db488..850b477a0 100644 --- a/python/python/tests/test_rerankers.py +++ b/python/python/tests/test_rerankers.py @@ -395,6 +395,38 @@ def test_mrr_multivector_rewards_consensus(): assert result["_rowid"].to_pylist()[0] == 2 +def test_rrf_multivector_ranks_each_result_list(): + # RRF scores a document by its rank *within each* result list. The top hit + # of the second list must score the same as the top hit of the first list, + # not as if it were ranked after every row of the first list. + reranker = RRFReranker(K=60) + + def ranking(row_ids): + return pa.table({"_rowid": pa.array(row_ids, type=pa.int64())}) + + rs1 = ranking([1, 2, 3]) + rs2 = ranking([4, 5, 6]) + + result = reranker.rerank_multivector([rs1, rs2]) + scores = dict( + zip(result["_rowid"].to_pylist(), result["_relevance_score"].to_pylist()) + ) + + assert scores[1] == pytest.approx(1 / 61) + assert scores[4] == pytest.approx(1 / 61) + assert scores[5] == pytest.approx(1 / 62) + # the second-ranked hit of the second list beats the third hit of the first + assert scores[5] > scores[3] + + # A document found by both lists sums its reciprocal ranks from each list. + result = reranker.rerank_multivector([ranking([1, 2]), ranking([2, 3])]) + scores = dict( + zip(result["_rowid"].to_pylist(), result["_relevance_score"].to_pylist()) + ) + assert scores[2] == pytest.approx(1 / 62 + 1 / 61) + assert result["_rowid"].to_pylist() == [2, 1, 3] + + def test_rrf_reranker_distance(): data = pa.table( { @@ -481,6 +513,14 @@ def test_cross_encoder_reranker(tmp_path): _run_test_reranker(reranker, table, "single player experience", None, schema) +def test_cross_encoder_reranker_kwargs(tmp_path): + pytest.importorskip("sentence_transformers") + reranker = CrossEncoderReranker(batch_size=4, max_length=64) + assert reranker.model.max_length == 64 + table, schema = get_test_table(tmp_path) + _run_test_reranker(reranker, table, "single player experience", None, schema) + + def test_colbert_reranker(tmp_path): pytest.importorskip("rerankers") reranker = ColbertReranker() diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 5c56fea38..e5ee3c3a2 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -113,6 +113,32 @@ def test_basic(mem_db: DBConnection): assert table.to_arrow() == expected_data +@pytest.mark.parametrize("enable_v2", [False, True]) +def test_migrate_v2_manifest_paths(tmp_path, enable_v2): + db = lancedb.connect( + tmp_path, + storage_options={"new_table_enable_v2_manifest_paths": str(enable_v2).lower()}, + ) + table = db.create_table("calls", [{"id": 1, "vector": [1.0, 1.0]}]) + table.add([{"id": 2, "vector": [2.0, 2.0]}]) + expected_data = table.to_arrow() + expected_versions = table.list_versions() + assert table.uses_v2_manifest_paths() == enable_v2 + + # Migration is also safe to repeat on a table already using v2 paths. + for _ in range(2): + table.migrate_v2_manifest_paths() + assert table.uses_v2_manifest_paths() + reopened = db.open_table("calls") + assert reopened.uses_v2_manifest_paths() + assert reopened.to_arrow() == expected_data + assert reopened.list_versions() == expected_versions + + manifests = list((tmp_path / "calls.lance" / "_versions").glob("*.manifest")) + assert len(manifests) == len(expected_versions) + assert all(len(path.stem) == 20 and path.stem.isdigit() for path in manifests) + + def test_search_preserves_nulls_from_sliced_arrow_table(mem_db: DBConnection): data = pa.table( { @@ -3751,6 +3777,24 @@ def test_empty_query(mem_db: DBConnection): assert df.num_rows == 42 +@pytest.mark.parametrize("query", [[], np.array([], dtype=np.float32)]) +@pytest.mark.parametrize("vector_column_name", [None, "vector"]) +@pytest.mark.parametrize("query_type", ["auto", "vector"]) +@pytest.mark.parametrize("multiple_vector_columns", [False, True]) +def test_search_empty_vector( + mem_db, query, vector_column_name, query_type, multiple_vector_columns +): + fields = [pa.field("vector", pa.list_(pa.float32(), 8))] + if multiple_vector_columns: + fields.append(pa.field("vec2", pa.list_(pa.float32(), 4))) + table = mem_db.create_table("empty_vector_query", schema=pa.schema(fields)) + + with pytest.raises(ValueError, match="^Query vector must not be empty$"): + table.search( + query, vector_column_name=vector_column_name, query_type=query_type + ).limit(3).to_arrow() + + def test_search_with_schema_inf_single_vector(mem_db: DBConnection): class MyTable(LanceModel): text: str @@ -4144,14 +4188,74 @@ async def test_add_columns_with_schema(mem_db_async: AsyncConnection): ) -def test_alter_columns(mem_db: DBConnection): +@pytest.mark.parametrize("rename_key", ["rename", "name"]) +def test_alter_columns(mem_db: DBConnection, rename_key): data = pa.table({"id": [0, 1]}) table = mem_db.create_table("my_table", data=data) - alter_columns_res = table.alter_columns({"path": "id", "rename": "new_id"}) + alter_columns_res = table.alter_columns({"path": "id", rename_key: "new_id"}) assert alter_columns_res.version == 2 assert table.to_arrow().column_names == ["new_id"] +INVALID_COLUMN_ALTERATIONS = [ + (({"path": "id"},), "One of rename, nullable or data_type"), + ( + ({"path": "id", "nulable": False},), # spellchecker:disable-line + "Unknown column alteration key 'nulable'", # spellchecker:disable-line + ), + ( + ( + { + "path": "id", + "rename": "new_id", + "nulable": False, # spellchecker:disable-line + }, + ), + "Unknown column alteration key 'nulable'", # spellchecker:disable-line + ), + ( + ({"path": "id", "rename": "new_id"}, {"path": "id"}), + "One of rename, nullable or data_type", + ), +] + + +def test_alter_columns_nullable_false(mem_db: DBConnection): + table = mem_db.create_table("my_table", data=pa.table({"id": [0, 1]})) + result = table.alter_columns({"path": "id", "nullable": False}) + assert result.version == 2 + assert not table.schema.field("id").nullable + + +@pytest.mark.parametrize("alterations, match", INVALID_COLUMN_ALTERATIONS) +def test_alter_columns_rejects_invalid(mem_db: DBConnection, alterations, match): + table = mem_db.create_table("my_table", data=pa.table({"id": [0, 1]})) + initial_version = table.version + initial_schema = table.schema + + with pytest.raises(ValueError, match=match): + table.alter_columns(*alterations) + + assert table.version == initial_version + assert table.schema == initial_schema + + +@pytest.mark.asyncio +@pytest.mark.parametrize("alterations, match", INVALID_COLUMN_ALTERATIONS) +async def test_alter_columns_rejects_invalid_async( + mem_db_async: AsyncConnection, alterations, match +): + table = await mem_db_async.create_table("my_table", data=pa.table({"id": [0, 1]})) + initial_version = await table.version() + initial_schema = await table.schema() + + with pytest.raises(ValueError, match=match): + await table.alter_columns(*alterations) + + assert await table.version() == initial_version + assert await table.schema() == initial_schema + + def test_update_field_metadata(mem_db: DBConnection): data = pa.table({"id": [0, 1], "category": ["a", "b"]}) table = mem_db.create_table("my_table", data=data) @@ -4177,10 +4281,11 @@ def test_update_field_metadata(mem_db: DBConnection): @pytest.mark.asyncio -async def test_alter_columns_async(mem_db_async: AsyncConnection): +@pytest.mark.parametrize("rename_key", ["rename", "name"]) +async def test_alter_columns_async(mem_db_async: AsyncConnection, rename_key): data = pa.table({"id": [0, 1]}) table = await mem_db_async.create_table("my_table", data=data) - alter_columns_res = await table.alter_columns({"path": "id", "rename": "new_id"}) + alter_columns_res = await table.alter_columns({"path": "id", rename_key: "new_id"}) assert alter_columns_res.version == 2 assert (await table.to_arrow()).column_names == ["new_id"] alter_columns_res = await table.alter_columns( @@ -4307,7 +4412,8 @@ async def test_optimize(mem_db_async: AsyncConnection): assert stats.prune.bytes_removed == 0 assert stats.prune.old_versions_removed == 0 - stats = await table.optimize(cleanup_older_than=timedelta(seconds=0)) + with pytest.warns(UserWarning, match="concurrent"): + stats = await table.optimize(cleanup_older_than=timedelta(seconds=0)) assert stats.prune.bytes_removed > 0 assert stats.prune.old_versions_removed == 3 @@ -4335,12 +4441,35 @@ async def test_optimize_delete_unverified(tmp_db_async: AsyncConnection, tmp_pat stats = await table.optimize(delete_unverified=False) assert stats.prune.old_versions_removed == 0 - stats = await table.optimize( - cleanup_older_than=timedelta(seconds=0), delete_unverified=True - ) + with pytest.warns(UserWarning, match="concurrent"): + stats = await table.optimize( + cleanup_older_than=timedelta(seconds=0), delete_unverified=True + ) assert stats.prune.old_versions_removed == 2 +@pytest.mark.asyncio +async def test_optimize_warns_on_zero_cleanup(mem_db_async: AsyncConnection): + table = await mem_db_async.create_table("test", data=[{"x": [1]}]) + with pytest.warns(UserWarning, match="concurrent"): + await table.optimize(cleanup_older_than=timedelta(0)) + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await table.optimize(cleanup_older_than=timedelta(days=1)) + await table.optimize() + + +def test_optimize_warns_on_zero_cleanup_sync(mem_db: DBConnection): + table = mem_db.create_table("test", data=[{"x": [1]}]) + with warnings.catch_warnings(record=True) as seen: + warnings.simplefilter("default") + table.optimize(cleanup_older_than=timedelta(0)) + table.optimize(cleanup_older_than=timedelta(0)) + assert [w.filename for w in seen] == [__file__, __file__] + assert all("concurrent" in str(w.message) for w in seen) + + def test_replace_field_metadata(tmp_path): db = lancedb.connect(tmp_path) table = db.create_table("my_table", data=[{"x": 0}]) diff --git a/python/python/tests/test_util.py b/python/python/tests/test_util.py index 29cfd7f0b..d9c574eb9 100644 --- a/python/python/tests/test_util.py +++ b/python/python/tests/test_util.py @@ -213,6 +213,35 @@ def test_value_to_sql_numpy_scalars(): assert value_to_sql(np.bool_(False)) == "FALSE" +def test_value_to_sql_nan_and_infinity(tmp_path): + # str(float("nan")) is "nan", which SQL reads as a column named nan, so + # table.update(values={"x": float("nan")}) failed with "No field named nan". + import math + + import numpy as np + + db = lancedb.connect(tmp_path) + table = db.create_table( + "test", + pa.table( + { + "id": [1, 2, 3, 4], + "f64": [1.0, 2.0, 3.0, 4.0], + "f32": pa.array([1.0, 2.0, 3.0, 4.0], pa.float32()), + } + ), + ) + table.update(where="id = 1", values={"f64": float("nan"), "f32": np.nan}) + table.update(where="id = 2", values={"f64": math.inf, "f32": np.float32("inf")}) + table.update(where="id = 3", values={"f64": -math.inf, "f32": -math.inf}) + + rows = {row["id"]: row for row in table.to_arrow().to_pylist()} + assert math.isnan(rows[1]["f64"]) and math.isnan(rows[1]["f32"]) + assert rows[2]["f64"] == math.inf and rows[2]["f32"] == math.inf + assert rows[3]["f64"] == -math.inf and rows[3]["f32"] == -math.inf + assert rows[4]["f64"] == 4.0 and rows[4]["f32"] == 4.0 + + def test_append_vector_columns(): registry = EmbeddingFunctionRegistry.get_instance() registry.register("test")(MockTextEmbeddingFunction) diff --git a/python/src/connection.rs b/python/src/connection.rs index 2959fe037..744ab34b3 100644 --- a/python/src/connection.rs +++ b/python/src/connection.rs @@ -407,7 +407,8 @@ impl Connection { }) } - #[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))] + #[allow(clippy::too_many_arguments)] + #[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false, *, vector_source_json=None))] pub fn create_materialized_view( self_: PyRef<'_, Self>, name: String, @@ -416,10 +417,17 @@ impl Connection { filter: Option, limit: Option, with_no_data: bool, + vector_source_json: Option, ) -> PyResult> { let inner = self_.get_inner()?.clone(); future_into_py(self_.py(), async move { let mut builder = inner.create_materialized_view(name, source); + if let Some(source) = vector_source_json { + builder = builder.vector_source( + serde_json::from_str(&source) + .map_err(|e| PyValueError::new_err(e.to_string()))?, + ); + } if let Some(projections) = projections { builder = builder.select(projections); } @@ -435,7 +443,8 @@ impl Connection { }) } - #[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))] + #[allow(clippy::too_many_arguments)] + #[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false, *, vector_source_json=None))] pub fn create_materialized_view_async( self_: PyRef<'_, Self>, name: String, @@ -444,10 +453,17 @@ impl Connection { filter: Option, limit: Option, with_no_data: bool, + vector_source_json: Option, ) -> PyResult> { let inner = self_.get_inner()?.clone(); future_into_py(self_.py(), async move { let mut builder = inner.create_materialized_view(name, source); + if let Some(source) = vector_source_json { + builder = builder.vector_source( + serde_json::from_str(&source) + .map_err(|e| PyValueError::new_err(e.to_string()))?, + ); + } if let Some(projections) = projections { builder = builder.select(projections); } @@ -738,31 +754,37 @@ impl Connection { }) } + #[pyo3(signature = (request_json, namespace_path=None))] pub fn create_function_async( self_: PyRef<'_, Self>, request_json: String, + namespace_path: Option>, ) -> PyResult> { let inner = self_.get_inner()?.clone(); let request = lancedb::function::FunctionRegistrationRequest::from_json(&request_json) .infer_error()?; + let namespace_path = namespace_path.unwrap_or_default(); future_into_py(self_.py(), async move { inner - .create_function_async(request) + .create_function_async(request, &namespace_path) .await .infer_error() .map(crate::job::Job::new_typed) }) } + #[pyo3(signature = (name, version, namespace_path=None))] pub fn get_function( self_: PyRef<'_, Self>, name: String, version: String, + namespace_path: Option>, ) -> PyResult> { let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); future_into_py(self_.py(), async move { inner - .get_function(name, version) + .get_function(name, version, &namespace_path) .await .infer_error()? .to_canonical_json() @@ -770,11 +792,16 @@ impl Connection { }) } - pub fn list_functions(self_: PyRef<'_, Self>) -> PyResult> { + #[pyo3(signature = (namespace_path=None))] + pub fn list_functions( + self_: PyRef<'_, Self>, + namespace_path: Option>, + ) -> PyResult> { let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); future_into_py(self_.py(), async move { inner - .list_functions() + .list_functions(&namespace_path) .await .infer_error()? .into_iter() @@ -783,26 +810,35 @@ impl Connection { }) } + #[pyo3(signature = (name, version, namespace_path=None))] pub fn drop_function( self_: PyRef<'_, Self>, name: String, version: String, + namespace_path: Option>, ) -> PyResult> { let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); future_into_py(self_.py(), async move { - inner.drop_function(name, version).await.infer_error() + inner + .drop_function(name, version, &namespace_path) + .await + .infer_error() }) } + #[pyo3(signature = (name, version, namespace_path=None))] pub fn drop_function_async( self_: PyRef<'_, Self>, name: String, version: String, + namespace_path: Option>, ) -> PyResult> { let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); future_into_py(self_.py(), async move { inner - .drop_function_async(name, version) + .drop_function_async(name, version, &namespace_path) .await .infer_error() .map(|(dropped, job)| (dropped, crate::job::Job::new(job))) diff --git a/python/src/query.rs b/python/src/query.rs index 2398376d5..1079fb314 100644 --- a/python/src/query.rs +++ b/python/src/query.rs @@ -180,6 +180,9 @@ impl<'py> IntoPyObject<'py> for PyLanceDB { .expect("Failed to import namespace"); match self.0 { + FtsQuery::CombinedFields(_) => Err(PyValueError::new_err( + "Combined-fields FTS queries are not yet supported in Python", + )), FtsQuery::Match(query) => { let kwargs = PyDict::new(py); kwargs.set_item("boost", query.boost)?; @@ -992,8 +995,9 @@ impl VectorQuery { self.inner = self.inner.clone().refine_factor(refine_factor); } - pub fn nprobes(&mut self, nprobe: u32) { - self.inner = self.inner.clone().nprobes(nprobe as usize); + pub fn nprobes(&mut self, nprobe: u32) -> PyResult<()> { + self.inner = self.inner.clone().nprobes(nprobe as usize).infer_error()?; + Ok(()) } pub fn minimum_nprobes(&mut self, minimum_nprobes: u32) -> PyResult<()> { @@ -1185,8 +1189,8 @@ impl HybridQuery { self.inner_vec.refine_factor(refine_factor); } - pub fn nprobes(&mut self, nprobe: u32) { - self.inner_vec.nprobes(nprobe); + pub fn nprobes(&mut self, nprobe: u32) -> PyResult<()> { + self.inner_vec.nprobes(nprobe) } pub fn ef(&mut self, ef: u32) { diff --git a/python/src/table.rs b/python/src/table.rs index e7b6f1c6f..d7519bdab 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -1897,6 +1897,17 @@ impl Table { let alterations = alterations .iter() .map(|alteration| { + for key in alteration.keys().iter() { + let key = key.extract::()?; + if !matches!( + key.as_str(), + "path" | "rename" | "name" | "nullable" | "data_type" + ) { + return Err(PyValueError::new_err(format!( + "Unknown column alteration key '{key}'" + ))); + } + } let path = alteration .get_item("path")? .ok_or_else(|| PyValueError::new_err("Missing path"))? diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index fbc9ab5ee..cdb0609d3 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.40.0-beta.11" +version = "0.40.0-beta.12" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true diff --git a/rust/lancedb/examples/ivf_pq.rs b/rust/lancedb/examples/ivf_pq.rs index 79f674615..b457558ba 100644 --- a/rust/lancedb/examples/ivf_pq.rs +++ b/rust/lancedb/examples/ivf_pq.rs @@ -142,7 +142,7 @@ async fn search_index(table: &Table) -> Result<()> { // Override the default of 10 to get more rows .limit(15) // Override the default of 20 to search more partitions - .nprobes(30) + .nprobes(30)? // Override the default of None to apply a refine step .refine_factor(1) .execute() diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index 0e5bbf5da..1d0a4a0ab 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -568,6 +568,7 @@ impl Connection { /// Creates a new table by cloning from an existing source table. /// By default, this performs a shallow clone where the new table shares /// the underlying data files with the source table. + /// The target table name must be unused. An existing table is left unchanged. /// /// # Parameters /// - `target_table_name`: The name of the new table to create @@ -587,7 +588,8 @@ impl Connection { ) } - /// Build and register a Python callable as an immutable Function version. + /// Build and register a Python callable as an immutable Function version + /// in `namespace_path`, which is empty for the root namespace. /// /// The server-side job builds the OCI image, then registers the completed /// artifact. Waiting on the returned typed job yields the durable @@ -597,8 +599,11 @@ impl Connection { pub async fn create_function_async( &self, request: crate::function::FunctionRegistrationRequest, + namespace_path: &[String], ) -> Result> { - self.internal.create_function_async(request).await + self.internal + .create_function_async(request, namespace_path) + .await } /// Look up one exact immutable Function version in the remote catalog. @@ -610,13 +615,15 @@ impl Connection { &self, name: impl AsRef, version: impl AsRef, + namespace_path: &[String], ) -> Result { self.internal - .get_function(name.as_ref(), version.as_ref()) + .get_function(name.as_ref(), version.as_ref(), namespace_path) .await } - /// List every published immutable Function version in the remote catalog. + /// List every published immutable Function version in `namespace_path` of + /// the remote catalog. Functions in child namespaces are not included. /// /// Results are ordered by Function name then version. The client walks all /// server pages before returning. Local databases return @@ -628,14 +635,17 @@ impl Connection { /// # async fn list_functions( /// # connection: &lancedb::Connection, /// # ) -> Result<(), Box> { - /// for function in connection.list_functions().await? { + /// for function in connection.list_functions(&[]).await? { /// println!("{} {}", function.name(), function.version()); /// } /// # Ok(()) /// # } /// ``` - pub async fn list_functions(&self) -> Result> { - self.internal.list_functions().await + pub async fn list_functions( + &self, + namespace_path: &[String], + ) -> Result> { + self.internal.list_functions(namespace_path).await } /// Remove the current Function name binding, retaining the object history. @@ -647,9 +657,10 @@ impl Connection { &self, name: impl AsRef, version: impl AsRef, + namespace_path: &[String], ) -> Result { self.internal - .drop_function(name.as_ref(), version.as_ref()) + .drop_function(name.as_ref(), version.as_ref(), namespace_path) .await } @@ -663,9 +674,10 @@ impl Connection { &self, name: impl AsRef, version: impl AsRef, + namespace_path: &[String], ) -> Result<(bool, crate::job::Job)> { self.internal - .drop_function_async(name.as_ref(), version.as_ref()) + .drop_function_async(name.as_ref(), version.as_ref(), namespace_path) .await } @@ -2182,6 +2194,63 @@ mod tests { assert_eq!(tables.len(), 0); } + #[rstest::rstest] + #[tokio::test] + async fn test_clone_table_target_already_exists( + #[values("source", "target")] target: &str, + #[values(false, true)] enable_v2_manifest_paths: bool, + ) { + use crate::query::ExecutableQuery; + use futures::TryStreamExt; + + let tmp_dir = tempdir().unwrap(); + let options = ListingDatabaseOptions::builder() + .enable_v2_manifest_paths(enable_v2_manifest_paths) + .build(); + let db = connect(tmp_dir.path().to_str().unwrap()) + .database_options(&options) + .execute() + .await + .unwrap(); + let source_data = arrow_array::record_batch!(("id", Int32, [0, 1, 2])).unwrap(); + let target_data = arrow_array::record_batch!(("id", Int32, [10, 11])).unwrap(); + db.create_table("source", source_data.clone()) + .execute() + .await + .unwrap(); + db.create_table("target", target_data.clone()) + .execute() + .await + .unwrap(); + + let source_uri = tmp_dir.path().join("source.lance"); + let err = db + .clone_table(target, source_uri.to_str().unwrap()) + .execute() + .await + .unwrap_err(); + assert!( + matches!(&err, Error::TableAlreadyExists { name } if name == target), + "unexpected clone error: {err:?}" + ); + + // Reopen both tables to verify that the failed clone left their data and versions intact. + for (name, expected_data) in [("source", source_data), ("target", target_data)] { + let table = db.open_table(name).execute().await.unwrap(); + assert_eq!(table.version().await.unwrap(), 1); + assert_eq!(table.list_versions().await.unwrap().len(), 1); + let batches = table + .query() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(batches, vec![expected_data]); + } + } + #[tokio::test] async fn test_clone_table() { let tmp_dir = tempdir().unwrap(); diff --git a/rust/lancedb/src/database.rs b/rust/lancedb/src/database.rs index 224fd8770..2518edf13 100644 --- a/rust/lancedb/src/database.rs +++ b/rust/lancedb/src/database.rs @@ -333,10 +333,12 @@ pub trait Database: /// /// See [`CloneTableRequest`] for detailed documentation and examples. async fn clone_table(&self, request: CloneTableRequest) -> Result>; - /// Submit a Function creation job that builds an image and registers it. + /// Submit a Function creation job that builds an image and registers it + /// in `namespace_path`. async fn create_function_async( &self, _request: crate::function::FunctionRegistrationRequest, + _namespace_path: &[String], ) -> Result> { function_catalog_not_supported() } @@ -412,15 +414,25 @@ pub trait Database: &self, _name: &str, _version: &str, + _namespace_path: &[String], ) -> Result { function_catalog_not_supported() } - /// List every published immutable Function version in the remote catalog. - async fn list_functions(&self) -> Result> { + /// List every published immutable Function version in one namespace of the + /// remote catalog. + async fn list_functions( + &self, + _namespace_path: &[String], + ) -> Result> { function_catalog_not_supported() } /// Remove the current Function name binding, retaining the object history. - async fn drop_function(&self, _name: &str, _version: &str) -> Result { + async fn drop_function( + &self, + _name: &str, + _version: &str, + _namespace_path: &[String], + ) -> Result { function_catalog_not_supported() } /// Start dropping a Function and return a handle to the cleanup job. @@ -431,8 +443,9 @@ pub trait Database: &self, name: &str, version: &str, + namespace_path: &[String], ) -> Result<(bool, crate::job::Job)> { - let dropped = self.drop_function(name, version).await?; + let dropped = self.drop_function(name, version, namespace_path).await?; Ok((dropped, crate::job::Job::new_done())) } /// Create a named Secret in this database. Fails if the name is taken, so diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index dc1627116..8255fef4b 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -1107,6 +1107,21 @@ impl Database for ListingDatabase { ..Default::default() }; + let target_uri = self.table_uri(&request.target_table_name)?; + match DatasetBuilder::from_uri(&target_uri) + .with_read_params(read_params.clone()) + .load() + .await + { + Ok(_) => { + return Err(Error::TableAlreadyExists { + name: request.target_table_name, + }); + } + Err(lance::Error::DatasetNotFound { .. }) => {} + Err(err) => return Err(err.into()), + } + let mut source_dataset = DatasetBuilder::from_uri(&request.source_uri) .with_read_params(read_params.clone()) .load() @@ -1122,7 +1137,6 @@ impl Database for ListingDatabase { }), }?; - let target_uri = self.table_uri(&request.target_table_name)?; source_dataset .shallow_clone(&target_uri, version_ref, Some(storage_params)) .await diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs index 3b7209bd5..f05633281 100644 --- a/rust/lancedb/src/function.rs +++ b/rust/lancedb/src/function.rs @@ -85,22 +85,25 @@ fn has_unknown_keys(value: &Value, allowed: &[&str]) -> bool { .is_some_and(|object| object.keys().any(|key| !allowed.contains(&key.as_str()))) } +/// Every field a [`FunctionVersionRef`] document may carry. An application or +/// binding with any other key comes from a newer contract. +pub(crate) const FUNCTION_VERSION_REF_FIELDS: &[&str] = &[ + "name", + "namespace_path", + "object_id", + "location", + "version", + "manifest_digest", +]; + fn application_has_unknown_nested_fields(value: &Value) -> bool { let Some(application) = value.as_object() else { return false; }; - if application.get("function").is_some_and(|value| { - has_unknown_keys( - value, - &[ - "name", - "object_id", - "location", - "version", - "manifest_digest", - ], - ) - }) { + if application + .get("function") + .is_some_and(|value| has_unknown_keys(value, FUNCTION_VERSION_REF_FIELDS)) + { return true; } if application @@ -431,9 +434,17 @@ fn deserialize_object_version<'de, D: Deserializer<'de>>( } /// One immutable Function object revision and its executable artifact. +/// +/// The Function is named by its parts, `name` and `namespace_path`, the way a +/// [`SecretReference`](crate::secrets::SecretReference) names a Secret. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct FunctionVersion { name: String, + /// The namespace holding the Function. Empty is the root, and is absent + /// from the wire, as a [`SecretReference`](crate::secrets::SecretReference) + /// omits it. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + namespace_path: Vec, object_id: String, location: String, #[serde(deserialize_with = "deserialize_object_version")] @@ -463,6 +474,7 @@ impl FunctionVersion { pub fn reference(&self) -> FunctionVersionRef { FunctionVersionRef { name: self.name.clone(), + namespace_path: self.namespace_path.clone(), object_id: self.object_id.clone(), location: self.location.clone(), version: self.version.clone(), @@ -472,6 +484,9 @@ impl FunctionVersion { pub fn name(&self) -> &str { &self.name } + pub fn namespace_path(&self) -> &[String] { + &self.namespace_path + } pub fn version(&self) -> &str { &self.version } @@ -556,6 +571,10 @@ impl_json!(FunctionRegistrationRequest); #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct FunctionVersionRef { pub name: String, + /// The namespace holding the Function; empty is the root and is absent + /// from the wire. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub namespace_path: Vec, pub object_id: String, pub location: String, #[serde(deserialize_with = "deserialize_object_version")] @@ -954,6 +973,46 @@ mod tests { ); } + /// A Function is named by its parts, and a root one states no namespace. + #[test] + fn function_version_carries_its_namespace_as_a_separate_field() { + const VERSION: &str = include_str!( + "../tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json" + ); + let root = FunctionVersion::from_json(VERSION).unwrap(); + assert!(root.namespace_path().is_empty()); + assert_eq!(root.to_canonical_json().unwrap(), VERSION.trim_end()); + + let mut document: Value = serde_json::from_str(VERSION).unwrap(); + document["namespace_path"] = serde_json::json!(["analytics", "features"]); + let nested = FunctionVersion::from_json(&document.to_string()).unwrap(); + assert_eq!(nested.name(), "embed"); + assert_eq!(nested.namespace_path(), ["analytics", "features"]); + let reference = nested.reference(); + assert_eq!(reference.name, "embed"); + assert_eq!(reference.namespace_path, ["analytics", "features"]); + let reference: Value = serde_json::to_value(&reference).unwrap(); + assert_eq!( + reference["namespace_path"], + serde_json::json!(["analytics", "features"]) + ); + let root_reference = serde_json::to_value(root.reference()).unwrap(); + assert!(root_reference.get("namespace_path").is_none()); + + // An application carrying the namespace is not from a newer contract. + let application = serde_json::json!({ + "function": reference, + "inputs": [], + "output": serde_json::to_value(nested.signature().output.clone()).unwrap(), + }); + let application = FunctionApplication::from_json(&application.to_string()).unwrap(); + assert!(!application.has_unknown_fields()); + assert_eq!( + application.function().namespace_path, + ["analytics", "features"] + ); + } + /// Arrays are ordered by the caller, so canonicalization must leave them /// alone -- sorting them would change what a signature means. #[test] diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs index 430750a00..7dec300a2 100644 --- a/rust/lancedb/src/materialized_view.rs +++ b/rust/lancedb/src/materialized_view.rs @@ -9,8 +9,12 @@ //! query this version cannot maintain reads back as unrefreshable, not as a //! plain table. Queries, indexes and search work on the view unchanged. +mod duplicate_pairs; mod grouped; mod grouped_units; +mod partitioned; +pub(crate) mod vector_dedup; +pub use duplicate_pairs::{VectorSource, VectorSourceKind}; pub use grouped::IVF_PARTITION; /// Refreshing a view grouped by [`IVF_PARTITION`] in units, one per index /// partition, so the work can be spread over several processes. The caller @@ -74,6 +78,10 @@ pub use grouped_units::{ GroupedRefreshPlan, WrittenUnit, commit_grouped_refresh, plan_grouped_refresh, write_grouped_unit, }; +pub use partitioned::{ + PartitionedRefreshPlan, WrittenPartition, commit_partitioned_refresh, plan_partitioned_refresh, + write_refresh_partition, write_refresh_partition_with_inputs, +}; mod query; pub mod refresh; @@ -146,18 +154,26 @@ const COLUMN_DEFINITIONS_META_KEY: &str = "lancedb::column_definitions"; /// The newest layout this version reads under [`DEFINITION_META_KEY`]: /// `{"format": N, "query": ""}`, the query as -/// [`MaterializedViewDefinition::to_sql`] renders it. Format 2 is a query -/// with `GROUP BY`; every other query is written as format 1, so a reader -/// that predates grouping reports a grouped view as unrefreshable rather -/// than failing to parse it. A reader refuses a newer format rather than +/// [`MaterializedViewDefinition::to_sql`] renders it. Format 3 identifies a +/// native duplicate-pair source, format 4 a dedup result source, format 2 a query with `GROUP BY`, and +/// format 1 an ordinary query. A reader refuses a newer format rather than /// guess at it. The layout also carries `"kind": "query"`, which readers /// older than the format number report as an unrefreshable view instead of /// failing to read the metadata. -pub const DEFINITION_FORMAT: u64 = 2; +pub const DEFINITION_FORMAT: u64 = 4; /// The format `definition` is written in; see [`DEFINITION_FORMAT`]. fn format_of(definition: &MaterializedViewDefinition) -> u64 { - if definition.is_grouped() { 2 } else { 1 } + if let Some(source) = &definition.vector_source { + match source.kind { + VectorSourceKind::Pairs => 3, + VectorSourceKind::Dedup => 4, + } + } else if definition.is_grouped() { + 2 + } else { + 1 + } } /// Legacy `kind` tag of the structured layout written before @@ -300,6 +316,8 @@ pub fn read_staging(metadata: &HashMap) -> Result>, /// Name of the source table, in the same database as the view. pub source_table: String, /// Namespace path holding the source table; empty is the root namespace. @@ -498,6 +516,7 @@ pub fn read_definition(metadata: &HashMap) -> Result, ) -> Result { + if definition.vector_source.is_some() { + return duplicate_pairs::plan(source_schema, definition); + } let filter = definition .filter .as_deref() @@ -712,6 +734,7 @@ pub(crate) fn plan( } let definition = MaterializedViewDefinition { + vector_source: None, source_table: definition.source_table.clone(), source_namespace: definition.source_namespace.clone(), lateral: definition.lateral.clone(), @@ -1145,6 +1168,11 @@ impl PreparedDeclaration { /// read: the column the view projects it to, if any, otherwise an /// internal projection added here, named by [`input_column_name`]. pub fn input_column(&mut self, source_column: &str) -> Result { + if self.definition.vector_source.is_some() { + return Err(Error::InvalidInput { + message: "native vector views have no per-source-row inputs".into(), + }); + } // A grouped view's rows are groups; no source row carries a value into one. if self.definition.is_grouped() { return Err(Error::InvalidInput { @@ -1245,6 +1273,11 @@ impl PreparedDeclaration { columns: Vec<(usize, ArrowField)>, bindings: &[FunctionBinding], ) -> Result { + if self.definition.vector_source.is_some() { + return Err(Error::InvalidInput { + message: "computed columns on native vector views are not supported".into(), + }); + } let invalid = |message: String| Error::InvalidInput { message }; if columns.is_empty() { return Err(invalid("at least one computed column is needed".into())); @@ -1451,6 +1484,7 @@ pub async fn prepare_declaration( limit: Option, ) -> Result { let definition = MaterializedViewDefinition { + vector_source: None, source_table: source.name().to_string(), source_namespace: source.namespace().to_vec(), lateral: None, @@ -1512,6 +1546,27 @@ pub async fn prepare_declaration( /// # Ok(()) /// # } /// ``` +/// +/// An indexed, pinned source can materialize retained original rows using the +/// default greedy direct-representative policy. A row is removed only when it +/// directly matches a retained row; an A-B-C chain keeps A and C. The source is +/// unchanged. Hosts distribute this through [`plan_partitioned_refresh`] and +/// [`write_refresh_partition_with_inputs`], with a barrier between selection +/// tasks and source-fragment materialization tasks. +/// +/// ``` +/// # #![recursion_limit = "256"] +/// use lancedb::materialized_view::{MaterializedViewDefinition, prepare_definition}; +/// # async fn dedup(images: &lancedb::Table) -> Result<(), Box> { +/// let version = images.version().await?; +/// let definition = MaterializedViewDefinition::from_sql(&format!( +/// "SELECT * FROM vector_dedup('images', {version}, 'phash', 4)" +/// ))?; +/// let clean = prepare_definition(images, definition).await?.create("images_clean").await?; +/// clean.refresh().execute().await?; +/// # Ok(()) +/// # } +/// ``` pub async fn prepare_definition( source: &Table, definition: MaterializedViewDefinition, @@ -1627,7 +1682,9 @@ async fn prepare_with( ), }); } - if !native.dataset.get().await?.manifest.uses_stable_row_ids() { + if definition.vector_source.is_none() + && !native.dataset.get().await?.manifest.uses_stable_row_ids() + { return Err(Error::InvalidInput { message: format!( "materialized views require stable row ids on the source table; \ @@ -1653,7 +1710,22 @@ async fn prepare_with( message: format!("view column name '{}' is reserved", reserved.output), }); } - let source_schema = resolved.schema().await?; + let source_schema = if let Some(pairs) = &definition.vector_source { + let ds = native + .dataset + .get() + .await? + .checkout_version(pairs.dataset_version) + .await?; + crate::table::datafusion::udtf::duplicate_pairs::plan_duplicate_pairs( + Arc::new(ds.clone()), + &pairs.config()?, + ) + .await?; + Arc::new(ArrowSchema::from(ds.schema())) + } else { + resolved.schema().await? + }; let source_metadata = source_schema.metadata().clone(); let Planned { definition, @@ -1670,11 +1742,13 @@ async fn prepare_with( None => source_schema.clone(), Some(unnest) => flattened_schema(&source_schema, &unnest)?, }; - fields.push(ArrowField::new( - SOURCE_ROW_ID_COLUMN, - DataType::UInt64, - false, - )); + if definition.vector_source.is_none() { + fields.push(ArrowField::new( + SOURCE_ROW_ID_COLUMN, + DataType::UInt64, + false, + )); + } // Only column-describing metadata comes along: structural declarations // describe how a table is written, and a view is written by refresh alone. let mut metadata: HashMap = HashMap::new(); @@ -1718,6 +1792,7 @@ pub struct CreateMaterializedViewBuilder { namespace: Vec, source: String, source_namespace: Vec, + vector_source: Option, projections: Vec<(String, String)>, filter: Option, limit: Option, @@ -1732,6 +1807,7 @@ impl CreateMaterializedViewBuilder { namespace: Vec::new(), source, source_namespace: Vec::new(), + vector_source: None, projections: Vec::new(), filter: None, limit: None, @@ -1752,6 +1828,38 @@ impl CreateMaterializedViewBuilder { self } + /// Read native pairs or retained original rows from a fixed indexed snapshot. + /// This source selects its full output schema and cannot be combined with + /// projections, filters or limits. Remote creation uses the ordinary MV + /// endpoint and its registry job; it does not execute a SQL query client-side. + pub fn vector_source(mut self, source: VectorSource) -> Self { + self.vector_source = Some(source); + self + } + + fn native_definition(&self) -> Result> { + let Some(source) = &self.vector_source else { + return Ok(None); + }; + source.config()?; + if !self.projections.is_empty() || self.filter.is_some() || self.limit.is_some() { + return Err(Error::InvalidInput { + message: "native vector sources cannot be combined with select, where or limit" + .into(), + }); + } + Ok(Some(MaterializedViewDefinition { + vector_source: Some(Box::new(source.clone())), + source_table: self.source.clone(), + source_namespace: self.source_namespace.clone(), + projections: vec![ViewProjection::star()], + filter: None, + lateral: None, + group_by: Vec::new(), + limit: None, + })) + } + /// The view's columns, as `(name, SQL expression)` pairs. Not calling /// this selects every source column, expanded at creation time. pub fn select( @@ -1784,7 +1892,10 @@ impl CreateMaterializedViewBuilder { self } - fn query(&self) -> String { + fn query(&self) -> Result { + if let Some(definition) = self.native_definition()? { + return Ok(definition.to_sql()); + } fn quote(name: &str) -> String { format!("\"{}\"", name.replace('"', "\"\"")) } @@ -1813,12 +1924,13 @@ impl CreateMaterializedViewBuilder { if let Some(limit) = self.limit { query.push_str(&format!(" LIMIT {limit}")); } - query + Ok(query) } /// Submit creation and initial population, returning a [`Job`] that - /// settles when the view is ready. The source must keep stable row ids -- - /// they hold provenance across compaction, and cannot be enabled later. + /// settles when the view is ready. Ordinary sources must keep stable row + /// ids for provenance across compaction. Native vector sources also support + /// physical row IDs because their definitions pin the dataset version. pub async fn execute_async(self) -> Result { if self.connection.uri().starts_with("db://") { return self @@ -1827,7 +1939,7 @@ impl CreateMaterializedViewBuilder { .create_materialized_view_async(CreateMaterializedViewRequest { name: self.name.clone(), namespace_path: self.namespace.clone(), - query: self.query(), + query: self.query()?, with_no_data: self.with_no_data, }) .await; @@ -1855,19 +1967,24 @@ impl CreateMaterializedViewBuilder { } async fn execute_native(self) -> Result { + let native_definition = self.native_definition()?; let source = self .connection .open_table(&self.source) .namespace(self.source_namespace.clone()) .execute() .await?; - let prepared = prepare_declaration( - &source, - (!self.projections.is_empty()).then_some(self.projections.as_slice()), - self.filter.as_deref(), - self.limit, - ) - .await?; + let prepared = if let Some(definition) = native_definition { + prepare_definition(&source, definition).await? + } else { + prepare_declaration( + &source, + (!self.projections.is_empty()).then_some(self.projections.as_slice()), + self.filter.as_deref(), + self.limit, + ) + .await? + }; let view = prepared.create_in(&self.namespace, &self.name).await?; if !self.with_no_data { view.refresh().execute().await?; @@ -2186,6 +2303,7 @@ mod tests { assert_eq!( view.definition(), &MaterializedViewDefinition { + vector_source: None, source_table: "people".into(), source_namespace: Vec::new(), projections: vec![ @@ -2309,13 +2427,49 @@ mod tests { .select([("double\"age", "age * 2")]) .only_if("age >= 18") .limit(10) - .query(); + .query() + .unwrap(); assert_eq!( query, "SELECT age * 2 AS \"double\"\"age\" FROM \"raw data\".\"odd\"\"source\" WHERE age >= 18 LIMIT 10" ); } + #[tokio::test] + async fn native_vector_builder_preserves_source_and_rejects_extra_clauses() { + let conn = people_db().await; + let source = VectorSource { + kind: VectorSourceKind::Dedup, + dataset_version: 7, + column: "vector'field".into(), + distance_threshold: "4.0".into(), + }; + let builder = || { + conn.create_materialized_view("clean", "odd.source'name") + .source_namespace(vec!["raw space".into()]) + .vector_source(source.clone()) + }; + let query = builder().query().unwrap(); + let definition = MaterializedViewDefinition::from_sql(&query).unwrap(); + assert_eq!(definition.source_table, "odd.source'name"); + assert_eq!(definition.source_namespace, ["raw space"]); + assert_eq!(definition.vector_source.as_deref(), Some(&source)); + assert!(definition.selects_star()); + assert!(builder().select([("id", "id")]).query().is_err()); + assert!(builder().only_if("true").query().is_err()); + assert!(builder().limit(1).query().is_err()); + let invalid_source = VectorSource { + dataset_version: 0, + ..source + }; + assert!( + conn.create_materialized_view("bad", "people") + .vector_source(invalid_source) + .query() + .is_err() + ); + } + /// No projection selects every source column, expanded now: the schema /// captured at creation is the definition. #[tokio::test] @@ -3235,6 +3389,7 @@ mod tests { fn definition(source_namespace: Vec) -> MaterializedViewDefinition { MaterializedViewDefinition { + vector_source: None, source_table: "people".to_string(), source_namespace, lateral: None, @@ -3321,8 +3476,8 @@ mod tests { #[test] fn a_newer_format_is_reported_not_guessed() { assert_eq!( - read(r#"{"format":3,"query":"SELECT name FROM people"}"#).unwrap(), - Some(StoredDefinition::Newer { format: "3".into() }) + read(r#"{"format":5,"query":"SELECT name FROM people"}"#).unwrap(), + Some(StoredDefinition::Newer { format: "5".into() }) ); assert_eq!( read(r#"{"kind":"join"}"#).unwrap(), diff --git a/rust/lancedb/src/materialized_view/duplicate_pairs.rs b/rust/lancedb/src/materialized_view/duplicate_pairs.rs new file mode 100644 index 000000000..d0dc219ad --- /dev/null +++ b/rust/lancedb/src/materialized_view/duplicate_pairs.rs @@ -0,0 +1,478 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Native pair generation as partitioned materialized-view refresh work. + +use std::sync::Arc; + +use arrow_schema::{Schema, SchemaRef}; +use datafusion_execution::{SendableRecordBatchStream, TaskContext}; +use datafusion_physical_plan::ExecutionPlan; +use datafusion_sql::sqlparser::ast::{ + Expr, FunctionArg, FunctionArgExpr, TableFunctionArgs, Value, +}; +use futures::TryStreamExt; +use lance::Dataset; +use lance::dataset::transaction::Operation; +use lance::dataset::{InsertBuilder, WriteDestination, WriteMode, WriteParams}; +use lance_table::format::Fragment; +use serde::{Deserialize, Serialize}; + +use super::grouped_units::GroupedRefreshPlan; +use super::refresh::{ + RefreshMaterializedViewResult, ensure_incarnation, ensure_no_mem_wal, open_source, +}; +use super::{MaterializedViewDefinition, Planned, ViewProjection}; +use crate::table::Table; +use crate::table::datafusion::udtf::duplicate_pairs::{ + DUPLICATE_PAIRS_OPERATOR_VERSION, DuplicatePairTask, DuplicatePairsConfig, DuplicatePairsExec, + duplicate_pair_schema, plan_duplicate_pairs, +}; +use crate::{Error, Result}; + +pub(super) const FUNCTION_NAME: &str = "vector_duplicate_pairs"; +pub(super) const DEDUP_FUNCTION_NAME: &str = "vector_dedup"; + +/// The result of a pinned native vector source. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum VectorSourceKind { + /// The qualifying pair relation. + Pairs, + /// Original source rows retained by greedy direct representatives. + Dedup, +} + +/// A native table-function source, pinned for the lifetime of the view. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct VectorSource { + pub kind: VectorSourceKind, + pub dataset_version: u64, + pub column: String, + /// Validated numeric SQL spelling; execution and task identity use f32 bits. + pub distance_threshold: String, +} + +impl VectorSource { + pub(super) fn function_name(&self) -> &'static str { + match self.kind { + VectorSourceKind::Pairs => FUNCTION_NAME, + VectorSourceKind::Dedup => DEDUP_FUNCTION_NAME, + } + } + pub(crate) fn config(&self) -> Result { + let threshold: f32 = self + .distance_threshold + .parse() + .map_err(|_| invalid("invalid pair threshold"))?; + if self.dataset_version == 0 || self.column.is_empty() || !threshold.is_finite() { + return Err(invalid( + "duplicate pairs require a positive snapshot, a column and a finite threshold", + )); + } + Ok(DuplicatePairsConfig { + dataset_version: self.dataset_version, + column: self.column.clone(), + distance_threshold: threshold, + }) + } +} + +fn invalid(message: impl Into) -> Error { + Error::InvalidInput { + message: message.into(), + } +} + +pub(super) fn source( + args: &TableFunctionArgs, + kind: VectorSourceKind, +) -> Result<(Vec, String, VectorSource)> { + if args.settings.is_some() { + return Err(invalid("duplicate pairs do not accept function settings")); + } + let exprs = args + .args + .iter() + .map(|arg| match arg { + FunctionArg::Unnamed(FunctionArgExpr::Expr(expr)) => Ok(expr), + _ => Err(invalid( + "duplicate pairs require four positional literal arguments", + )), + }) + .collect::>>()?; + let [table, version, column, threshold] = exprs.as_slice() else { + return Err(invalid( + "native vector sources require (table, dataset_version, column, distance_threshold)", + )); + }; + let string = |expr: &Expr| match expr { + Expr::Value(value) => match &value.value { + Value::SingleQuotedString(value) => Ok(value.clone()), + _ => Err(invalid("table and column must be string literals")), + }, + _ => Err(invalid("table and column must be string literals")), + }; + let table = string(table)?; + let plain = MaterializedViewDefinition::from_sql(&format!("SELECT * FROM {table}"))?; + if plain.vector_source.is_some() + || plain.lateral.is_some() + || plain.filter.is_some() + || plain.is_grouped() + || plain.limit.is_some() + { + return Err(invalid("duplicate pairs must name one source table")); + } + let dataset_version = match version { + Expr::Value(value) => match &value.value { + Value::Number(number, _) => number + .parse() + .map_err(|_| invalid("invalid source version"))?, + _ => return Err(invalid("source version must be an integer literal")), + }, + _ => return Err(invalid("source version must be an integer literal")), + }; + let config = VectorSource { + kind, + dataset_version, + column: string(column)?, + distance_threshold: threshold.to_string(), + }; + config.config()?; + Ok((plain.source_namespace, plain.source_table, config)) +} + +pub(super) fn check_shape(definition: &MaterializedViewDefinition) -> Result<()> { + if definition + .vector_source + .as_ref() + .is_some_and(|s| s.kind == VectorSourceKind::Dedup) + { + if !definition.selects_star() + || definition.filter.is_some() + || definition.limit.is_some() + || definition.lateral.is_some() + || definition.is_grouped() + { + return Err(invalid( + "a vector_dedup view must select * without other clauses", + )); + } + return Ok(()); + } + let schema = duplicate_pair_schema(); + let projections = schema + .fields() + .iter() + .map(|field| ViewProjection { + output: field.name().clone(), + expression: field.name().clone(), + }) + .collect::>(); + if (!definition.selects_star() && definition.projections != projections) + || definition.filter.is_some() + || definition.limit.is_some() + || definition.lateral.is_some() + || definition.is_grouped() + { + return Err(invalid( + "a native pair view selects *, or row_id_a, row_id_b, distance, without other clauses", + )); + } + Ok(()) +} + +pub(super) fn plan( + source_schema: SchemaRef, + definition: &MaterializedViewDefinition, +) -> Result { + check_shape(definition)?; + let config = definition.vector_source.as_ref().expect("native source"); + config.config()?; + source_schema.field_with_name(&config.column)?; + if config.kind == VectorSourceKind::Dedup { + return Ok(Planned { + definition: definition.clone(), + fields: source_schema + .fields() + .iter() + .map(|f| super::without_declarations(f)) + .collect(), + lineage: source_schema + .fields() + .iter() + .map(|f| (f.name().clone(), vec![f.name().clone()])) + .collect(), + inputs: source_schema + .fields() + .iter() + .map(|f| f.name().clone()) + .collect(), + }); + } + let schema = duplicate_pair_schema(); + let projections = schema + .fields() + .iter() + .map(|field| ViewProjection { + output: field.name().clone(), + expression: field.name().clone(), + }) + .collect(); + let mut definition = definition.clone(); + let inputs = vec![config.column.clone()]; + definition.projections = projections; + Ok(Planned { + definition, + fields: schema.fields().iter().map(|f| f.as_ref().clone()).collect(), + lineage: Default::default(), + inputs, + }) +} + +/// A saved native refresh plan, including every physical partition identity. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DuplicatePairsRefreshPlan { + pub snapshot: GroupedRefreshPlan, + pub source_uri: String, + pub definition_sql: String, + pub operator_version: u32, + pub threshold_bits: u32, + pub tasks: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WrittenPairsPartition { + pub(super) unit: u32, + pub(super) plan_id: String, + pub(super) task: DuplicatePairTask, + pub(super) fragments: Vec, + pub(super) rows: u64, +} + +impl DuplicatePairsRefreshPlan { + pub(super) fn identity(&self) -> Result { + use sha2::{Digest, Sha256}; + let encoded = serde_json::to_vec(&( + &self.snapshot, + &self.source_uri, + &self.definition_sql, + self.operator_version, + self.threshold_bits, + )) + .map_err(|e| invalid(e.to_string()))?; + Ok(format!("{:x}", Sha256::digest(encoded))) + } +} + +async fn open(view: &Table) -> Result<(Dataset, MaterializedViewDefinition, Arc)> { + let native = view + .as_native() + .ok_or_else(|| invalid("native pair refresh requires a local view"))?; + native.dataset.ensure_mutable()?; + native.dataset.reload().await?; + let view_ds = native.dataset.get().await?.as_ref().clone(); + let Some(super::StoredDefinition::Query(definition)) = + super::read_definition(&view_ds.schema().metadata)? + else { + return Err(invalid("view does not carry a supported definition")); + }; + let config = definition + .vector_source + .as_ref() + .ok_or_else(|| invalid("not a native pair view"))? + .config()?; + let source = Arc::new( + open_source(view, &definition, None) + .await? + .checkout_version(config.dataset_version) + .await?, + ); + ensure_no_mem_wal(&source, "source table", &definition.source_table).await?; + let planned = plan(Arc::new(Schema::from(source.schema())), &definition)?; + let expected = Schema::new(planned.fields); + let physical = Schema::from(view_ds.schema()); + if physical.fields().len() != expected.fields().len() + || physical + .fields() + .iter() + .zip(expected.fields()) + .any(|(a, b)| { + a.name() != b.name() + || a.data_type() != b.data_type() + || a.is_nullable() != b.is_nullable() + }) + { + return Err(invalid("native pair view schema changed since creation")); + } + Ok((view_ds, planned.definition, source)) +} + +pub(super) async fn plan_refresh( + view: &Table, + pinned: Option, +) -> Result { + let (view_ds, definition, source) = open(view).await?; + let config = definition + .vector_source + .as_ref() + .expect("native") + .config()?; + if pinned.is_some_and(|v| v != config.dataset_version) { + return Err(invalid( + "refresh source version differs from the native pair view's fixed snapshot", + )); + } + let tasks = plan_duplicate_pairs(source.clone(), &config).await?; + let units = u32::try_from(tasks.len()).map_err(|_| invalid("too many index partitions"))?; + let incarnation = view_ds + .schema() + .metadata + .get(super::INCARNATION_META_KEY) + .cloned() + .ok_or_else(|| invalid("native pair view has no incarnation"))?; + Ok(DuplicatePairsRefreshPlan { + snapshot: GroupedRefreshPlan { + units, + source_version: config.dataset_version, + view_version: view_ds.version().version, + incarnation, + }, + source_uri: source.uri().to_string(), + definition_sql: definition.to_sql(), + operator_version: DUPLICATE_PAIRS_OPERATOR_VERSION, + threshold_bits: config.distance_threshold.to_bits(), + tasks, + }) +} + +pub(super) async fn open_planned( + view: &Table, + plan: &DuplicatePairsRefreshPlan, +) -> Result<(Dataset, MaterializedViewDefinition, Arc)> { + let (view_ds, definition, source) = open(view).await?; + ensure_incarnation(&view_ds, Some(&plan.snapshot.incarnation), view.name()).await?; + let config = definition + .vector_source + .as_ref() + .expect("native") + .config()?; + if source.uri() != plan.source_uri + || source.version().version != plan.snapshot.source_version + || definition.to_sql() != plan.definition_sql + || config.distance_threshold.to_bits() != plan.threshold_bits + || plan.operator_version != DUPLICATE_PAIRS_OPERATOR_VERSION + || plan.tasks.len() != plan.snapshot.units as usize + { + return Err(invalid( + "native pair task belongs to a different snapshot, configuration or operator", + )); + } + Ok((view_ds, definition, source)) +} + +pub(super) async fn write_unit( + view: &Table, + unit: u32, + plan: &DuplicatePairsRefreshPlan, +) -> Result { + let (view_ds, definition, source) = open_planned(view, plan).await?; + let task = plan + .tasks + .get(unit as usize) + .ok_or_else(|| invalid("pair partition is outside the plan"))? + .clone(); + let config = definition + .vector_source + .as_ref() + .expect("native") + .config()?; + let exec = DuplicatePairsExec::try_new(source, config, vec![task.clone()])?; + let stream = exec.execute(0, Arc::new(TaskContext::default()))?; + let rows = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let count = rows.clone(); + let stream = stream.inspect_ok(move |batch| { + count.fetch_add( + batch.num_rows() as u64, + std::sync::atomic::Ordering::Relaxed, + ); + }); + let stream: SendableRecordBatchStream = Box::pin( + datafusion_physical_plan::stream::RecordBatchStreamAdapter::new( + duplicate_pair_schema(), + stream, + ), + ); + let txn = InsertBuilder::new(WriteDestination::Dataset(Arc::new(view_ds))) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted_stream(stream) + .await?; + let Operation::Append { fragments } = txn.operation else { + return Err(invalid( + "native pair partition did not stage append fragments", + )); + }; + Ok(WrittenPairsPartition { + unit, + plan_id: plan.identity()?, + task, + fragments, + rows: rows.load(std::sync::atomic::Ordering::Relaxed), + }) +} + +pub(super) async fn commit( + view: &Table, + plan: &DuplicatePairsRefreshPlan, + units: Vec, + expected: Option<&str>, +) -> Result { + let (_, definition, source) = open_planned(view, plan).await?; + let config = definition + .vector_source + .as_ref() + .expect("native") + .config()?; + if plan_duplicate_pairs(source.clone(), &config).await? != plan.tasks { + return Err(invalid( + "pair plan does not cover the pinned index partitions exactly", + )); + } + let identity = plan.identity()?; + let mut ordered = vec![None; plan.snapshot.units as usize]; + for written in units { + if written.plan_id != identity + || plan.tasks.get(written.unit as usize) != Some(&written.task) + { + return Err(invalid("pair partition receipt belongs to another plan")); + } + let slot = ordered + .get_mut(written.unit as usize) + .ok_or_else(|| invalid("unknown pair partition receipt"))?; + if slot.is_some() { + return Err(invalid("duplicate pair partition receipt")); + } + *slot = Some(written); + } + let units = ordered + .into_iter() + .collect::>>() + .ok_or_else(|| invalid("missing pair partition receipt"))?; + let rows = units.iter().map(|u| u.rows).sum(); + let fragments = units.into_iter().flat_map(|u| u.fragments).collect(); + super::grouped_units::commit_fragments( + view, + &plan.snapshot, + &definition, + source.manifest.timestamp_nanos, + fragments, + rows, + expected, + ) + .await +} diff --git a/rust/lancedb/src/materialized_view/grouped_units.rs b/rust/lancedb/src/materialized_view/grouped_units.rs index 44bbd7381..7f408be4d 100644 --- a/rust/lancedb/src/materialized_view/grouped_units.rs +++ b/rust/lancedb/src/materialized_view/grouped_units.rs @@ -53,6 +53,7 @@ const PROBE_ROWS: usize = 256; /// How a grouped view can be refreshed in units. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct GroupedRefreshPlan { /// Units `0..units`: unit `p` before the last holds index partition /// `p`'s groups; the last holds the rows the index cannot place (null or @@ -177,6 +178,7 @@ pub async fn plan_grouped_refresh( /// accounted for has to be refusable. Travels to whoever commits, so it /// serializes, but its fields are not a caller's to write. #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct WrittenUnit { unit: u32, plan: GroupedRefreshPlan, @@ -491,10 +493,38 @@ pub async fn commit_grouped_refresh( plan: &GroupedRefreshPlan, units: Vec, expected_incarnation: Option<&str>, +) -> Result { + let grouped = open_planned(view, plan).await?; + let units = complete_set(view.name(), plan, units)?; + let rows_written = units.iter().map(|unit| unit.rows).sum(); + let new_fragments = units.into_iter().flat_map(|unit| unit.fragments).collect(); + commit_fragments( + view, + plan, + &grouped.definition, + grouped.source.manifest.timestamp_nanos, + new_fragments, + rows_written, + expected_incarnation, + ) + .await +} + +/// The common all-or-nothing unit sink, shared by grouped and native views. +#[allow(clippy::too_many_arguments)] +pub(super) async fn commit_fragments( + view: &Table, + plan: &GroupedRefreshPlan, + definition: &MaterializedViewDefinition, + source_ts: u128, + new_fragments: Vec, + rows_written: u64, + expected_incarnation: Option<&str>, ) -> Result { let view_native = view.as_native().ok_or_else(|| Error::NotSupported { message: "materialized views are supported only on local tables".into(), })?; + view_native.dataset.ensure_mutable()?; let lock = refresh_lock(view_native.dataset.get().await?.uri()); let _guard = lock.lock().await; if let Some(expected) = expected_incarnation @@ -508,10 +538,17 @@ pub async fn commit_grouped_refresh( ), }); } - let grouped = open_planned(view, plan).await?; - let source_ts = grouped.source.manifest.timestamp_nanos; - - let ds = Arc::new(grouped.view_ds); + view_native.dataset.reload().await?; + let ds = view_native.dataset.get().await?.as_ref().clone(); + ensure_incarnation(&ds, Some(&plan.incarnation), view.name()).await?; + if super::read_definition(&ds.schema().metadata)? + != Some(super::StoredDefinition::Query(definition.clone())) + { + return Err(Error::InvalidInput { + message: "view definition changed while its units were written".into(), + }); + } + let ds = Arc::new(ds); if ds.version().version != plan.view_version { return Err(Error::Runtime { message: format!( @@ -523,11 +560,8 @@ pub async fn commit_grouped_refresh( ), }); } - let units = complete_set(view.name(), plan, units)?; let removed_fragment_ids: Vec = ds.get_fragments().iter().map(|f| f.id() as u64).collect(); // Fragment ids are assigned at commit; the units' fragments all carry 0. - let rows_written = units.iter().map(|unit| unit.rows).sum(); - let new_fragments = units.into_iter().flat_map(|unit| unit.fragments).collect(); let transaction = Transaction::new( plan.view_version, Operation::Update { @@ -566,7 +600,7 @@ pub async fn commit_grouped_refresh( committed, plan.source_version, source_ts, - Some(&grouped.definition), + Some(definition), expected_incarnation, ) .await?; diff --git a/rust/lancedb/src/materialized_view/partitioned.rs b/rust/lancedb/src/materialized_view/partitioned.rs new file mode 100644 index 000000000..58192089b --- /dev/null +++ b/rust/lancedb/src/materialized_view/partitioned.rs @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Shared plan/write/commit surface for materialized views with independent units. + +use serde::{Deserialize, Serialize}; + +use super::duplicate_pairs::{ + self, DuplicatePairsRefreshPlan, VectorSourceKind, WrittenPairsPartition, +}; +use super::vector_dedup::{self, VectorDedupPlan, WrittenDedupUnit}; +use super::{GroupedRefreshPlan, RefreshMaterializedViewResult, StoredDefinition, WrittenUnit}; +use crate::table::Table; +use crate::{Error, Result}; + +/// Serializable work description consumed by a host's existing job workers. +/// The grouped variant preserves the previous saved-plan representation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum PartitionedRefreshPlan { + VectorDedup(VectorDedupPlan), + DuplicatePairs(DuplicatePairsRefreshPlan), + Grouped(GroupedRefreshPlan), +} + +impl PartitionedRefreshPlan { + /// The shared source snapshot and destination generation fence. + pub fn snapshot(&self) -> &GroupedRefreshPlan { + match self { + Self::VectorDedup(plan) => &plan.snapshot, + Self::DuplicatePairs(plan) => &plan.snapshot, + Self::Grouped(plan) => plan, + } + } + /// Selection tasks that must complete before remaining tasks can start. + pub fn dependency_units(&self) -> u32 { + match self { + Self::VectorDedup(plan) => plan.dependencies(), + _ => 0, + } + } + /// Number of independent units in this immutable plan. + pub fn units(&self) -> u32 { + self.snapshot().units + } +} + +/// One completed unit's uncommitted fragments. Failed attempts produce no receipt. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum WrittenPartition { + VectorDedup(WrittenDedupUnit), + DuplicatePairs(WrittenPairsPartition), + Grouped(WrittenUnit), +} + +impl WrittenPartition { + /// Number of pair or grouped rows staged by this unit. + pub fn rows(&self) -> u64 { + match self { + Self::VectorDedup(unit) => unit.rows, + Self::DuplicatePairs(unit) => unit.rows, + Self::Grouped(unit) => unit.rows(), + } + } + /// Uncommitted output files, retained only after task completion. + pub fn fragments(&self) -> &[lance_table::format::Fragment] { + match self { + Self::VectorDedup(unit) => &unit.fragments, + Self::DuplicatePairs(unit) => &unit.fragments, + Self::Grouped(unit) => unit.fragments(), + } + } +} + +/// Plan native pair or indexed grouped work; return `None` for ordinary views. +/// `pinned`, when supplied, must match a native view's declared source version. +pub async fn plan_partitioned_refresh( + view: &Table, + pinned: Option, +) -> Result> { + if let Some(StoredDefinition::Query(definition)) = + super::read_definition(view.schema().await?.metadata())? + && definition.vector_source.is_some() + { + if definition + .vector_source + .as_ref() + .is_some_and(|s| s.kind == VectorSourceKind::Dedup) + { + return vector_dedup::plan_refresh(view, pinned) + .await + .map(|p| Some(PartitionedRefreshPlan::VectorDedup(p))); + } + return duplicate_pairs::plan_refresh(view, pinned) + .await + .map(|p| Some(PartitionedRefreshPlan::DuplicatePairs(p))); + } + super::plan_grouped_refresh(view, pinned) + .await + .map(|plan| plan.map(PartitionedRefreshPlan::Grouped)) +} + +/// Stream one unit into uncommitted fragments. Persist the returned receipt only +/// on success. A failed or interrupted call must be retried from the unit's start. +pub async fn write_refresh_partition( + view: &Table, + unit: u32, + plan: &PartitionedRefreshPlan, +) -> Result { + write_refresh_partition_with_inputs(view, unit, plan, &[]).await +} + +/// Write a unit after its durable selection dependencies have completed. +/// Supply exactly the first `plan.dependency_units()` receipts for a dependent +/// unit. A missing, duplicate or foreign receipt fails before output is staged. +pub async fn write_refresh_partition_with_inputs( + view: &Table, + unit: u32, + plan: &PartitionedRefreshPlan, + dependencies: &[WrittenPartition], +) -> Result { + match plan { + PartitionedRefreshPlan::VectorDedup(plan) => { + let dependencies = dependencies + .iter() + .map(|unit| match unit { + WrittenPartition::VectorDedup(unit) => Ok(unit.clone()), + _ => Err(Error::InvalidInput { + message: "foreign dedup dependency".into(), + }), + }) + .collect::>>()?; + vector_dedup::write_unit(view, unit, plan, &dependencies) + .await + .map(WrittenPartition::VectorDedup) + } + PartitionedRefreshPlan::DuplicatePairs(plan) => { + duplicate_pairs::write_unit(view, unit, plan) + .await + .map(WrittenPartition::DuplicatePairs) + } + PartitionedRefreshPlan::Grouped(plan) => super::write_grouped_unit(view, unit, plan) + .await + .map(WrittenPartition::Grouped), + } +} + +/// Publish a complete set of unit receipts through the common MV commit sink. +/// `expected` fences the destination incarnation. No partial set is publishable. +pub async fn commit_partitioned_refresh( + view: &Table, + plan: &PartitionedRefreshPlan, + units: Vec, + expected: Option<&str>, +) -> Result { + let foreign = || Error::InvalidInput { + message: "refresh receipt does not match the plan kind".into(), + }; + match plan { + PartitionedRefreshPlan::VectorDedup(plan) => { + let units = units + .into_iter() + .map(|unit| match unit { + WrittenPartition::VectorDedup(unit) => Ok(unit), + _ => Err(foreign()), + }) + .collect::>>()?; + vector_dedup::commit(view, plan, units, expected).await + } + PartitionedRefreshPlan::DuplicatePairs(plan) => { + let units = units + .into_iter() + .map(|unit| match unit { + WrittenPartition::DuplicatePairs(unit) => Ok(unit), + _ => Err(foreign()), + }) + .collect::>>()?; + duplicate_pairs::commit(view, plan, units, expected).await + } + PartitionedRefreshPlan::Grouped(plan) => { + let units = units + .into_iter() + .map(|unit| match unit { + WrittenPartition::Grouped(unit) => Ok(unit), + _ => Err(foreign()), + }) + .collect::>>()?; + super::commit_grouped_refresh(view, plan, units, expected).await + } + } +} diff --git a/rust/lancedb/src/materialized_view/query.rs b/rust/lancedb/src/materialized_view/query.rs index c1a987b2c..4bfd5542e 100644 --- a/rust/lancedb/src/materialized_view/query.rs +++ b/rust/lancedb/src/materialized_view/query.rs @@ -264,7 +264,36 @@ fn extract(query: &Query) -> Result { }; let mut from = select.from.iter(); + let mut vector_source = None; let (source_namespace, source_table) = match from.next().map(|f| &f.relation) { + Some(TableFactor::Table { + name, + args: Some(args), + alias: None, + with_hints, + version: None, + with_ordinality: false, + partitions, + json_path: None, + sample: None, + index_hints, + .. + }) if matches!( + name.to_string().as_str(), + super::duplicate_pairs::FUNCTION_NAME | super::duplicate_pairs::DEDUP_FUNCTION_NAME + ) && with_hints.is_empty() + && partitions.is_empty() + && index_hints.is_empty() => + { + let kind = if name.to_string() == super::duplicate_pairs::FUNCTION_NAME { + super::VectorSourceKind::Pairs + } else { + super::VectorSourceKind::Dedup + }; + let (namespace, table, config) = super::duplicate_pairs::source(args, kind)?; + vector_source = Some(Box::new(config)); + (namespace, table) + } Some(TableFactor::Table { args: Some(_), .. }) => { return Err(invalid( "a view reads a table; a Function in FROM position follows it: \ @@ -416,6 +445,7 @@ fn extract(query: &Query) -> Result { }; let definition = MaterializedViewDefinition { + vector_source, source_table, source_namespace, lateral, @@ -424,6 +454,9 @@ fn extract(query: &Query) -> Result { group_by, limit, }; + if definition.vector_source.is_some() { + super::duplicate_pairs::check_shape(&definition)?; + } check_grouping(&definition)?; Ok(definition) } @@ -483,7 +516,18 @@ pub fn render(definition: &MaterializedViewDefinition) -> String { .chain(std::iter::once(&definition.source_table)) .map(|part| ident_sql(part)) .collect(); - sql.push_str(&table.join(".")); + if let Some(native) = &definition.vector_source { + sql.push_str(&format!( + "{}('{}', {}, '{}', {})", + native.function_name(), + table.join(".").replace('\'', "''"), + native.dataset_version, + native.column.replace('\'', "''"), + native.distance_threshold + )); + } else { + sql.push_str(&table.join(".")); + } if let Some(lateral) = &definition.lateral { match &lateral.source { LateralSource::Unnest { column } => { @@ -511,6 +555,45 @@ pub fn render(definition: &MaterializedViewDefinition) -> String { mod tests { use super::*; + #[test] + fn native_pairs_are_a_table_source_with_literal_snapshot_arguments() { + for threshold in ["4", "4.0", "-1", "0.05"] { + let sql = + format!("SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', {threshold})"); + let definition = parse(&sql).unwrap(); + assert_eq!(definition.source_table, "images"); + assert!(definition.lateral.is_none()); + assert_eq!( + definition.vector_source.as_ref().unwrap().dataset_version, + 3 + ); + assert_eq!(parse(&render(&definition)).unwrap(), definition); + let metadata = std::collections::HashMap::from([( + super::super::DEFINITION_META_KEY.to_string(), + definition.to_json().unwrap(), + )]); + assert_eq!( + super::super::read_definition(&metadata).unwrap(), + Some(super::super::StoredDefinition::Query(definition)) + ); + } + for sql in [ + "SELECT row_id_a FROM vector_duplicate_pairs('images', 3, 'phash', 4)", + "SELECT * FROM vector_duplicate_pairs('images', 0, 'phash', 4)", + "SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', 1e100)", + "SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', cutoff)", + "SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', 4) LIMIT 1", + "SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', 4) WHERE distance < 2", + "SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', 4) AS p", + "SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', 4), f(phash) AS p", + ] { + assert!( + parse(sql).is_err(), + "accepted unsupported native view: {sql}" + ); + } + } + #[test] fn a_query_round_trips_through_its_canonical_form() { for (sql, canonical) in [ diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs index 5e9fe13e3..9f0ea2646 100644 --- a/rust/lancedb/src/materialized_view/refresh.rs +++ b/rust/lancedb/src/materialized_view/refresh.rs @@ -117,6 +117,26 @@ pub(crate) async fn execute_refresh( pinned: Option, expected_incarnation: Option<&str>, ) -> Result { + // Native table functions use the same partition writer and commit sink in + // local execution; Sophon drives these units through its job registry. + if let Some(super::StoredDefinition::Query(definition)) = + super::read_definition(view.schema().await?.metadata())? + && definition.vector_source.is_some() + { + let plan = super::plan_partitioned_refresh(view, pinned) + .await? + .ok_or_else(|| Error::Runtime { + message: "native pair view has no partition plan".into(), + })?; + let mut units = Vec::with_capacity(plan.units() as usize); + for unit in 0..plan.units() { + let dependencies = &units[..units.len().min(plan.dependency_units() as usize)]; + units.push( + super::write_refresh_partition_with_inputs(view, unit, &plan, dependencies).await?, + ); + } + return super::commit_partitioned_refresh(view, &plan, units, expected_incarnation).await; + } let view_native = view.as_native().ok_or_else(|| Error::NotSupported { message: "materialized views are supported only on local tables".into(), })?; @@ -728,7 +748,7 @@ pub(super) async fn open_source( message: "materialized views are supported only on local tables".into(), })?; let dataset = native.dataset.get().await?.as_ref().clone(); - if !dataset.manifest.uses_stable_row_ids() { + if definition.vector_source.is_none() && !dataset.manifest.uses_stable_row_ids() { return Err(Error::InvalidInput { message: format!( "source table '{}' does not have stable row ids; it is not the \ @@ -3674,6 +3694,7 @@ pub(crate) mod tests { let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await; let replacement = crate::materialized_view::MaterializedViewDefinition { + vector_source: None, source_table: "src".into(), source_namespace: Vec::new(), projections: vec![ @@ -3715,6 +3736,7 @@ pub(crate) mod tests { let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await; let narrower = crate::materialized_view::MaterializedViewDefinition { + vector_source: None, source_table: "src".into(), source_namespace: Vec::new(), projections: vec![crate::materialized_view::ViewProjection { diff --git a/rust/lancedb/src/materialized_view/vector_dedup.rs b/rust/lancedb/src/materialized_view/vector_dedup.rs new file mode 100644 index 000000000..dc2759433 --- /dev/null +++ b/rust/lancedb/src/materialized_view/vector_dedup.rs @@ -0,0 +1,636 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Two dependent waves in the ordinary refresh fleet: index partition selection, +//! then source fragment materialization. Only the latter publishes data files. + +use std::collections::BTreeMap; +use std::io::Cursor; +use std::sync::Arc; + +use arrow_array::{Array, RecordBatch, UInt64Array}; +use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use datafusion::prelude::{SessionConfig, SessionContext, col}; +use datafusion_common::DataFusionError; +use datafusion_execution::{ + SendableRecordBatchStream, TaskContext, runtime_env::RuntimeEnvBuilder, +}; +use datafusion_physical_plan::{ExecutionPlan, stream::RecordBatchStreamAdapter}; +use futures::TryStreamExt; +use lance::Dataset; +use lance::dataset::rowids::{get_row_id_index, load_row_id_sequence}; +use lance::dataset::scanner::{RowAddrMask, RowAddrTreeMap}; +use lance::dataset::transaction::Operation; +use lance::dataset::{InsertBuilder, WriteDestination, WriteMode, WriteParams}; +use lance_core::datatypes::BlobHandling; +use lance_core::utils::address::RowAddress; +use lance_datafusion::exec::{OneShotExec, SessionContextExt}; +use lance_table::format::Fragment; +use roaring::{RoaringBitmap, RoaringTreemap}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use super::duplicate_pairs::{self, DuplicatePairsRefreshPlan, VectorSourceKind}; +use super::{GroupedRefreshPlan, RefreshMaterializedViewResult}; +use crate::table::Table; +use crate::table::datafusion::udtf::duplicate_pairs::{DuplicatePairsExec, plan_duplicate_pairs}; +use crate::{Error, Result}; + +const POLICY_VERSION: u32 = 1; +const SORT_MEMORY_BYTES: usize = 256 * 1024 * 1024; +const MATERIALIZE_BATCH_ROWS: usize = 1024; +const MASK_FILE: &str = "dedup.bitmap"; +const MASK_MAGIC: &[u8] = b"LANCEDB_DEDUP_1"; + +fn invalid(message: impl Into) -> Error { + Error::InvalidInput { + message: message.into(), + } +} +fn io_error(error: std::io::Error) -> Error { + invalid(format!("invalid dedup bitmap: {error}")) +} + +/// Immutable snapshot, partition tasks and source fragments for both waves. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct VectorDedupPlan { + pub snapshot: GroupedRefreshPlan, + native: DuplicatePairsRefreshPlan, + source_fragments: Vec, + policy_version: u32, +} +impl VectorDedupPlan { + pub(super) fn dependencies(&self) -> u32 { + self.native.snapshot.units + } + fn identity(&self) -> Result { + Ok(format!( + "{:x}", + Sha256::digest(serde_json::to_vec(self).map_err(|e| invalid(e.to_string()))?) + )) + } +} + +/// Small receipt for a durable selection mask. Row IDs and pairs are not stored +/// in the host's task registry. UUID directories use ordinary index-file GC. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct MaskFile { + source_fragment: u32, + object_id: Uuid, + bytes: usize, + digest: String, +} + +/// A selection task produces masks; a materialization task produces data files. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WrittenDedupUnit { + unit: u32, + plan_id: String, + masks: Vec, + pub(super) fragments: Vec, + pub(super) rows: u64, +} + +pub(super) async fn plan_refresh(view: &Table, pinned: Option) -> Result { + let native = duplicate_pairs::plan_refresh(view, pinned).await?; + let (_, _, source) = duplicate_pairs::open_planned(view, &native).await?; + let source_fragments = source + .get_fragments() + .iter() + .map(|f| f.id() as u32) + .collect::>(); + let mut snapshot = native.snapshot.clone(); + snapshot.units = snapshot + .units + .checked_add( + u32::try_from(source_fragments.len()) + .map_err(|_| invalid("too many source fragments"))?, + ) + .ok_or_else(|| invalid("too many dedup refresh units"))?; + Ok(VectorDedupPlan { + snapshot, + native, + source_fragments, + policy_version: POLICY_VERSION, + }) +} + +async fn open( + view: &Table, + plan: &VectorDedupPlan, +) -> Result<(Dataset, super::MaterializedViewDefinition, Arc)> { + let (view_ds, definition, source) = duplicate_pairs::open_planned(view, &plan.native).await?; + let mut snapshot = plan.native.snapshot.clone(); + snapshot.units = snapshot + .units + .checked_add( + u32::try_from(plan.source_fragments.len()) + .map_err(|_| invalid("too many dedup fragments"))?, + ) + .ok_or_else(|| invalid("too many dedup units"))?; + if plan.policy_version != POLICY_VERSION + || snapshot != plan.snapshot + || definition.vector_source.as_ref().map(|s| s.kind) != Some(VectorSourceKind::Dedup) + || source + .get_fragments() + .iter() + .map(|f| f.id() as u32) + .collect::>() + != plan.source_fragments + { + return Err(invalid("dedup task belongs to another snapshot or policy")); + } + Ok((view_ds, definition, source)) +} + +fn edge_schema() -> SchemaRef { + Arc::new(Schema::new(vec![ + Field::new("a", DataType::UInt64, false), + Field::new("b", DataType::UInt64, false), + ])) +} + +/// Sorting canonical edges makes the first unremoved endpoint the smallest +/// unassigned representative. The sort spills; only vertex masks stay in RAM. +pub async fn select_representatives( + stream: SendableRecordBatchStream, + memory: usize, +) -> Result<(RoaringTreemap, RoaringTreemap)> { + let schema = edge_schema(); + let batch_schema = schema.clone(); + let canonical = stream + .map_ok(move |batch| { + let a = batch + .column(0) + .as_any() + .downcast_ref::() + .expect("native row id"); + let b = batch + .column(1) + .as_any() + .downcast_ref::() + .expect("native row id"); + RecordBatch::try_new( + batch_schema.clone(), + vec![ + Arc::new(UInt64Array::from_iter_values( + a.values().iter().zip(b.values()).map(|(&a, &b)| a.min(b)), + )), + Arc::new(UInt64Array::from_iter_values( + a.values().iter().zip(b.values()).map(|(&a, &b)| a.max(b)), + )), + ], + ) + .map_err(DataFusionError::from) + }) + .and_then(futures::future::ready); + let canonical = Box::pin(RecordBatchStreamAdapter::new(schema, canonical)); + let mut config = SessionConfig::default().with_batch_size(MATERIALIZE_BATCH_ROWS); + // Leave enough room to merge spilled runs while the final in-memory run + // is still resident. Small batches also bound each merge input buffer. + config.options_mut().execution.sort_spill_reservation_bytes = memory / 2; + let ctx = SessionContext::new_with_config_rt( + config, + RuntimeEnvBuilder::new() + .with_memory_limit(memory, 1.0) + .build_arc()?, + ); + let mut sorted = ctx + .read_one_shot(canonical)? + .sort_by(vec![col("a"), col("b")])? + .execute_stream() + .await?; + let mut removed = RoaringTreemap::new(); + let mut representatives = RoaringTreemap::new(); + while let Some(batch) = sorted.try_next().await? { + let a = batch + .column(0) + .as_any() + .downcast_ref::() + .expect("sorted id"); + let b = batch + .column(1) + .as_any() + .downcast_ref::() + .expect("sorted id"); + for (&a, &b) in a.values().iter().zip(b.values()) { + if a != b && !removed.contains(a) && !removed.contains(b) { + representatives.insert(a); + removed.insert(b); + } + } + } + Ok((removed, representatives)) +} + +#[derive(Default)] +struct Masks { + removed: RoaringBitmap, + representatives: RoaringBitmap, +} + +async fn persist_masks( + view: &Dataset, + source: &Dataset, + removed: RoaringTreemap, + representatives: RoaringTreemap, +) -> Result> { + let row_index = get_row_id_index(source).await?; + let mut masks: BTreeMap = BTreeMap::new(); + for (ids, is_removed) in [(removed, true), (representatives, false)] { + for id in ids { + let address = match &row_index { + Some(index) => index + .get(id)? + .ok_or_else(|| invalid("pair row id missing from pinned source"))?, + None => RowAddress::new_from_u64(id), + }; + let mask = masks.entry(address.fragment_id()).or_default(); + if is_removed { + mask.removed.insert(address.row_offset()); + } else { + mask.representatives.insert(address.row_offset()); + } + } + } + let store = view.object_store(None).await?; + let mut files = Vec::new(); + for (source_fragment, mask) in masks { + if source.get_fragment(source_fragment as usize).is_none() { + return Err(invalid("pair row address is outside the pinned snapshot")); + } + let mut bytes = MASK_MAGIC.to_vec(); + mask.removed.serialize_into(&mut bytes).map_err(io_error)?; + mask.representatives + .serialize_into(&mut bytes) + .map_err(io_error)?; + let object_id = Uuid::new_v4(); + store + .put( + &view + .indices_dir() + .join(object_id.to_string()) + .join(MASK_FILE), + &bytes, + ) + .await?; + files.push(MaskFile { + source_fragment, + object_id, + bytes: bytes.len(), + digest: format!("{:x}", Sha256::digest(&bytes)), + }); + } + Ok(files) +} + +fn validate_receipts<'a>( + plan: &VectorDedupPlan, + receipts: &'a [WrittenDedupUnit], + count: u32, +) -> Result> { + let identity = plan.identity()?; + let mut ordered = vec![None; count as usize]; + for receipt in receipts { + if receipt.plan_id != identity { + return Err(invalid("dedup receipt belongs to another plan")); + } + let slot = ordered + .get_mut(receipt.unit as usize) + .ok_or_else(|| invalid("unexpected dedup receipt"))?; + if slot.is_some() { + return Err(invalid("duplicate dedup receipt")); + } + if receipt.unit < plan.dependencies() { + if !receipt.fragments.is_empty() + || receipt.rows != 0 + || receipt + .masks + .iter() + .any(|m| !plan.source_fragments.contains(&m.source_fragment)) + { + return Err(invalid("invalid selection receipt")); + } + } else if !receipt.masks.is_empty() { + return Err(invalid("invalid materialization receipt")); + } + *slot = Some(receipt); + } + ordered + .into_iter() + .collect::>>() + .ok_or_else(|| invalid("missing dedup dependency receipt")) +} + +async fn load_masks( + view: &Dataset, + source_fragment: u32, + dependencies: &[&WrittenDedupUnit], +) -> Result { + let store = view.object_store(None).await?; + let mut masks = Masks::default(); + for file in dependencies + .iter() + .flat_map(|u| &u.masks) + .filter(|m| m.source_fragment == source_fragment) + { + let reader = store + .open( + &view + .indices_dir() + .join(file.object_id.to_string()) + .join(MASK_FILE), + ) + .await?; + if reader.size().await? != file.bytes { + return Err(invalid("dedup mask length mismatch")); + } + let bytes = reader.get_range(0..file.bytes).await?; + if !bytes.starts_with(MASK_MAGIC) || format!("{:x}", Sha256::digest(&bytes)) != file.digest + { + return Err(invalid("dedup mask checksum mismatch")); + } + let mut cursor = Cursor::new(&bytes[MASK_MAGIC.len()..]); + masks.removed |= RoaringBitmap::deserialize_from(&mut cursor).map_err(io_error)?; + masks.representatives |= RoaringBitmap::deserialize_from(&mut cursor).map_err(io_error)?; + if cursor.position() as usize != bytes.len() - MASK_MAGIC.len() { + return Err(invalid("dedup mask has trailing data")); + } + } + // Native index scopes normally have disjoint rows. Fail closed if an + // overlapping scope would delete a representative used by another scope. + if !masks.removed.is_disjoint(&masks.representatives) { + return Err(invalid( + "overlapping index scopes disagree on retained representatives", + )); + } + Ok(masks.removed) +} + +pub(super) async fn write_unit( + view: &Table, + unit: u32, + plan: &VectorDedupPlan, + dependencies: &[WrittenDedupUnit], +) -> Result { + let (view_ds, definition, source) = open(view, plan).await?; + let mut written = WrittenDedupUnit { + unit, + plan_id: plan.identity()?, + masks: Vec::new(), + fragments: Vec::new(), + rows: 0, + }; + if unit < plan.dependencies() { + let config = definition + .vector_source + .as_ref() + .expect("dedup source") + .config()?; + let exec = DuplicatePairsExec::try_new( + source.clone(), + config, + vec![plan.native.tasks[unit as usize].clone()], + )?; + let stream = exec.execute(0, Arc::new(TaskContext::default()))?; + let (removed, representatives) = select_representatives(stream, SORT_MEMORY_BYTES).await?; + written.masks = persist_masks(&view_ds, &source, removed, representatives).await?; + return Ok(written); + } + let source_fragment = *plan + .source_fragments + .get((unit - plan.dependencies()) as usize) + .ok_or_else(|| invalid("dedup unit is outside the plan"))?; + let dependencies = validate_receipts(plan, dependencies, plan.dependencies())?; + let removed = load_masks(&view_ds, source_fragment, &dependencies).await?; + let fragment = source + .get_fragment(source_fragment as usize) + .ok_or_else(|| invalid("source fragment missing"))?; + // The scanner prefilter uses snapshot row IDs. Our persisted masks use + // physical offsets so each writer only loads its own fragment's artifacts. + let blocked = if source.manifest.uses_stable_row_ids() { + let sequence = load_row_id_sequence(&source, fragment.metadata()).await?; + if removed + .max() + .is_some_and(|offset| offset as u64 >= sequence.len()) + { + return Err(invalid("dedup mask offset is outside the source fragment")); + } + RowAddrTreeMap::from_iter(sequence.select(removed.iter().map(|offset| offset as usize))) + } else { + RowAddrTreeMap::from_iter( + removed + .iter() + .map(|offset| u64::from(RowAddress::new_from_parts(source_fragment, offset))), + ) + }; + let mut scanner = source.scan(); + scanner + .with_fragments(vec![fragment.metadata().clone()]) + .with_row_addr_prefilter(RowAddrMask::from_block(blocked)) + .blob_handling(BlobHandling::AllBinary) + .batch_size(MATERIALIZE_BATCH_ROWS) + .strict_batch_size(true); + let stream = scanner.try_into_stream().await?; + let schema = lance_io::stream::RecordBatchStream::schema(&stream); + let rows = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let count = rows.clone(); + let stream = stream + .map_err(|e| DataFusionError::External(Box::new(e))) + .inspect_ok(move |batch| { + count.fetch_add( + batch.num_rows() as u64, + std::sync::atomic::Ordering::Relaxed, + ); + }); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(schema, stream)); + // Materialized binary payloads must become the destination's logical blob + // structs before the Lance writer encodes them. Reuse the normal add path's + // coercions, including nulls and nested fields. + let coerced = crate::table::datafusion::cast::cast_to_table_schema( + Arc::new(OneShotExec::new(stream)), + &Schema::from(view_ds.schema()), + )?; + let stream = coerced.execute(0, Arc::new(TaskContext::default()))?; + let txn = InsertBuilder::new(WriteDestination::Dataset(Arc::new(view_ds))) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted_stream(stream) + .await?; + let Operation::Append { fragments } = txn.operation else { + return Err(invalid( + "dedup materialization did not stage append fragments", + )); + }; + written.fragments = fragments; + written.rows = rows.load(std::sync::atomic::Ordering::Relaxed); + Ok(written) +} + +pub(super) async fn commit( + view: &Table, + plan: &VectorDedupPlan, + units: Vec, + expected: Option<&str>, +) -> Result { + let (_, definition, source) = open(view, plan).await?; + let config = definition + .vector_source + .as_ref() + .expect("dedup source") + .config()?; + if plan_duplicate_pairs(source.clone(), &config).await? != plan.native.tasks { + return Err(invalid("dedup plan does not cover the pinned index")); + } + validate_receipts(plan, &units, plan.snapshot.units)?; + let rows = units.iter().map(|u| u.rows).sum(); + let fragments = units.into_iter().flat_map(|u| u.fragments).collect(); + super::grouped_units::commit_fragments( + view, + &plan.snapshot, + &definition, + source.manifest.timestamp_nanos, + fragments, + rows, + expected, + ) + .await +} + +#[cfg(test)] +mod tests { + use super::*; + + fn edges(values: &[(u64, u64)], batch_size: usize) -> SendableRecordBatchStream { + let schema = edge_schema(); + let batches = values + .chunks(batch_size) + .map(|edges| { + Ok(RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt64Array::from_iter_values(edges.iter().map(|e| e.0))), + Arc::new(UInt64Array::from_iter_values(edges.iter().map(|e| e.1))), + ], + ) + .unwrap()) + }) + .collect::>(); + Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::iter(batches), + )) + } + + #[tokio::test] + async fn direct_representatives_handle_chains_orientation_and_uint64() { + let high = u64::MAX - 2; + let pairs = [ + (2, 1), + (1, 0), + (high + 2, high + 1), + (high, high + 1), + (1, 0), + ]; + let (removed, representatives) = + select_representatives(edges(&pairs, 1), SORT_MEMORY_BYTES) + .await + .unwrap(); + assert_eq!(removed.iter().collect::>(), [1, high + 1]); + assert_eq!(representatives.iter().collect::>(), [0, high]); + } + + #[tokio::test] + async fn masks_reject_overlapping_representatives_and_corrupt_artifacts() { + let dir = tempfile::tempdir().unwrap(); + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::UInt64, false)])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(UInt64Array::from(vec![0, 1, 2]))], + ) + .unwrap(); + let ds = Dataset::write( + arrow_array::RecordBatchIterator::new(vec![Ok(batch)], schema), + dir.path().join("source.lance").to_str().unwrap(), + None, + ) + .await + .unwrap(); + let first = persist_masks( + &ds, + &ds, + [1].into_iter().collect(), + [0].into_iter().collect(), + ) + .await + .unwrap(); + let second = persist_masks( + &ds, + &ds, + [2].into_iter().collect(), + [1].into_iter().collect(), + ) + .await + .unwrap(); + let receipt = |masks| WrittenDedupUnit { + unit: 0, + plan_id: "test".into(), + masks, + fragments: vec![], + rows: 0, + }; + let first = receipt(first); + let second = receipt(second); + assert_eq!( + load_masks(&ds, 0, &[&first]).await.unwrap(), + [1].into_iter().collect() + ); + assert!( + load_masks(&ds, 0, &[&first, &second]) + .await + .unwrap_err() + .to_string() + .contains("overlapping") + ); + let file = &first.masks[0]; + ds.object_store(None) + .await + .unwrap() + .put( + &ds.indices_dir() + .join(file.object_id.to_string()) + .join(MASK_FILE), + &vec![0; file.bytes], + ) + .await + .unwrap(); + assert!( + load_masks(&ds, 0, &[&first]) + .await + .unwrap_err() + .to_string() + .contains("checksum") + ); + } + + #[tokio::test] + async fn external_sort_handles_more_edges_than_its_budget() { + let pairs = (0..262144) + .rev() + .map(|i| ((i + 1) as u64, i as u64)) + .collect::>(); + let (removed, reps) = select_representatives(edges(&pairs, 4096), 2 * 1024 * 1024) + .await + .unwrap(); + assert_eq!(removed.len(), 131072); + assert_eq!(reps.len(), 131072); + assert!(removed.iter().all(|id| id % 2 == 1)); + assert!(reps.iter().all(|id| id % 2 == 0)); + } +} diff --git a/rust/lancedb/src/query.rs b/rust/lancedb/src/query.rs index c8b19f013..fffb703fb 100644 --- a/rust/lancedb/src/query.rs +++ b/rust/lancedb/src/query.rs @@ -1252,10 +1252,17 @@ impl VectorQuery { /// This method sets both the minimum and maximum number of partitions to search. /// For more fine-grained control see [`VectorQuery::minimum_nprobes`] and /// [`VectorQuery::maximum_nprobes`]. - pub fn nprobes(mut self, nprobes: usize) -> Self { + /// + /// Returns an error if `nprobes` is not greater than 0. + pub fn nprobes(mut self, nprobes: usize) -> Result { + if nprobes == 0 { + return Err(Error::InvalidInput { + message: "nprobes must be greater than 0".to_string(), + }); + } self.request.minimum_nprobes = nprobes; self.request.maximum_nprobes = Some(nprobes); - self + Ok(self) } /// Set the minimum number of partitions to search @@ -2361,6 +2368,7 @@ mod tests { .nearest_to(&[9.8, 8.7]) .unwrap() .nprobes(1000) + .unwrap() .postfilter() .distance_type(DistanceType::Cosine) .approx_mode(ApproxMode::Accurate) @@ -2386,6 +2394,31 @@ mod tests { assert_eq!(query.request.refine_factor, Some(999)); } + #[tokio::test] + async fn test_nprobes_validation() { + let conn = connect("memory://").execute().await.unwrap(); + let table = conn + .create_table("my_table", make_test_batches()) + .execute() + .await + .unwrap(); + let query = table.query().nearest_to(&[0.1, 0.2]).unwrap(); + + assert!(matches!( + query.clone().nprobes(0), + Err(Error::InvalidInput { message }) if message == "nprobes must be greater than 0" + )); + + // Fixed probe counts replace both bounds, even when moving below or + // above the previous range. + let mut query = query; + for nprobes in [30, 1, 50] { + query = query.nprobes(nprobes).unwrap(); + assert_eq!(query.request.minimum_nprobes, nprobes); + assert_eq!(query.request.maximum_nprobes, Some(nprobes)); + } + } + #[test] fn test_approx_mode_serde_parse_default_and_display() { assert_eq!(ApproxMode::default(), ApproxMode::Normal); diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 39bb5a528..27606cdfe 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -979,8 +979,9 @@ impl Database for RemoteDatabase { async fn create_function_async( &self, request: FunctionRegistrationRequest, + namespace_path: &[String], ) -> Result> { - let function_id = build_object_identifier("Function name", &request.name, &[])?; + let function_id = build_object_identifier("Function name", &request.name, namespace_path)?; let req = self .client .post(&format!("/v1/function/{function_id}/create")) @@ -1005,8 +1006,13 @@ impl Database for RemoteDatabase { )))) } - async fn get_function(&self, name: &str, version: &str) -> Result { - let function_id = build_object_identifier("Function name", name, &[])?; + async fn get_function( + &self, + name: &str, + version: &str, + namespace_path: &[String], + ) -> Result { + let function_id = build_object_identifier("Function name", name, namespace_path)?; let req = self .client .post(&format!("/v1/function/{function_id}/describe")) @@ -1018,8 +1024,8 @@ impl Database for RemoteDatabase { response.json().await.err_to_http(request_id) } - async fn list_functions(&self) -> Result> { - let namespace_id = build_namespace_identifier(&[])?; + async fn list_functions(&self, namespace_path: &[String]) -> Result> { + let namespace_id = build_namespace_identifier(namespace_path)?; let path = format!("/v1/namespace/{namespace_id}/function/list"); let mut functions = Vec::new(); let mut page_token: Option = None; @@ -1059,12 +1065,25 @@ impl Database for RemoteDatabase { Ok(functions) } - async fn drop_function(&self, name: &str, version: &str) -> Result { - Ok(self.drop_function_async(name, version).await?.0) + async fn drop_function( + &self, + name: &str, + version: &str, + namespace_path: &[String], + ) -> Result { + Ok(self + .drop_function_async(name, version, namespace_path) + .await? + .0) } - async fn drop_function_async(&self, name: &str, version: &str) -> Result<(bool, Job)> { - let function_id = build_object_identifier("Function name", name, &[])?; + async fn drop_function_async( + &self, + name: &str, + version: &str, + namespace_path: &[String], + ) -> Result<(bool, Job)> { + let function_id = build_object_identifier("Function name", name, namespace_path)?; let req = self .client .post(&format!("/v1/function/{function_id}/drop")) @@ -2004,7 +2023,7 @@ mod tests { .body(serde_json::json!({"dropped": true, "job_id": "j1-fn-drop"}).to_string()) .unwrap() }); - let (dropped, job) = db.drop_function_async("embed", "1").await.unwrap(); + let (dropped, job) = db.drop_function_async("embed", "1", &[]).await.unwrap(); assert!(dropped); assert_eq!(job.id(), Some("j1-fn-drop")); } @@ -2019,7 +2038,7 @@ mod tests { .body(serde_json::json!({"dropped": false}).to_string()) .unwrap() }); - let (dropped, job) = db.drop_function_async("embed", "1").await.unwrap(); + let (dropped, job) = db.drop_function_async("embed", "1", &[]).await.unwrap(); assert!(!dropped); assert_eq!(job.id(), None); assert_eq!(job.status().await.unwrap(), "finished"); @@ -2036,7 +2055,11 @@ mod tests { let db = super::RemoteDatabase::new_mock(move |_| { http::Response::builder().status(202).body(body).unwrap() }); - let error = db.drop_function_async("embed", "1").await.err().unwrap(); + let error = db + .drop_function_async("embed", "1", &[]) + .await + .err() + .unwrap(); assert!(error.to_string().contains("valid job_id")); } } @@ -3905,7 +3928,7 @@ mod tests { http::Response::builder().status(200).body("{}").unwrap() }); let error = conn - .drop_function(name, "fv_1") + .drop_function(name, "fv_1", &[]) .await .expect_err("a dot-only Function name must be refused"); assert!( @@ -4307,7 +4330,7 @@ mod tests { path => panic!("unexpected path: {path}"), }); let request = crate::function::FunctionRegistrationRequest::from_json(REQUEST).unwrap(); - let job = conn.create_function_async(request).await.unwrap(); + let job = conn.create_function_async(request, &[]).await.unwrap(); assert_eq!(job.id(), Some("job-function-1")); let version = job.wait().await.unwrap(); assert_eq!(version.name(), "embed"); @@ -4327,7 +4350,7 @@ mod tests { assert_eq!(body, serde_json::json!({"version": "1"})); http::Response::builder().status(200).body(VERSION).unwrap() }); - let version = conn.get_function("embed", "1").await.unwrap(); + let version = conn.get_function("embed", "1", &[]).await.unwrap(); assert_eq!(version.name(), "embed"); assert_eq!(version.version(), "1"); } @@ -4370,7 +4393,7 @@ mod tests { } } }); - let functions = conn.list_functions().await.unwrap(); + let functions = conn.list_functions(&[]).await.unwrap(); assert_eq!(functions.len(), 1); assert_eq!(functions[0].name(), "embed"); assert_eq!(functions[0].version(), "1"); @@ -4393,7 +4416,7 @@ mod tests { .unwrap() }); - let functions = conn.list_functions().await.unwrap(); + let functions = conn.list_functions(&[]).await.unwrap(); assert!(functions.is_empty()); assert_eq!(requests.load(Ordering::SeqCst), 1); } @@ -4434,7 +4457,7 @@ mod tests { .unwrap() }); - let error = conn.list_functions().await.unwrap_err(); + let error = conn.list_functions(&[]).await.unwrap_err(); assert!( matches!( &error, @@ -4461,7 +4484,82 @@ mod tests { .body(r#"{"dropped":false}"#) .unwrap() }); - assert!(!conn.drop_function("embed", "1").await.unwrap()); + assert!(!conn.drop_function("embed", "1", &[]).await.unwrap()); + } + + /// A Function's namespace is addressed in the path the way a Secret's is: + /// every route takes the joined identifier and no body carries the + /// namespace, so a namespaced request differs from a root one only in its + /// path. + #[tokio::test] + async fn test_a_function_namespace_path_is_addressed_in_the_path() { + const REQUEST: &str = include_str!( + "../../tests/fixtures/first_class_functions/v1/remote_function_registration_request.json" + ); + const VERSION: &str = include_str!( + "../../tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json" + ); + let namespace = ["analytics".to_string(), "features".to_string()]; + let mut expected_create: serde_json::Value = serde_json::from_str(REQUEST).unwrap(); + expected_create.as_object_mut().unwrap().remove("name"); + let conn = Connection::new_with_handler(move |request| { + let body = request + .body() + .and_then(|body| body.as_bytes()) + .map(|bytes| serde_json::from_slice::(bytes).unwrap()); + match request.url().path() { + "/v1/function/analytics$features$normalize_score/create" => { + assert_eq!(body.unwrap(), expected_create); + http::Response::builder() + .status(202) + .body(r#"{"job_id":"job-function-1"}"#.to_string()) + .unwrap() + } + "/v1/function/analytics$features$embed/describe" => { + assert_eq!(body.unwrap(), serde_json::json!({"version": "1"})); + http::Response::builder() + .status(200) + .body(VERSION.to_string()) + .unwrap() + } + "/v1/function/analytics$features$embed/drop" => { + assert_eq!(body.unwrap(), serde_json::json!({"version": "1"})); + http::Response::builder() + .status(200) + .body(r#"{"dropped":true}"#.to_string()) + .unwrap() + } + // Listing is namespace-scoped, so the namespace is the whole + // identifier. + "/v1/namespace/analytics$features/function/list" => http::Response::builder() + .status(200) + .body(r#"{"functions":[]}"#.to_string()) + .unwrap(), + path => panic!("unexpected path: {path}"), + } + }); + let request = crate::function::FunctionRegistrationRequest::from_json(REQUEST).unwrap(); + let job = conn + .create_function_async(request, &namespace) + .await + .unwrap(); + assert_eq!(job.id(), Some("job-function-1")); + conn.get_function("embed", "1", &namespace).await.unwrap(); + assert!(conn.list_functions(&namespace).await.unwrap().is_empty()); + assert!(conn.drop_function("embed", "1", &namespace).await.unwrap()); + } + + /// Each namespace segment is checked before a request is built, so an + /// empty segment cannot collapse the identifier onto the parent namespace. + #[tokio::test] + async fn test_an_unaddressable_function_namespace_segment_is_refused() { + let conn = Connection::new_with_handler(|request| -> http::Response { + panic!("reached the transport: {}", request.url().path()) + }); + let namespace = ["analytics".to_string(), String::new()]; + assert!(conn.get_function("embed", "1", &namespace).await.is_err()); + assert!(conn.list_functions(&namespace).await.is_err()); + assert!(conn.drop_function("embed", "1", &namespace).await.is_err()); } #[tokio::test] diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index d6abfea9a..5756d7788 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -96,6 +96,8 @@ const SCHEMA_SELECTOR_CHANGED: &str = "table selector changed while fetching sch fn fts_query_requires_document_granularity_support(query: &FtsQuery) -> bool { match query { + // Combined-fields queries do not expose a document granularity option. + FtsQuery::CombinedFields(_) => false, FtsQuery::Match(query) => query .document_granularity .is_some_and(|granularity| granularity.is_list_element()), @@ -2140,6 +2142,7 @@ impl BaseTable for RemoteTable { let definition = match response.query { Some(query) => MaterializedViewDefinition::from_sql(&query)?, None => MaterializedViewDefinition { + vector_source: None, source_table: response.source_table, source_namespace: response.source_namespace, lateral: None, @@ -3098,7 +3101,13 @@ impl BaseTable for RemoteTable { // loop can re-execute the plan (and re-stream the body) on each retry. // This mirrors the old `send_streaming(with_retry=true)` path, which // likewise buffered the reader to support retries. - let batches = new_data.collect::, _>>()?; + let schema = RecordBatchReader::schema(new_data.as_ref()); + let mut batches = new_data.collect::, _>>()?; + // An empty reader still carries a schema. Keep it in an empty batch so + // the buffered source remains scannable and can be replayed on retries. + if batches.is_empty() { + batches.push(RecordBatch::new_empty(schema)); + } let source: Box = Box::new(batches); let rescannable = source.rescannable(); let input: Arc = @@ -4685,6 +4694,25 @@ mod tests { assert_eq!(result.rows_updated, if old_server { 0 } else { 5 }); } + #[tokio::test] + async fn test_alter_columns_rejects_missing_changes_before_request() { + let table = Table::new_with_handler::("my_table", |request| { + panic!("Unexpected request: {}", request.url().path()) + }); + + for alterations in [ + vec![ColumnAlteration::new("id".into())], + vec![ + ColumnAlteration::new("id".into()).rename("new_id".into()), + ColumnAlteration::new("id".into()), + ], + ] { + let err = table.alter_columns(&alterations).await.unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. }), "got {err:?}"); + assert!(err.to_string().contains("path 'id'")); + } + } + #[rstest] #[case(true)] #[case(false)] @@ -4798,6 +4826,104 @@ mod tests { } } + #[rstest] + #[case::no_batches_insert(false, false)] + #[case::empty_batch_insert(true, false)] + #[case::no_batches_delete(false, true)] + #[case::empty_batch_delete(true, true)] + #[tokio::test] + async fn test_merge_insert_empty_source( + #[case] has_batch: bool, + #[case] delete_unmatched: bool, + ) { + let mut fields = vec![Field::new("id", DataType::Int64, false)]; + if !delete_unmatched { + fields.extend([ + Field::new("k", DataType::Int64, false), + Field::new( + "vector", + DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Float32, true)), + 2, + ), + true, + ), + Field::new("s", DataType::Utf8, true), + ]); + } + let schema = Arc::new(Schema::new_with_metadata( + fields, + HashMap::from([("source".to_string(), "empty".to_string())]), + )); + let batches = if has_batch { + vec![Ok(RecordBatch::new_empty(schema.clone()))] + } else { + vec![] + }; + let data: Box = + Box::new(RecordBatchIterator::new(batches, schema.clone())); + let attempts = Arc::new(AtomicUsize::new(0)); + let attempts_ref = attempts.clone(); + let num_deleted_rows = if delete_unmatched { 3 } else { 0 }; + + let table = Table::new_with_handler("my_table", move |request| { + assert_eq!(request.method(), "POST"); + assert_eq!(request.url().path(), "/v1/table/my_table/merge_insert/"); + assert_eq!(request.headers()[CONTENT_TYPE], ARROW_STREAM_CONTENT_TYPE); + let params = request.url().query_pairs().collect::>(); + assert_eq!(params["on"], "id"); + assert_eq!( + params["when_not_matched_insert_all"], + (!delete_unmatched).to_string() + ); + assert_eq!( + params["when_not_matched_by_source_delete"], + delete_unmatched.to_string() + ); + + let body = request.body().unwrap().as_bytes().unwrap(); + let reader = StreamReader::try_new(Cursor::new(body), None).unwrap(); + assert_eq!(reader.schema(), schema); + for batch in reader { + assert_eq!(batch.unwrap().num_rows(), 0); + } + + // The empty source must retain its schema when replayed after a conflict. + if attempts_ref.fetch_add(1, Ordering::SeqCst) == 0 { + http::Response::builder() + .status(409) + .body(String::new()) + .unwrap() + } else { + http::Response::builder() + .status(200) + .body( + json!({ + "version": 43, + "num_deleted_rows": num_deleted_rows, + "num_inserted_rows": 0, + "num_updated_rows": 0, + }) + .to_string(), + ) + .unwrap() + } + }); + + let mut merge = table.merge_insert(&["id"]); + if delete_unmatched { + merge.when_not_matched_by_source_delete(None); + } else { + merge.when_not_matched_insert_all(); + } + let result = merge.execute(data).await.unwrap(); + assert_eq!(attempts.load(Ordering::SeqCst), 2); + assert_eq!(result.version, 43); + assert_eq!(result.num_deleted_rows, num_deleted_rows); + assert_eq!(result.num_inserted_rows, 0); + assert_eq!(result.num_updated_rows, 0); + } + #[tokio::test] async fn test_merge_insert_composite_key() { let batch = RecordBatch::try_new( @@ -5186,6 +5312,193 @@ mod tests { assert_eq!(blobs.value(2), b"gamma"); } + #[tokio::test] + async fn test_fetch_blobs_splits_row_ids_at_one_version_and_preserves_order() { + let request_sizes = Arc::new(std::sync::Mutex::new(Vec::new())); + let seen = request_sizes.clone(); + let table = Table::new_with_handler_version( + "my_table", + semver::Version::new(0, 5, 0), + move |request| { + if request.url().path() == "/v1/table/my_table/describe/" { + return http::Response::builder() + .status(200) + .body(br#"{"version":7,"schema":{"fields":[]}}"#.to_vec()) + .unwrap(); + } + assert_eq!(request.url().path(), "/v1/table/my_table/fetch_blobs/"); + let body = request_body_json(&request); + assert_eq!(body["version"], 7); + let ids = body["row_ids"].as_array().unwrap(); + seen.lock().unwrap().push(ids.len()); + if ids.len() > 1024 { + return http::Response::builder() + .status(400) + .body(b"fetch_blobs accepts at most 1024 row IDs".to_vec()) + .unwrap(); + } + let mut builder = LargeBinaryBuilder::new(); + for id in ids { + let id = id.as_u64().unwrap(); + if id == 1023 { + builder.append_null(); + } else { + builder.append_value(id.to_string().as_bytes()); + } + } + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "image", + DataType::LargeBinary, + true, + )])), + vec![Arc::new(builder.finish())], + ) + .unwrap(); + http::Response::builder() + .status(200) + .header(CONTENT_TYPE, ARROW_STREAM_CONTENT_TYPE) + .body(write_ipc_stream_uncompressed(&batch)) + .unwrap() + }, + ); + + let ids: Vec = (0..1024).chain([42]).collect(); + let blobs = table.fetch_blobs("image", &ids).await.unwrap(); + assert_eq!(blobs.len(), 1025); + assert_eq!(blobs.value(0), b"0"); + assert_eq!(blobs.value(1022), b"1022"); + assert!(blobs.is_null(1023)); + assert_eq!(blobs.value(1024), b"42"); + assert_eq!(request_sizes.lock().unwrap().as_slice(), &[1024, 1]); + } + + #[tokio::test] + async fn test_fetch_blobs_splits_byte_limited_requests_and_reads_large_blob_by_range() { + // Simulate a lower byte cap so this test exercises the same 400 response + // without allocating 64 MiB of blob data. + let requests = Arc::new(std::sync::Mutex::new(Vec::new())); + let seen = requests.clone(); + let table = Table::new_with_handler_version( + "my_table", + semver::Version::new(0, 5, 0), + move |request| { + let path = request.url().path(); + if path == "/v1/table/my_table/describe/" { + return http::Response::builder() + .status(200) + .body(br#"{"version":42,"schema":{"fields":[]}}"#.to_vec()) + .unwrap(); + } + if path == "/v1/table/my_table/fetch_blobs/" { + let body = request_body_json(&request); + let ids = body["row_ids"].as_array().unwrap(); + if body["version"].is_null() { + // Only the initial failed request may read live latest. + assert_eq!(ids.len(), 5); + } else { + assert_eq!(body["version"], 42); + } + seen.lock().unwrap().push(format!("POST {}", ids.len())); + let mut builder = LargeBinaryBuilder::new(); + let mut total_bytes = 0; + for id in ids { + let value: Option<&[u8]> = match id.as_u64().unwrap() { + 10 => Some(b"aaaa"), + 20 => Some(b"bbb"), + 30 => None, + 40 => Some(b"0123456789"), + id => panic!("unexpected row id {id}"), + }; + if let Some(value) = value { + total_bytes += value.len(); + builder.append_value(value); + } else { + builder.append_null(); + } + } + if total_bytes > 6 { + return http::Response::builder() + .status(400) + .body(br#"{"error":"Bad request: fetch_blobs accepts at most 67108864 total blob bytes"}"#.to_vec()) + .unwrap(); + } + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "image", + DataType::LargeBinary, + true, + )])), + vec![Arc::new(builder.finish())], + ) + .unwrap(); + return http::Response::builder() + .status(200) + .header(CONTENT_TYPE, ARROW_STREAM_CONTENT_TYPE) + .body(write_ipc_stream_uncompressed(&batch)) + .unwrap(); + } + assert_eq!(path, "/v1/table/my_table/blob/image/40/bytes"); + assert!(request.url().query().unwrap().contains("version=42")); + let range = request + .headers() + .get(reqwest::header::RANGE) + .unwrap() + .to_str() + .unwrap(); + seen.lock().unwrap().push(format!("GET {range}")); + match range { + "bytes=0-0" => http::Response::builder() + .status(206) + .header(reqwest::header::CONTENT_RANGE, "bytes 0-0/10") + .header(VERSION_HEADER, "42") + .body(b"0".to_vec()) + .unwrap(), + "bytes=0-" => http::Response::builder() + .status(206) + .header(reqwest::header::CONTENT_RANGE, "bytes 0-9/10") + .body(b"0123456789".to_vec()) + .unwrap(), + _ => panic!("unexpected range: {range}"), + } + }, + ); + + let blobs = table + .fetch_blobs("image", &[10, 20, 30, 40, 20]) + .await + .unwrap(); + assert_eq!(blobs.len(), 5); + assert_eq!(blobs.value(0), b"aaaa"); + assert_eq!(blobs.value(1), b"bbb"); + assert!(blobs.is_null(2)); + assert_eq!(blobs.value(3), b"0123456789"); + assert_eq!(blobs.value(4), b"bbb"); + let requests = requests.lock().unwrap(); + assert!(requests.contains(&"GET bytes=0-0".to_string())); + assert!(requests.contains(&"GET bytes=0-".to_string())); + } + + #[tokio::test] + async fn test_fetch_blobs_does_not_split_unrelated_bad_requests() { + let requests = Arc::new(AtomicUsize::new(0)); + let seen = requests.clone(); + let table = + Table::new_with_handler_version("my_table", semver::Version::new(0, 5, 0), move |_| { + seen.fetch_add(1, Ordering::SeqCst); + http::Response::builder() + .status(400) + .body(b"unknown blob column".to_vec()) + .unwrap() + }); + + assert_fetch_blobs_http_error( + table.fetch_blobs("missing", &[10, 20]).await.unwrap_err(), + "unknown blob column", + ); + assert_eq!(requests.load(Ordering::SeqCst), 1); + } + fn table_with_fetch_blobs_response(body: Vec) -> Table { table_with_fetch_blobs_content_type(Some(ARROW_STREAM_CONTENT_TYPE), body) } @@ -5747,6 +6060,22 @@ mod tests { assert_eq!(data[0].as_ref().unwrap(), &expected_data); } + #[tokio::test] + async fn test_query_vector_nprobes_zero() { + let table = Table::new_with_handler::<&str>("my_table", |_| { + panic!("invalid nprobes must be rejected before sending a request") + }); + let result = table + .query() + .nearest_to(vec![0.1, 0.2, 0.3]) + .unwrap() + .nprobes(0); + assert!(matches!( + result, + Err(Error::InvalidInput { message }) if message == "nprobes must be greater than 0" + )); + } + #[tokio::test] async fn test_query_vector_all_params() { let table = Table::new_with_handler("my_table", |request| { @@ -5820,6 +6149,7 @@ mod tests { .postfilter() .distance_type(crate::DistanceType::Cosine) .nprobes(12) + .unwrap() .refine_factor(2) .bypass_vector_index() .execute() @@ -6151,6 +6481,53 @@ mod tests { .unwrap(); } + #[tokio::test] + async fn test_query_combined_fields_uses_structured_fts() { + use lance_index::scalar::inverted::query::CombinedFieldsQuery; + + let table = + Table::new_with_handler_version("my_table", semver::Version::new(0, 3, 0), |request| { + let body = request.body().unwrap().as_bytes().unwrap(); + let body: serde_json::Value = serde_json::from_slice(body).unwrap(); + assert_eq!( + body["full_text_query"]["query"], + serde_json::json!({ + "combined_fields": { + "query": "hello world", + "columns": ["title", "text"], + "boost": [1.0, 1.0], + "operator": "Or" + } + }) + ); + + let data = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])), + vec![Arc::new(Int32Array::from(vec![1]))], + ) + .unwrap(); + http::Response::builder() + .status(200) + .header(CONTENT_TYPE, ARROW_FILE_CONTENT_TYPE) + .body(write_ipc_file(&data)) + .unwrap() + }); + + table + .query() + .full_text_search(FullTextSearchQuery::new_query( + CombinedFieldsQuery::try_new( + "hello world".into(), + vec!["title".into(), "text".into()], + ) + .unwrap() + .into(), + )) + .execute() + .await + .unwrap(); + } + #[tokio::test] async fn test_query_row_document_granularity_uses_structured_fts() { let table = diff --git a/rust/lancedb/src/remote/table/blobs.rs b/rust/lancedb/src/remote/table/blobs.rs index 597b41782..ac37bf7ae 100644 --- a/rust/lancedb/src/remote/table/blobs.rs +++ b/rust/lancedb/src/remote/table/blobs.rs @@ -3,25 +3,46 @@ //! Cloud blob column listing, whole-byte fetch, and seekable HTTP byte-range handles. +use std::collections::HashMap; use std::ops::Range; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; -use arrow_array::{Array, LargeBinaryArray}; +use arrow_array::{Array, LargeBinaryArray, StructArray, UInt8Array, UInt64Array}; use arrow_schema::DataType; use bytes::{Bytes, BytesMut}; +use datafusion_expr::{col, lit}; use futures::{StreamExt, TryStreamExt}; +use lance::dataset::ROW_ID; +use lance_core::datatypes::BlobKind; use reqwest::{Response, StatusCode, header}; use tokio::sync::Mutex; use crate::Error; use crate::blob::BlobFile; use crate::error::Result; +use crate::query::{QueryFilter, QueryRequest, Select}; use crate::remote::client::{HttpSend, RequestResultExt, RestfulLanceDbClient}; -use crate::table::BaseTable; +use crate::table::{AnyQuery, BaseTable}; -use super::{FreshnessHeaders, FreshnessState, RemoteTable, freshness_headers_snapshot}; +use super::{ + FreshnessHeaders, FreshnessState, ReadSnapshot, RemoteTable, VERSION_HEADER, + freshness_headers_snapshot, +}; + +// The Cloud route rejects larger row-id lists. Its separate 64 MiB byte limit +// is handled by splitting a rejected request below. +const MAX_FETCH_BLOBS_ROW_IDS: usize = 1024; + +fn is_fetch_blobs_byte_limit_error(error: &Error) -> bool { + matches!(error, Error::Http { + source, + status_code: Some(StatusCode::BAD_REQUEST), + .. + } if source.to_string().contains("fetch_blobs accepts at most") + && source.to_string().contains("total blob bytes")) +} #[derive(Debug, Clone, Copy)] enum RangeRequestMode { @@ -35,6 +56,7 @@ trait BlobRangeRequester: Send + Sync + std::fmt::Debug { &self, range_header: &str, mode: RangeRequestMode, + version: Option, ) -> Result<(String, Response)>; } @@ -56,13 +78,14 @@ impl BlobRangeRequester for TableBlobRangeRequester { &self, range_header: &str, mode: RangeRequestMode, + version: Option, ) -> Result<(String, Response)> { let freshness_request = freshness_headers_snapshot(&self.freshness, self.read_consistency_interval); let mut request = freshness_request .apply(self.client.get(&self.path)) .header(header::RANGE, range_header); - if let Some(version) = self.version { + if let Some(version) = version.or(self.version) { request = request.query(&[("version", version)]); } if let Some(branch) = &self.branch { @@ -97,21 +120,25 @@ struct RemoteBlobState { } /// Seekable Cloud blob handle over HTTP Range. +/// +/// Nonempty handles read the table version returned by their size probe. #[derive(Debug)] pub struct RemoteBlobFile { requester: Arc, state: Mutex, closed: AtomicBool, size: u64, + version: Option, } impl RemoteBlobFile { - fn new(requester: Arc, size: u64) -> Self { + fn new(requester: Arc, size: u64, version: Option) -> Self { Self { requester, state: Mutex::new(RemoteBlobState::default()), closed: AtomicBool::new(false), size, + version, } } @@ -160,7 +187,7 @@ impl RemoteBlobFile { let range_header = format!("bytes={}-{}", range.start, range.end - 1); let (request_id, response) = self .requester - .request_range(&range_header, RangeRequestMode::DataRead) + .request_range(&range_header, RangeRequestMode::DataRead, self.version) .await .map_err(remote_blob_error)?; self.ensure_open()?; @@ -236,7 +263,7 @@ impl RemoteBlobFile { let range_header = format!("bytes={cursor}-"); let (request_id, response) = self .requester - .request_range(&range_header, RangeRequestMode::DataRead) + .request_range(&range_header, RangeRequestMode::DataRead, self.version) .await .map_err(remote_blob_error)?; self.ensure_open()?; @@ -369,7 +396,62 @@ impl RemoteTable { message: "fetch_blobs is not supported on this LanceDB Cloud server".into(), }); } - let read_snapshot = self.snapshot_read_state().await; + let mut read_snapshot = self.snapshot_read_state().await; + // A selection spanning requests must use one exact dataset version. + // Resolve latest before the first chunk; a checked-out version is exact already. + if row_ids.len() > MAX_FETCH_BLOBS_ROW_IDS && read_snapshot.version.is_none() { + read_snapshot.version = Some(self.describe_read_snapshot(read_snapshot).await?.version); + } + let mut pending: Vec<&[u64]> = row_ids.chunks(MAX_FETCH_BLOBS_ROW_IDS).rev().collect(); + let mut chunks = Vec::new(); + while let Some(ids) = pending.pop() { + match self.fetch_blobs_chunk(column, ids, read_snapshot).await { + Ok(blobs) => chunks.push(blobs), + Err(error) if is_fetch_blobs_byte_limit_error(&error) => { + // A single-request call may turn into several requests after a + // byte-cap error. No bytes from the failed request were used. + if read_snapshot.version.is_none() { + read_snapshot.version = + Some(self.describe_read_snapshot(read_snapshot).await?.version); + } + if ids.len() == 1 { + // The whole-byte route cannot serve this blob. The Range route + // has no aggregate response limit and preserves null alignment. + let mut files = self + .fetch_blob_files_with_snapshot(column, ids, read_snapshot) + .await?; + let blob = match files.pop().unwrap() { + Some(file) => Some(file.read().await?), + None => None, + }; + chunks.push(LargeBinaryArray::from(vec![blob.as_deref()])); + } else { + let mid = ids.len() / 2; + pending.push(&ids[mid..]); + pending.push(&ids[..mid]); + } + } + Err(error) => return Err(error), + } + } + + if chunks.len() == 1 { + return Ok(chunks.pop().unwrap()); + } + let chunk_refs: Vec<&dyn Array> = chunks.iter().map(|chunk| chunk as &dyn Array).collect(); + Ok(arrow::compute::concat(&chunk_refs)? + .as_any() + .downcast_ref::() + .expect("concatenating LargeBinary arrays returns LargeBinary") + .clone()) + } + + async fn fetch_blobs_chunk( + &self, + column: &str, + row_ids: &[u64], + read_snapshot: ReadSnapshot, + ) -> Result { let mut body = serde_json::json!({ "version": read_snapshot.version, "column": column, @@ -443,7 +525,11 @@ impl RemoteTable { /// Open seekable handles for `row_ids`. /// - /// Each non-null row is probed once to determine its size. + /// Descriptors for the requested rows come from one `_rowid` take query per + /// [`MAX_FETCH_BLOBS_ROW_IDS`] unique row ids, all read at one exact table + /// version. Each handle is sized from its descriptor and reads that + /// version. Only external blobs whose descriptor records no size are + /// probed, because their length is the whole object's. pub(super) async fn fetch_blob_files_impl( &self, column: &str, @@ -453,36 +539,251 @@ impl RemoteTable { if row_ids.is_empty() { return Ok(Vec::new()); } - if !self.server_version.support_blobs() { - return Err(Error::NotSupported { - message: "fetch_blob_files requires LanceDB Cloud server 0.5.0 or newer".into(), - }); - } + self.ensure_blob_files_supported()?; - let read_snapshot = self.snapshot_read_state().await; - let encoded_column = urlencoding::encode(column); + let mut read_snapshot = self.snapshot_read_state().await; + let sizes = self + .take_blob_descriptor_sizes(column, row_ids, &mut read_snapshot) + .await?; + let version = read_snapshot.version; + + let mut files = Vec::with_capacity(row_ids.len()); + let mut probe_indices = Vec::new(); + let mut probe_requesters = Vec::new(); + for (index, row_id) in row_ids.iter().enumerate() { + match sizes[row_id] { + DescriptorSize::Null => files.push(None), + DescriptorSize::Known(size) => { + let requester = self.blob_range_requester(column, *row_id, &read_snapshot); + files.push(Some(RemoteBlobFile::new(requester, size, version).into())); + } + DescriptorSize::Unresolved => { + files.push(None); + probe_indices.push(index); + probe_requesters.push(self.blob_range_requester( + column, + *row_id, + &read_snapshot, + )); + } + } + } + let probed = probe_blob_files(probe_requesters).await?; + for (index, file) in probe_indices.into_iter().zip(probed) { + files[index] = file; + } + Ok(files) + } + + /// Probe each row once for its size. + /// + /// Serves callers that must not issue a query, such as a single blob that + /// the whole-byte route could not return. + async fn fetch_blob_files_with_snapshot( + &self, + column: &str, + row_ids: &[u64], + read_snapshot: ReadSnapshot, + ) -> Result>> { let requesters = row_ids .iter() - .map(|row_id| { - let path = format!( - "/v1/table/{}/blob/{encoded_column}/{row_id}/bytes", - self.identifier - ); - let requester: Arc = Arc::new(TableBlobRangeRequester { - client: self.client.clone(), - path, - version: read_snapshot.version, - branch: self.branch.clone(), - freshness: Arc::new(std::sync::Mutex::new(read_snapshot.freshness_state)), - parent_freshness: self.freshness.clone(), - parent_freshness_request: read_snapshot.freshness, - read_consistency_interval: self.client.read_consistency_interval, - }); - requester - }) + .map(|row_id| self.blob_range_requester(column, *row_id, &read_snapshot)) .collect(); probe_blob_files(requesters).await } + + /// Read the blob descriptor of every unique row in `row_ids`. + /// + /// Pins `read_snapshot` to the version the first take read, so later chunks + /// and the returned handles all see one table version. + async fn take_blob_descriptor_sizes( + &self, + column: &str, + row_ids: &[u64], + read_snapshot: &mut ReadSnapshot, + ) -> Result> { + let mut unique_row_ids = row_ids.to_vec(); + unique_row_ids.sort_unstable(); + unique_row_ids.dedup(); + + let mut sizes = HashMap::with_capacity(unique_row_ids.len()); + for chunk in unique_row_ids.chunks(MAX_FETCH_BLOBS_ROW_IDS) { + let (mut chunk_sizes, response_version) = self + .take_blob_descriptor_chunk(column, chunk, *read_snapshot) + .await?; + if read_snapshot.version.is_none() { + match response_version { + Some(version) => read_snapshot.version = Some(version), + None => { + // A server that does not report its read version may have read + // an older version than a describe now returns, so the take is + // repeated at the described version. + read_snapshot.version = + Some(self.describe_read_snapshot(*read_snapshot).await?.version); + (chunk_sizes, _) = self + .take_blob_descriptor_chunk(column, chunk, *read_snapshot) + .await?; + } + } + } + sizes.extend(chunk_sizes); + } + let found = unique_row_ids + .iter() + .filter(|row_id| sizes.contains_key(row_id)) + .count(); + if found < unique_row_ids.len() { + return Err(Error::InvalidInput { + message: format!( + "blob read for column '{column}' requested {} row ids but only {found} exist \ + in the table; pass row ids collected from this table", + unique_row_ids.len() + ), + }); + } + Ok(sizes) + } + + /// Take one chunk of descriptors, also returning the version the server + /// reported reading. + async fn take_blob_descriptor_chunk( + &self, + column: &str, + row_ids: &[u64], + read_snapshot: ReadSnapshot, + ) -> Result<(HashMap, Option)> { + let query = AnyQuery::Query(QueryRequest { + filter: Some(QueryFilter::Datafusion( + col(ROW_ID).in_list(row_ids.iter().map(|id| lit(*id)).collect(), false), + )), + select: Select::columns(&[column]), + with_row_id: true, + // Row ids name base-table rows, and the MemWAL scanner has no stable `_rowid`. + use_lsm: Some(false), + ..Default::default() + }); + let body = self + .prepare_query_bodies(&query, read_snapshot.version)? + .pop() + .expect("a plain query has one body"); + let request = self + .client + .post(&format!("/v1/table/{}/query/", self.identifier)) + .json(&body); + let (request_id, response) = self + .send_with_freshness(request, true, read_snapshot.freshness) + .await?; + let response_version = response + .headers() + .get(&VERSION_HEADER) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse().ok()); + let mut stream = self.read_arrow_response(&request_id, response).await?; + + let mut sizes = HashMap::with_capacity(row_ids.len()); + let invalid_response = |message: String| Error::Http { + source: message.into(), + request_id: request_id.clone(), + status_code: None, + }; + while let Some(batch) = stream.try_next().await? { + let taken_row_ids = batch + .column_by_name(ROW_ID) + .and_then(|array| array.as_any().downcast_ref::()) + .ok_or_else(|| { + invalid_response(format!("blob descriptor take is missing UInt64 '{ROW_ID}'")) + })?; + let values = batch.column_by_name(column).ok_or_else(|| { + invalid_response(format!("blob descriptor take is missing '{column}'")) + })?; + // Only blob v2 columns read back as descriptors. + let descriptors = values + .as_any() + .downcast_ref::() + .and_then(BlobDescriptorSizes::try_new) + .ok_or_else(|| Error::InvalidInput { + message: format!("column '{column}' is not a blob column"), + })?; + for (index, row_id) in taken_row_ids.values().iter().enumerate() { + sizes.insert(*row_id, descriptors.size(index)?); + } + } + Ok((sizes, response_version)) + } + + fn ensure_blob_files_supported(&self) -> Result<()> { + if self.server_version.support_blobs() { + Ok(()) + } else { + Err(Error::NotSupported { + message: "fetch_blob_files requires LanceDB Cloud server 0.5.0 or newer".into(), + }) + } + } + + fn blob_range_requester( + &self, + column: &str, + row_id: u64, + read_snapshot: &ReadSnapshot, + ) -> Arc { + let path = format!( + "/v1/table/{}/blob/{}/{row_id}/bytes", + self.identifier, + urlencoding::encode(column) + ); + Arc::new(TableBlobRangeRequester { + client: self.client.clone(), + path, + version: read_snapshot.version, + branch: self.branch.clone(), + freshness: Arc::new(std::sync::Mutex::new(read_snapshot.freshness_state)), + parent_freshness: self.freshness.clone(), + parent_freshness_request: read_snapshot.freshness, + read_consistency_interval: self.client.read_consistency_interval, + }) + } +} + +/// What a blob v2 descriptor says about a row's size. +#[derive(Debug, Clone, Copy)] +enum DescriptorSize { + Null, + Known(u64), + /// An external blob that recorded no size. Its length is the whole + /// object's, which only the server can resolve. + Unresolved, +} + +/// The `kind` and `size` children of a blob v2 descriptor array. +struct BlobDescriptorSizes<'a> { + descriptors: &'a StructArray, + kinds: &'a UInt8Array, + sizes: &'a UInt64Array, +} + +impl<'a> BlobDescriptorSizes<'a> { + fn try_new(descriptors: &'a StructArray) -> Option { + let child = |name: &str| descriptors.column_by_name(name); + Some(Self { + descriptors, + kinds: child("kind")?.as_any().downcast_ref::()?, + sizes: child("size")?.as_any().downcast_ref::()?, + }) + } + + fn size(&self, index: usize) -> Result { + if self.descriptors.is_null(index) || self.kinds.is_null(index) { + return Ok(DescriptorSize::Null); + } + let kind = BlobKind::try_from(self.kinds.value(index))?; + let size = self.sizes.value(index); + Ok(if kind == BlobKind::External && size == 0 { + DescriptorSize::Unresolved + } else { + DescriptorSize::Known(size) + }) + } } /// Probe blob sizes while preserving row order. @@ -499,12 +800,25 @@ async fn probe_blob_files( const BLOB_REQUEST_CONCURRENCY: usize = 8; +fn probe_blob_version(response: &Response, request_id: &str) -> Result { + response + .headers() + .get(&VERSION_HEADER) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse().ok()) + .ok_or_else(|| Error::Http { + source: "blob size probe returned a missing or invalid x-lancedb-version header".into(), + request_id: request_id.to_string(), + status_code: Some(response.status()), + }) +} + /// Probe one blob's size. /// /// `204` represents null. `416` with `bytes */0` represents an empty blob. async fn probe_blob_file(requester: Arc) -> Result> { let (request_id, response) = requester - .request_range("bytes=0-0", RangeRequestMode::SizeProbe) + .request_range("bytes=0-0", RangeRequestMode::SizeProbe, None) .await?; match response.status() { StatusCode::NO_CONTENT => { @@ -537,7 +851,8 @@ async fn probe_blob_file(requester: Arc) -> Result { let size = response @@ -553,6 +868,7 @@ async fn probe_blob_file(requester: Arc) -> Result) -> Result Err(Error::Http { source: format!("blob size probe expected HTTP 206 Partial Content, got {status}") @@ -578,13 +896,15 @@ async fn probe_blob_file(requester: Arc) -> Result]) -> StructArray { + let kinds = rows + .iter() + .map(|row| row.map_or(0, |(kind, _)| kind as u8)) + .collect::(); + let sizes = rows + .iter() + .map(|row| row.map_or(0, |(_, size)| size)) + .collect::(); + let len = rows.len(); + StructArray::new( + lance_core::datatypes::BLOB_V2_DESC_FIELDS.clone(), + vec![ + Arc::new(kinds), + Arc::new(UInt64Array::from(vec![0; len])), + Arc::new(sizes), + Arc::new(arrow_array::UInt32Array::from(vec![0; len])), + Arc::new(arrow_array::StringArray::from(vec![""; len])), + ], + Some(rows.iter().map(Option::is_some).collect()), + ) + } + + impl RemoteTable { + /// Open handles through the per-row size probe alone. + async fn probe_files( + &self, + column: &str, + row_ids: &[u64], + ) -> Result>> { + let read_snapshot = self.snapshot_read_state().await; + self.fetch_blob_files_with_snapshot(column, row_ids, read_snapshot) + .await + } + } + + /// A request the take mock received. + #[derive(Debug, Clone, PartialEq)] + enum TakeMockRequest { + Query { + row_ids: Vec, + version: Option, + }, + Describe, + Range { + row_id: u64, + range: String, + version: Option, + }, + } + + /// Row ids named by a `_rowid IN (...)` filter. + fn filter_row_ids(filter: &str) -> Vec { + let list = filter.split_once('(').unwrap().1.trim_end_matches(')'); + list.split(',') + .map(|id| id.trim().parse().unwrap()) + .collect() + } + + /// A table whose query route answers `_rowid` takes from `rows` and whose + /// byte routes serve [`PAYLOAD`]. Takes return rows in reverse order, and + /// report `query_version` in `x-lancedb-version` when it is set. + fn mock_take_blob_table( + rows: Vec<(u64, Option<(BlobKind, u64)>)>, + query_version: Option<&'static str>, + ) -> ( + RemoteTable, + Arc>>, + ) { + let requests = Arc::new(StdMutex::new(Vec::new())); + let captured = requests.clone(); + let table = RemoteTable::new_mock( + "my_table".to_string(), + move |request| { + let path = request.url().path().to_string(); + if path == "/v1/table/my_table/query/" { + let body: serde_json::Value = + serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()) + .unwrap(); + assert_eq!(body["columns"], serde_json::json!(["image"])); + assert_eq!(body["with_row_id"], serde_json::json!(true)); + assert_eq!(body["use_lsm"], serde_json::json!(false)); + let requested = filter_row_ids(body["filter"].as_str().unwrap()); + captured.lock().unwrap().push(TakeMockRequest::Query { + row_ids: requested.clone(), + version: body["version"].as_u64(), + }); + let taken = rows + .iter() + .rev() + .filter(|(row_id, _)| requested.contains(row_id)) + .collect::>(); + let batch = RecordBatch::try_from_iter([ + ( + "image", + Arc::new(descriptors( + &taken.iter().map(|(_, row)| *row).collect::>(), + )) as Arc, + ), + ( + ROW_ID, + Arc::new(UInt64Array::from_iter_values( + taken.iter().map(|(row_id, _)| *row_id), + )) as Arc, + ), + ]) + .unwrap(); + let mut body = Vec::new(); + let mut writer = + arrow_ipc::writer::FileWriter::try_new(&mut body, &batch.schema()).unwrap(); + writer.write(&batch).unwrap(); + writer.finish().unwrap(); + drop(writer); + let mut response = http::Response::builder().status(200); + if let Some(version) = query_version { + response = response.header(VERSION_HEADER, version); + } + return response.body(body).unwrap(); + } + if path == "/v1/table/my_table/describe/" { + captured.lock().unwrap().push(TakeMockRequest::Describe); + return http::Response::builder() + .status(200) + .body(r#"{"version":9,"schema":{"fields":[]}}"#.as_bytes().to_vec()) + .unwrap(); + } + let row_id = path + .strip_prefix("/v1/table/my_table/blob/image/") + .and_then(|rest| rest.strip_suffix("/bytes")) + .unwrap_or_else(|| panic!("unexpected path: {path}")) + .parse() + .unwrap(); + captured.lock().unwrap().push(TakeMockRequest::Range { + row_id, + range: request.headers()[header::RANGE] + .to_str() + .unwrap() + .to_string(), + version: request + .url() + .query_pairs() + .find(|(name, _)| name == "version") + .map(|(_, value)| value.into_owned()), + }); + range_response(&request, PAYLOAD) + }, + Some(Version::new(0, 5, 0)), + ); + (table, requests) + } + + #[tokio::test] + async fn remote_blob_files_are_sized_from_one_descriptor_take() { + let payload_size = PAYLOAD.len() as u64; + let (table, requests) = mock_take_blob_table( + vec![ + (10, Some((BlobKind::Inline, payload_size))), + (20, None), + (30, Some((BlobKind::Inline, 0))), + (40, Some((BlobKind::Packed, payload_size))), + (50, Some((BlobKind::Dedicated, payload_size))), + (60, Some((BlobKind::External, 0))), + ], + Some("7"), + ); + + // Reordered and duplicated row ids come back in request order. + let files = table + .fetch_blob_files_impl("image", &[60, 10, 20, 30, 40, 50, 10]) + .await + .unwrap(); + + let sizes = files + .iter() + .map(|file| file.as_ref().map(BlobFile::size)) + .collect::>(); + assert_eq!( + sizes, + [ + Some(payload_size), + Some(payload_size), + None, + Some(0), + Some(payload_size), + Some(payload_size), + Some(payload_size), + ] + ); + // One take, then a probe for the external blob that recorded no size. + assert_eq!( + requests.lock().unwrap().as_slice(), + [ + TakeMockRequest::Query { + row_ids: vec![10, 20, 30, 40, 50, 60], + version: None, + }, + TakeMockRequest::Range { + row_id: 60, + range: "bytes=0-0".into(), + version: Some("7".into()), + }, + ] + ); + + // Sized handles read the version the take read. + assert_eq!( + files[1].as_ref().unwrap().read_range(5..12).await.unwrap(), + &PAYLOAD[5..12] + ); + assert!(files[3].as_ref().unwrap().read().await.unwrap().is_empty()); + assert_eq!( + requests.lock().unwrap().last().unwrap(), + &TakeMockRequest::Range { + row_id: 10, + range: "bytes=5-11".into(), + version: Some("7".into()), + } + ); + } + + #[tokio::test] + async fn remote_blob_files_take_every_chunk_at_the_first_version() { + let row_ids = (0..MAX_FETCH_BLOBS_ROW_IDS as u64 + 1).collect::>(); + let (table, requests) = mock_take_blob_table( + row_ids.iter().map(|row_id| (*row_id, None)).collect(), + Some("7"), + ); + + let files = table + .fetch_blob_files_impl("image", &row_ids) + .await + .unwrap(); + + assert_eq!(files.len(), row_ids.len()); + let versions = requests + .lock() + .unwrap() + .iter() + .map(|request| match request { + TakeMockRequest::Query { row_ids, version } => (row_ids.len(), *version), + other => panic!("unexpected request: {other:?}"), + }) + .collect::>(); + assert_eq!(versions, [(MAX_FETCH_BLOBS_ROW_IDS, None), (1, Some(7))]); + } + + #[tokio::test] + async fn remote_blob_files_retake_at_the_described_version_when_a_take_does_not_report_it() { + let payload_size = PAYLOAD.len() as u64; + let (table, requests) = + mock_take_blob_table(vec![(10, Some((BlobKind::Inline, payload_size)))], None); + + let file = table + .fetch_blob_files_impl("image", &[10]) + .await + .unwrap() + .pop() + .flatten() + .unwrap(); + file.read_range(0..4).await.unwrap(); + + // The unversioned take is repeated at the described version. + assert_eq!( + requests.lock().unwrap().as_slice(), + [ + TakeMockRequest::Query { + row_ids: vec![10], + version: None, + }, + TakeMockRequest::Describe, + TakeMockRequest::Query { + row_ids: vec![10], + version: Some(9), + }, + TakeMockRequest::Range { + row_id: 10, + range: "bytes=0-3".into(), + version: Some("9".into()), + }, + ] + ); + } + + #[tokio::test] + async fn remote_blob_files_take_at_the_checked_out_version() { + let payload_size = PAYLOAD.len() as u64; + let (table, requests) = mock_take_blob_table( + vec![(10, Some((BlobKind::Inline, payload_size)))], + Some("7"), + ); + table.checkout(9).await.unwrap(); + requests.lock().unwrap().clear(); + + table.fetch_blob_files_impl("image", &[10]).await.unwrap(); + + assert_eq!( + requests.lock().unwrap().as_slice(), + [TakeMockRequest::Query { + row_ids: vec![10], + version: Some(9), + }] + ); + } + + #[tokio::test] + async fn remote_blob_files_reject_row_ids_missing_from_the_take() { + let (table, requests) = mock_take_blob_table(vec![(10, None)], Some("7")); + + let error = table + .fetch_blob_files_impl("image", &[10, 99]) + .await + .unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. }), "{error}"); + assert!(error.to_string().contains("only 1 exist"), "{error}"); + assert_eq!(requests.lock().unwrap().len(), 1); + } + + #[tokio::test] + async fn remote_blob_files_reject_a_non_blob_column() { + let table = RemoteTable::new_mock( + "my_table".to_string(), + |request| { + assert_eq!(request.url().path(), "/v1/table/my_table/query/"); + let batch = RecordBatch::try_from_iter([ + ( + "id", + Arc::new(arrow_array::Int64Array::from(vec![1])) as Arc, + ), + ( + ROW_ID, + Arc::new(UInt64Array::from(vec![10])) as Arc, + ), + ]) + .unwrap(); + let mut body = Vec::new(); + let mut writer = + arrow_ipc::writer::FileWriter::try_new(&mut body, &batch.schema()).unwrap(); + writer.write(&batch).unwrap(); + writer.finish().unwrap(); + drop(writer); + http::Response::builder().status(200).body(body).unwrap() + }, + Some(Version::new(0, 5, 0)), + ); + + let error = table.fetch_blob_files_impl("id", &[10]).await.unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. }), "{error}"); + assert!( + error.to_string().contains("'id' is not a blob column"), + "{error}" + ); + } + #[tokio::test] async fn remote_blob_file_reads_the_requested_range() { let requests = Arc::new(StdMutex::new(Vec::new())); let table = mock_remote_blob_table(requests.clone()); let file = table - .fetch_blob_files_impl("image", &[10]) + .probe_files("image", &[10]) .await .unwrap() .pop() @@ -698,6 +1372,68 @@ mod tests { assert!(requests.lock().unwrap().contains(&"bytes=5-11".to_string())); } + #[tokio::test] + async fn remote_blob_file_reads_the_probed_version_after_table_changes() { + let changed = Arc::new(AtomicBool::new(false)); + let requests = Arc::new(StdMutex::new(Vec::new())); + let table = RemoteTable::new_mock( + "my_table".to_string(), + { + let changed = changed.clone(); + let requests = requests.clone(); + move |request| { + let range = request + .headers() + .get(header::RANGE) + .unwrap() + .to_str() + .unwrap(); + let version = request + .url() + .query_pairs() + .find(|(name, _)| name == "version") + .map(|(_, value)| value.into_owned()); + requests + .lock() + .unwrap() + .push((range.to_string(), version.clone())); + if changed.load(Ordering::SeqCst) && version.as_deref() != Some("5") { + return http::Response::builder() + .status(StatusCode::BAD_REQUEST) + .body(b"row id is absent from the latest version".to_vec()) + .unwrap(); + } + range_response(&request, PAYLOAD) + } + }, + Some(Version::new(0, 5, 0)), + ); + + let file = table + .probe_files("image", &[10]) + .await + .unwrap() + .pop() + .flatten() + .unwrap(); + changed.store(true, Ordering::SeqCst); + + assert_eq!(file.read_range(5..9).await.unwrap(), &PAYLOAD[5..9]); + assert_eq!(file.read_up_to(4).await.unwrap(), &PAYLOAD[..4]); + file.seek(10).await.unwrap(); + assert_eq!(file.read().await.unwrap(), &PAYLOAD[10..]); + + assert_eq!( + requests.lock().unwrap().as_slice(), + [ + ("bytes=0-0".to_string(), None), + ("bytes=5-8".to_string(), Some("5".to_string())), + ("bytes=0-".to_string(), Some("5".to_string())), + ("bytes=10-".to_string(), Some("5".to_string())), + ] + ); + } + #[tokio::test] async fn remote_blob_file_keeps_the_open_timeline_after_parent_checkout() { let range_requests = Arc::new(StdMutex::new(Vec::new())); @@ -723,7 +1459,7 @@ mod tests { table.checkout(5).await.unwrap(); let file = table - .fetch_blob_files_impl("image", &[10]) + .probe_files("image", &[10]) .await .unwrap() .pop() @@ -743,7 +1479,7 @@ mod tests { let requests = Arc::new(StdMutex::new(Vec::new())); let table = mock_remote_blob_table(requests.clone()); let file = table - .fetch_blob_files_impl("image", &[10]) + .probe_files("image", &[10]) .await .unwrap() .pop() @@ -762,21 +1498,20 @@ mod tests { } #[tokio::test] - async fn remote_blob_files_return_a_handle_for_an_empty_blob() { + async fn remote_blob_files_allow_an_empty_blob_without_a_version_header() { let requests = Arc::new(StdMutex::new(Vec::new())); let table = mock_remote_blob_table(requests.clone()); - let mut files = table - .fetch_blob_files_impl("image", &[10, 20, 30]) - .await - .unwrap(); + let mut files = table.probe_files("image", &[10, 20, 30]).await.unwrap(); assert_eq!(files.len(), 3); assert!(files[1].is_none()); let empty = files[2].take().unwrap(); assert_eq!(empty.size(), 0); + let probe_requests = requests.lock().unwrap().len(); assert!(empty.read_range(0..0).await.unwrap().is_empty()); assert!(empty.read().await.unwrap().is_empty()); + assert_eq!(requests.lock().unwrap().len(), probe_requests); let nonempty = files[0].take().unwrap(); assert_eq!(nonempty.size(), PAYLOAD.len() as u64); } @@ -795,10 +1530,7 @@ mod tests { Some(Version::new(0, 5, 0)), ); - let error = table - .fetch_blob_files_impl("image", &[10]) - .await - .unwrap_err(); + let error = table.probe_files("image", &[10]).await.unwrap_err(); assert!( error .to_string() @@ -822,10 +1554,7 @@ mod tests { Some(Version::new(0, 5, 0)), ); - let error = table - .fetch_blob_files_impl("image", &[10]) - .await - .unwrap_err(); + let error = table.probe_files("image", &[10]).await.unwrap_err(); assert!( error .to_string() @@ -834,6 +1563,27 @@ mod tests { ); } + #[tokio::test] + async fn remote_blob_files_reject_a_probe_without_a_version() { + let table = RemoteTable::new_mock( + "my_table".to_string(), + |request| { + let mut response = range_response(&request, PAYLOAD); + response.headers_mut().remove(VERSION_HEADER); + response + }, + Some(Version::new(0, 5, 0)), + ); + + let error = table.probe_files("image", &[10]).await.unwrap_err(); + assert!( + error + .to_string() + .contains("missing or invalid x-lancedb-version"), + "got: {error}" + ); + } + #[tokio::test] async fn remote_blob_files_reject_unsupported_server_version() { let table = RemoteTable::new_mock( @@ -883,7 +1633,7 @@ mod tests { Some(Version::new(0, 5, 0)), ); let file = table - .fetch_blob_files_impl("image", &[10]) + .probe_files("image", &[10]) .await .unwrap() .pop() @@ -923,7 +1673,7 @@ mod tests { Some(Version::new(0, 5, 0)), ); let file = table - .fetch_blob_files_impl("image", &[10]) + .probe_files("image", &[10]) .await .unwrap() .pop() @@ -974,7 +1724,7 @@ mod tests { Some(Version::new(0, 5, 0)), ); let file = table - .fetch_blob_files_impl("image", &[10]) + .probe_files("image", &[10]) .await .unwrap() .pop() @@ -1001,7 +1751,7 @@ mod tests { let requests = Arc::new(StdMutex::new(Vec::new())); let table = mock_remote_blob_table(requests.clone()); let file = table - .fetch_blob_files_impl("image", &[10]) + .probe_files("image", &[10]) .await .unwrap() .pop() @@ -1025,6 +1775,7 @@ mod tests { &self, range_header: &str, _mode: RangeRequestMode, + _version: Option, ) -> Result<(String, Response)> { assert_eq!(range_header, "bytes=0-0"); let now = self.in_flight.fetch_add(1, Ordering::SeqCst) + 1; @@ -1039,6 +1790,7 @@ mod tests { header::CONTENT_RANGE, format!("bytes 0-0/{}", 100 + self.index), ) + .header(VERSION_HEADER, "5") .body(vec![0u8]) .unwrap(); Ok((format!("probe-{}", self.index), Response::from(response))) @@ -1078,7 +1830,7 @@ mod tests { let requests = Arc::new(StdMutex::new(Vec::new())); let table = mock_remote_blob_table(requests.clone()); - let mut files = table.fetch_blob_files_impl("image", &[10]).await.unwrap(); + let mut files = table.probe_files("image", &[10]).await.unwrap(); let file = files.remove(0).unwrap(); assert_eq!(file.size(), PAYLOAD.len() as u64); @@ -1093,7 +1845,7 @@ mod tests { let requests = Arc::new(StdMutex::new(Vec::new())); let table = mock_remote_blob_table(requests.clone()); - let mut files = table.fetch_blob_files_impl("image", &[10]).await.unwrap(); + let mut files = table.probe_files("image", &[10]).await.unwrap(); let file = files.remove(0).unwrap(); let probe_requests = requests.lock().unwrap().len(); @@ -1118,7 +1870,7 @@ mod tests { let requests = Arc::new(StdMutex::new(Vec::new())); let table = mock_remote_blob_table(requests.clone()); - let mut files = table.fetch_blob_files_impl("image", &[10]).await.unwrap(); + let mut files = table.probe_files("image", &[10]).await.unwrap(); let file = files.remove(0).unwrap(); let probe_requests = requests.lock().unwrap().len(); @@ -1157,6 +1909,7 @@ mod tests { header::CONTENT_RANGE, format!("bytes 0-0/{}", PAYLOAD.len()), ) + .header(VERSION_HEADER, "5") .body(vec![PAYLOAD[0]]) .unwrap(); } @@ -1169,7 +1922,7 @@ mod tests { Some(Version::new(0, 5, 0)), ); - let mut files = table.fetch_blob_files_impl("image", &[10]).await.unwrap(); + let mut files = table.probe_files("image", &[10]).await.unwrap(); let file = files.remove(0).unwrap(); let error = file.read_range(1..3).await.unwrap_err(); @@ -1203,6 +1956,7 @@ mod tests { &self, range_header: &str, _mode: RangeRequestMode, + _version: Option, ) -> Result<(String, Response)> { let now = self.in_flight.fetch_add(1, Ordering::SeqCst) + 1; self.max_in_flight.fetch_max(now, Ordering::SeqCst); @@ -1235,7 +1989,7 @@ mod tests { in_flight, max_in_flight: max_in_flight.clone(), }); - let file = RemoteBlobFile::new(requester, PAYLOAD.len() as u64); + let file = RemoteBlobFile::new(requester, PAYLOAD.len() as u64, Some(5)); let ranges: Vec<_> = (0..16u64).map(|start| start..start + 2).collect(); let output = file.read_ranges(&ranges).await.unwrap(); @@ -1259,7 +2013,7 @@ mod tests { let requests = Arc::new(StdMutex::new(Vec::new())); let table = mock_remote_blob_table(requests.clone()); let file = table - .fetch_blob_files_impl("image", &[10]) + .probe_files("image", &[10]) .await .unwrap() .pop() @@ -1292,6 +2046,7 @@ mod tests { header::CONTENT_RANGE, format!("bytes 0-0/{}", PAYLOAD.len()), ) + .header(VERSION_HEADER, "5") .body(vec![PAYLOAD[0]]) .unwrap(); } @@ -1304,7 +2059,7 @@ mod tests { Some(Version::new(0, 5, 0)), ); - let mut files = table.fetch_blob_files_impl("image", &[10]).await.unwrap(); + let mut files = table.probe_files("image", &[10]).await.unwrap(); let file = files.remove(0).unwrap(); let error = file.read_range(1..3).await.unwrap_err(); assert!(error.to_string().contains("206"), "got: {error}"); @@ -1322,6 +2077,7 @@ mod tests { &self, range_header: &str, _mode: RangeRequestMode, + _version: Option, ) -> Result<(String, Response)> { if range_header.ends_with('-') { self.started.notify_one(); @@ -1347,7 +2103,11 @@ mod tests { release: release.clone(), started: started.clone(), }); - let file = Arc::new(RemoteBlobFile::new(requester, PAYLOAD.len() as u64)); + let file = Arc::new(RemoteBlobFile::new( + requester, + PAYLOAD.len() as u64, + Some(5), + )); let reader = { let file = file.clone(); diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index dcb3e1578..ea6e512fd 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -330,7 +330,8 @@ pub use self::merge::MergeResult; /// date) and [`LsmWriteSpec::with_writer_config_defaults`] (default /// `ShardWriter` configuration recorded in the MemWAL index). /// -/// A fresh spec maintains every index on the table, resolved on install. +/// A fresh spec maintains every index on the table, including ones created +/// after it is installed. /// /// Install a spec with [`Table::set_lsm_write_spec`] and remove it with /// [`Table::unset_lsm_write_spec`]. The actual `merge_insert` dispatch @@ -348,9 +349,8 @@ pub enum LsmWriteSpec { num_buckets: u32, /// Indexes the MemWAL maintains in-memory as rows are appended. /// - /// `None` means every index it can maintain, resolved on install — a - /// snapshot, so indexes created later need the spec unset and re-set. - /// `Some([])` maintains nothing. + /// `None` means every index the table has, including ones created + /// later. `Some([])` maintains nothing. maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, @@ -363,9 +363,8 @@ pub enum LsmWriteSpec { column: String, /// Indexes the MemWAL maintains in-memory as rows are appended. /// - /// `None` means every index it can maintain, resolved on install — a - /// snapshot, so indexes created later need the spec unset and re-set. - /// `Some([])` maintains nothing. + /// `None` means every index the table has, including ones created + /// later. `Some([])` maintains nothing. maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, @@ -374,9 +373,8 @@ pub enum LsmWriteSpec { Unsharded { /// Indexes the MemWAL maintains in-memory as rows are appended. /// - /// `None` means every index it can maintain, resolved on install — a - /// snapshot, so indexes created later need the spec unset and re-set. - /// `Some([])` maintains nothing. + /// `None` means every index the table has, including ones created + /// later. `Some([])` maintains nothing. maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, @@ -422,14 +420,15 @@ impl LsmWriteSpec { /// Set which indexes the MemWAL maintains. /// - /// `None` (the default) resolves to every index on the table at install, - /// failing if one cannot be maintained — name the set to install anyway. A - /// list is verbatim: each name must already exist and be maintainable, and - /// an empty list maintains nothing. + /// `None` (the default) is every index on the table, re-read as the table + /// changes, so an index created later is maintained too; one of a kind the + /// MemWAL cannot mirror is skipped rather than failing the table. A list is + /// verbatim: each name must already exist and be maintainable, and an empty + /// list maintains nothing. /// /// ``` /// # use lancedb::table::LsmWriteSpec; - /// // Every index the table has when the spec is installed: + /// // Every index the table has, now and later: /// LsmWriteSpec::unsharded().with_maintained_indexes(None); /// // Exactly these: /// LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_idx".to_string()]); @@ -1251,7 +1250,10 @@ impl Table { /// Materialize blob bytes for the given row ids. /// /// Output matches `row_ids` in length and order. Null blobs are null; - /// valid empty blobs contain empty byte strings. Prefer + /// valid empty blobs contain empty byte strings. Cloud limits individual + /// requests to 1024 row ids and 64 MiB of blob bytes; the remote client + /// splits requests and reads a single larger blob through the Range route. + /// This method still materializes all bytes in memory, so prefer /// [`Self::fetch_blob_files`] for large selections. /// /// `_rowid` values stay valid after compaction when the table has stable @@ -1672,6 +1674,7 @@ impl Table { /// .unwrap() /// .refine_factor(5) /// .nprobes(10) + /// .unwrap() /// .execute() /// .await /// .unwrap(); @@ -1917,11 +1920,28 @@ impl Table { self.inner.function_errors(&request).await } - /// Change a column's name or nullability. + /// Change a column's name, data type, or nullability. + /// + /// Each alteration must specify at least one of `rename`, `data_type`, or + /// `nullable`. Returns [`Error::InvalidInput`] before applying any changes + /// if an alteration does not specify any of these fields. pub async fn alter_columns( &self, alterations: &[ColumnAlteration], ) -> Result { + for alteration in alterations { + if alteration.rename.is_none() + && alteration.nullable.is_none() + && alteration.data_type.is_none() + { + return Err(Error::InvalidInput { + message: format!( + "One of rename, nullable or data_type must be specified for path '{}'", + alteration.path + ), + }); + } + } self.inner.alter_columns(alterations).await } @@ -1985,6 +2005,10 @@ impl Table { /// - [`LsmWriteSpec::identity`] — shard by the raw value of a scalar column. /// - [`LsmWriteSpec::unsharded`] — route every write to a single shard. /// + /// A table carries one spec: this fails while one is installed, since the + /// generations already written were homed under it. + /// [`Table::unset_lsm_write_spec`] removes one. + /// /// # Example /// /// ``` @@ -3788,7 +3812,20 @@ impl BaseTable for NativeTable { let Some(segment) = segments.first() else { continue; }; - let params = load_segment_params(&dataset, segment).await?; + // The listing itself only needs the manifest. Missing index files must + // not hide every other index, or callers cannot find the one to repair. + let params = match load_segment_params(&dataset, segment).await { + Ok(params) => params, + Err(err) => { + log::warn!( + "Failed to read full text search configuration for index '{}': {}", + index.name, + err + ); + index.index_details = None; + continue; + } + }; let details = serde_json::to_string(¶ms).map_err(|source| Error::Other { message: format!( "Failed to serialize full text search configuration for index '{}'", @@ -5648,29 +5685,22 @@ mod tests { assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); // Identity sharding round-trips (column recovered from the schema). - // A spec left at its default maintains every index on the table, so it - // reads back naming the one on the table rather than as "infer". + // A spec left at its default round-trips as the default: what is stored + // is the intent to maintain everything, not the set it resolves to now. let spec = LsmWriteSpec::identity("region"); table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!( - table.get_lsm_write_spec().await.unwrap(), - Some(spec.with_maintained_indexes(vec![idx_name.clone()])) - ); + assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); table.unset_lsm_write_spec().await.unwrap(); // Unsharded round-trips (no routing column). let spec = LsmWriteSpec::unsharded(); table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!( - table.get_lsm_write_spec().await.unwrap(), - Some(spec.with_maintained_indexes(vec![idx_name])) - ); + assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); } - /// The maintained set defaults to every index on the table, resolved at - /// install. An index the memtable cannot build fails the install rather - /// than being dropped: maintaining it would take the table offline for - /// writes, dropping it would hide that from the caller. + /// The maintained set defaults to every index on the table. A named index + /// the memtable cannot build fails the install, because the caller asked + /// for it; an unnamed one is skipped, because it arrived by existing. #[tokio::test] async fn test_set_lsm_write_spec_infers_maintained_indexes() { let tmp_dir = tempdir().unwrap(); @@ -5723,19 +5753,27 @@ mod tests { ); assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); - // The default covers every index, so the bitmap fails it too. - let err = table + // The default names nothing, so the bitmap is skipped rather than + // refusing the table: it is maintained by existing, not by being asked + // for. The intent is what is stored, not the set it resolves to today. + table .set_lsm_write_spec(LsmWriteSpec::unsharded()) .await - .unwrap_err(); - assert!( - matches!(err, Error::InvalidInput { ref message } - if message.contains("tag_bitmap") && message.contains("maintained_indexes")), - "expected the inferred set to be rejected, got {err:?}" + .unwrap(); + assert_eq!( + table + .get_lsm_write_spec() + .await + .unwrap() + .unwrap() + .maintained_indexes(), + None, + "an unnamed set is stored as the intent to maintain everything" ); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); - // Naming the maintainable subset installs. + // Narrowing to a named subset: unset first, since an installed spec + // cannot be set over. + table.unset_lsm_write_spec().await.unwrap(); table .set_lsm_write_spec( LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_btree".to_string()]), @@ -5767,6 +5805,28 @@ mod tests { .maintained_indexes(), Some([].as_slice()) ); + + // An installed spec is never set over, whatever the new one asks for, + // and the refused call leaves it alone. + let err = table + .set_lsm_write_spec(LsmWriteSpec::bucket("id", 4)) + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { ref message } + if message.contains("already set")), + "expected the repeat to be refused, got {err:?}" + ); + assert_eq!( + table + .get_lsm_write_spec() + .await + .unwrap() + .unwrap() + .maintained_indexes(), + Some([].as_slice()), + "a refused call changes nothing" + ); } #[tokio::test] diff --git a/rust/lancedb/src/table/add_data.rs b/rust/lancedb/src/table/add_data.rs index 0bfec9d68..c82b85971 100644 --- a/rust/lancedb/src/table/add_data.rs +++ b/rust/lancedb/src/table/add_data.rs @@ -1019,14 +1019,23 @@ mod tests { #[case::utf8(DataType::Utf8)] #[case::large_utf8(DataType::LargeUtf8)] #[tokio::test] - async fn test_add_arrow_json_into_lance_json_table(#[case] input_type: DataType) { + async fn test_add_arrow_json_into_lance_json_table( + #[case] input_type: DataType, + #[values(false, true)] table_has_serialized_metadata: bool, + #[values(false, true)] input_has_serialized_metadata: bool, + ) { use arrow_array::{Array, cast::AsArray}; use lance_arrow::ARROW_EXT_NAME_KEY; use lance_arrow::json::{ARROW_JSON_EXT_NAME, JSON_EXT_NAME}; // Build a table whose "data" column is lance.json (LargeBinary + // ARROW:extension:name = "lance.json"). - let lance_json_field = lance_arrow::json::json_field("data", true); + let mut lance_json_field = lance_arrow::json::json_field("data", true); + if table_has_serialized_metadata { + let mut metadata = lance_json_field.metadata().clone(); + metadata.insert(lance_arrow::ARROW_EXT_META_KEY.to_string(), String::new()); + lance_json_field = lance_json_field.with_metadata(metadata); + } let table_schema = Arc::new(Schema::new(vec![lance_json_field])); let db = connect("memory://").execute().await.unwrap(); @@ -1050,10 +1059,13 @@ mod tests { // Build an arrow.json input field (Utf8/LargeUtf8 + arrow.json extension). // This is what PyArrow produces for pa.json_() arrays. - let arrow_json_metadata = std::collections::HashMap::from([( + let mut arrow_json_metadata = std::collections::HashMap::from([( ARROW_EXT_NAME_KEY.to_string(), ARROW_JSON_EXT_NAME.to_string(), )]); + if input_has_serialized_metadata { + arrow_json_metadata.insert(lance_arrow::ARROW_EXT_META_KEY.to_string(), String::new()); + } let arrow_json_field = Field::new("data", input_type.clone(), true).with_metadata(arrow_json_metadata); let arrow_json_schema = Arc::new(Schema::new(vec![arrow_json_field])); diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index 98b4e1627..d92599294 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -704,13 +704,7 @@ fn ensure_known_binding_shape(value: &Value) -> Result<()> { object .get("function") .ok_or_else(|| invalid_function("Function binding is missing its exact version"))?, - &[ - "name", - "object_id", - "location", - "version", - "manifest_digest", - ], + crate::function::FUNCTION_VERSION_REF_FIELDS, "version reference", )?; for input in object @@ -3760,6 +3754,20 @@ mod tests { ensure_known_binding_shape(&raw_binding).unwrap(); } + /// A binding names its Function by parts, so one declared against a + /// namespaced Function is part of the known contract too. + #[test] + fn test_namespaced_bindings_are_a_known_shape() { + let mut raw_binding: Value = serde_json::from_str(include_str!( + "../../tests/fixtures/first_class_functions/v1/remote_initialized_function_binding.json" + )) + .unwrap(); + raw_binding["function"]["namespace_path"] = serde_json::json!(["analytics", "features"]); + ensure_known_binding_shape(&raw_binding).unwrap(); + let binding = FunctionBinding::from_json(&raw_binding.to_string()).unwrap(); + assert_eq!(binding.function().namespace_path, ["analytics", "features"]); + } + #[test] fn test_named_struct_can_be_kept_as_one_nullable_physical_column() { let application = named_struct_application("{}"); diff --git a/rust/lancedb/src/table/create_index.rs b/rust/lancedb/src/table/create_index.rs index 0a0d00da1..916b28ed1 100644 --- a/rust/lancedb/src/table/create_index.rs +++ b/rust/lancedb/src/table/create_index.rs @@ -160,6 +160,12 @@ impl NativeTable { builder = builder.name(name); } builder.await?; + // A writer already open was built before this index existed, so it has + // to pick it up here or a read needing it is refused until it reopens. + self.dataset + .shard_writer() + .refresh_maintained_indexes(&dataset) + .await; self.dataset.update(dataset); Ok(()) } @@ -2078,6 +2084,50 @@ mod tests { assert_eq!(index.columns, vec!["tags".to_string()]); } + #[tokio::test] + async fn test_list_indices_with_missing_fts_index_files() { + let tmp_dir = tempdir().unwrap(); + let uri = tmp_dir.path().to_str().unwrap(); + let conn = connect(uri).execute().await.unwrap(); + let batch = record_batch!( + ("id", Int32, [1, 2, 3]), + ("text", Utf8, ["alpha", "beta", "gamma"]) + ) + .unwrap(); + let table = conn.create_table("t", batch).execute().await.unwrap(); + table + .create_index(&["text"], Index::FTS(FtsIndexBuilder::default())) + .execute() + .await + .unwrap(); + table + .create_index(&["id"], Index::BTree(BTreeIndexBuilder::default())) + .execute() + .await + .unwrap(); + + let fts_uuid = table + .list_indices() + .await + .unwrap() + .into_iter() + .find(|index| index.name == "text_idx") + .and_then(|index| index.index_uuid) + .unwrap(); + std::fs::remove_dir_all(tmp_dir.path().join("t.lance/_indices").join(fts_uuid)).unwrap(); + + // A new connection, so the index cache cannot serve the deleted files. + let conn = connect(uri).execute().await.unwrap(); + let table = conn.open_table("t").execute().await.unwrap(); + let mut indices = table.list_indices().await.unwrap(); + indices.sort_by(|a, b| a.name.cmp(&b.name)); + let names = indices.iter().map(|i| i.name.as_str()).collect::>(); + assert_eq!(names, vec!["id_idx", "text_idx"]); + assert_eq!(indices[1].index_type, crate::index::IndexType::FTS); + assert_eq!(indices[1].index_details, None); + assert!(indices[0].index_details.is_some()); + } + #[tokio::test] async fn test_create_inverted_index() { let conn = connect("memory://").execute().await.unwrap(); diff --git a/rust/lancedb/src/table/datafusion/udtf.rs b/rust/lancedb/src/table/datafusion/udtf.rs index 11f0cc61b..53eb55fcc 100644 --- a/rust/lancedb/src/table/datafusion/udtf.rs +++ b/rust/lancedb/src/table/datafusion/udtf.rs @@ -4,3 +4,7 @@ //! User-Defined Table Functions (UDTFs) for DataFusion integration pub mod fts; + +pub mod duplicate_pairs; + +pub mod vector_query; diff --git a/rust/lancedb/src/table/datafusion/udtf/duplicate_pairs.rs b/rust/lancedb/src/table/datafusion/udtf/duplicate_pairs.rs new file mode 100644 index 000000000..20de93148 --- /dev/null +++ b/rust/lancedb/src/table/datafusion/udtf/duplicate_pairs.rs @@ -0,0 +1,698 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Index-partition duplicate pairs. No source-vector scan or shuffle is needed. + +use std::collections::BTreeSet; +use std::sync::Arc; + +use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use async_trait::async_trait; +use datafusion::catalog::TableFunctionImpl; +use datafusion_catalog::{Session, TableProvider}; +use datafusion_common::{DataFusionError, Result, ScalarValue, plan_err}; +use datafusion_execution::{SendableRecordBatchStream, TaskContext}; +use datafusion_expr::{Expr, TableType}; +use datafusion_physical_expr::EquivalenceProperties; +use datafusion_physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, + execution_plan::{Boundedness, EmissionType}, + metrics::{ExecutionPlanMetricsSet, MetricBuilder, MetricsSet}, + projection::ProjectionExec, + stream::RecordBatchStreamAdapter, +}; +use futures::{StreamExt, TryStreamExt, stream}; +use lance::{ + Dataset, + index::{DatasetIndexExt, DatasetIndexInternalExt}, +}; +use lance_index::metrics::NoOpMetricsCollector; +use uuid::Uuid; + +use super::super::BaseTableAdapter; +use crate::table::NativeTableExt; + +// Admission estimates for Lance 13's bounded pairwise kernel. Keep these with +// the minimum dependency when the kernel's buffering contract changes. +/// Version of the native task descriptor and its execution semantics. +pub const DUPLICATE_PAIRS_OPERATOR_VERSION: u32 = 2; + +const CODE_STAGING_BYTES: usize = 16 * 1024 * 1024; +const INDEX_READ_ROWS: usize = 8192; +const VECTOR_BATCH_ROWS: usize = 8192; +const VECTOR_BATCH_BYTES: usize = 16 * 1024 * 1024; +const MIN_VECTOR_BATCH_ROWS: usize = 32; +const ANCHOR_BLOCK_ROWS: usize = 32; +const SCORING_CONCURRENCY: usize = 4; +const MAX_COORDINATE_BYTES: usize = 8; +const STAGED_ROW_OVERHEAD_BYTES: usize = 64; +const PAIR_ROW_BYTES: usize = 20; + +/// Configuration belongs to one immutable source snapshot. Threshold uses +/// Lance's float32 index distance, including symmetric reconstructed distances +/// for quantized indexes; it is not an exact source-embedding distance. +#[derive(Debug, Clone)] +pub struct DuplicatePairsConfig { + pub dataset_version: u64, + pub column: String, + pub distance_threshold: f32, +} + +/// One worker input, never a row of source vectors. The complete task identity +/// also includes the dataset URI/version, column, threshold and operator version. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct DuplicatePairTask { + pub segment_id: Uuid, + pub partition_id: usize, +} + +/// Resolve an authorized source table and preserve its catalog identity. +pub trait DuplicatePairsResolver: std::fmt::Debug + Send + Sync { + fn resolve( + &self, + table: &str, + config: DuplicatePairsConfig, + tasks_only: bool, + ) -> Result>; +} + +/// `vector_duplicate_pairs('table', version, 'column', threshold)`. +/// Registration is explicit so the host controls catalog access and execution. +#[derive(Debug)] +pub struct DuplicatePairsTableFunction { + resolver: Arc, + tasks_only: bool, +} + +impl DuplicatePairsTableFunction { + pub fn new(resolver: Arc) -> Self { + Self { + resolver, + tasks_only: false, + } + } +} + +impl DuplicatePairsTableFunction { + /// Register as `vector_duplicate_pair_tasks` to inspect or persist exactly + /// the descriptors used by the pair operator, without producing any pairs. + pub fn task_manifest(resolver: Arc) -> Self { + Self { + resolver, + tasks_only: true, + } + } +} + +impl TableFunctionImpl for DuplicatePairsTableFunction { + fn call(&self, args: &[Expr]) -> Result> { + if args.len() != 4 { + return plan_err!( + "vector_duplicate_pairs requires (table, dataset_version, column, distance_threshold)" + ); + } + let string = |expr: &Expr| match expr { + Expr::Literal(ScalarValue::Utf8(Some(s)) | ScalarValue::LargeUtf8(Some(s)), _) => { + Ok(s.clone()) + } + _ => plan_err!("table and column must be string literals"), + }; + let dataset_version = match &args[1] { + Expr::Literal(ScalarValue::Int64(Some(v)), _) if *v > 0 => *v as u64, + Expr::Literal(ScalarValue::UInt64(Some(v)), _) if *v > 0 => *v, + _ => return plan_err!("dataset_version must be a positive integer literal"), + }; + let distance_threshold = match &args[3] { + Expr::Literal(value, _) if value.data_type().is_numeric() => { + value.cast_to(&DataType::Float32)? + } + _ => return plan_err!("distance_threshold must be a finite numeric literal"), + }; + let ScalarValue::Float32(Some(distance_threshold)) = distance_threshold else { + return plan_err!("distance_threshold cannot be null"); + }; + if !distance_threshold.is_finite() { + return plan_err!("distance_threshold must be finite"); + } + self.resolver.resolve( + &string(&args[0])?, + DuplicatePairsConfig { + dataset_version, + column: string(&args[2])?, + distance_threshold, + }, + self.tasks_only, + ) + } +} + +/// Three non-null columns; pair direction is storage traversal order, not row-ID order. +pub fn duplicate_pair_schema() -> SchemaRef { + Arc::new(Schema::new(vec![ + Field::new("row_id_a", DataType::UInt64, false), + Field::new("row_id_b", DataType::UInt64, false), + Field::new("distance", DataType::Float32, false), + ])) +} + +#[derive(Debug)] +pub struct DuplicatePairsTable { + source: Arc, + config: DuplicatePairsConfig, + tasks_only: bool, +} + +impl BaseTableAdapter { + /// Bind the builtin to this authorized table; only metadata is read while + /// planning. The source snapshot must remain retained until execution ends. + pub fn duplicate_pairs( + &self, + config: DuplicatePairsConfig, + tasks_only: bool, + ) -> DuplicatePairsTable { + DuplicatePairsTable { + source: self.table.clone(), + config, + tasks_only, + } + } +} + +#[async_trait] +impl TableProvider for DuplicatePairsTable { + fn schema(&self) -> SchemaRef { + if self.tasks_only { + duplicate_pair_task_schema() + } else { + duplicate_pair_schema() + } + } + fn table_type(&self) -> TableType { + TableType::View + } + async fn scan( + &self, + _state: &dyn Session, + projection: Option<&Vec>, + _filters: &[Expr], + _limit: Option, + ) -> Result> { + let native = self.source.as_native().ok_or_else(|| { + DataFusionError::Plan("duplicate pairs requires a native indexed table".into()) + })?; + let current = native + .dataset + .get() + .await + .map_err(|e| DataFusionError::External(Box::new(e)))?; + let dataset = Arc::new( + current + .checkout_version(self.config.dataset_version) + .await?, + ); + let tasks = plan_duplicate_pairs(dataset.clone(), &self.config).await?; + if self.tasks_only { + let batch = duplicate_pair_task_batch(&dataset, &self.config, &tasks)?; + let table = + datafusion::datasource::MemTable::try_new(batch.schema(), vec![vec![batch]])?; + return table.scan(_state, projection, _filters, _limit).await; + } + let plan: Arc = Arc::new(DuplicatePairsExec::try_new( + dataset, + self.config.clone(), + tasks, + )?); + match projection { + Some(indices) => { + let schema = plan.schema(); + let exprs = indices + .iter() + .map(|&i| { + ( + Arc::new(datafusion_physical_expr::expressions::Column::new( + schema.field(i).name(), + i, + )) as _, + schema.field(i).name().clone(), + ) + }) + .collect::>(); + Ok(Arc::new(ProjectionExec::try_new(exprs, plan)?)) + } + None => Ok(plan), + } + } +} + +/// Enumerate segment-local IVF partitions using metadata only. Live-row masks +/// and index codes are opened by the scoped worker, never by a source-vector +/// scan on the coordinator. The high-level pair API is only a test oracle. +pub async fn plan_duplicate_pairs( + dataset: Arc, + config: &DuplicatePairsConfig, +) -> Result> { + if dataset.version().version != config.dataset_version || !config.distance_threshold.is_finite() + { + return plan_err!("duplicate pairs source snapshot or threshold is invalid"); + } + let field = dataset.schema().field_id(&config.column)?; + let indices = dataset.load_indices().await?; + let segments = indices + .iter() + .filter(|meta| { + meta.keyed_fields() == [field] + && meta.index_details.as_ref().map_or_else( + || { + meta.files.as_ref().is_some_and(|files| { + files + .iter() + .any(|file| file.path == lance_index::INDEX_FILE_NAME) + }) + }, + |details| details.type_url.ends_with("VectorIndexDetails"), + ) + }) + .collect::>(); + let names = segments + .iter() + .map(|meta| meta.name.as_str()) + .collect::>(); + if names.len() != 1 { + return plan_err!( + "column '{}' requires exactly one current-format vector index, found {}", + config.column, + names.len() + ); + } + let name = names.into_iter().next().expect("one index name"); + if !dataset.unindexed_fragments(name).await?.is_empty() { + return plan_err!( + "vector index on column '{}' does not cover all fragments; optimize the index first", + config.column + ); + } + let mut tasks = Vec::new(); + for meta in segments { + if dataset.fragments().iter().any(|fragment| { + meta.fragment_bitmap + .as_ref() + .is_none_or(|ids| ids.contains(fragment.id as u32)) + && fragment.overlays.iter().any(|overlay| { + overlay.committed_version > meta.dataset_version + && overlay.data_file.fields.contains(&field) + }) + }) { + return plan_err!( + "column '{}' has stale index values in segment {}; rebuild the index", + config.column, + meta.uuid + ); + } + let index = dataset + .open_vector_index(&config.column, &meta.uuid, &NoOpMetricsCollector) + .await?; + if !index.supports_pairwise_vectors() { + return plan_err!( + "segment {} requires a current-format pairwise vector index; rebuild the index", + meta.uuid + ); + } + for partition_id in 0..index.ivf_model().num_partitions() { + tasks.push(DuplicatePairTask { + segment_id: meta.uuid, + partition_id, + }); + } + } + if tasks.is_empty() { + return plan_err!("no vector index partitions found"); + } + Ok(tasks) +} + +/// Optional query columns. The default preserves the primitive's three-column +/// schema; SQL convenience calls include physical scope identity and may map +/// endpoints to an application key without fetching source vectors. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct DuplicatePairsOutput { + pub scope: bool, + pub id_column: Option, +} + +impl DuplicatePairsOutput { + pub fn schema(&self, dataset: &Dataset) -> Result { + let mut fields = duplicate_pair_schema().fields().to_vec(); + if self.scope { + fields.extend([ + Arc::new(Field::new("segment_id", DataType::Utf8, false)), + Arc::new(Field::new("partition_id", DataType::UInt64, false)), + Arc::new(Field::new("dataset_version", DataType::UInt64, false)), + ]); + } + if let Some(column) = &self.id_column { + let schema = Schema::from(dataset.schema()); + let field = schema.field_with_name(column)?; + if column.contains('.') + || matches!( + field.data_type(), + DataType::FixedSizeList(..) + | DataType::List(..) + | DataType::LargeList(..) + | DataType::Struct(..) + | DataType::Map(..) + ) + { + return plan_err!("id_column must be a top-level scalar column"); + } + fields.push(Arc::new(Field::new( + "id_a", + field.data_type().clone(), + field.is_nullable(), + ))); + fields.push(Arc::new(Field::new( + "id_b", + field.data_type().clone(), + field.is_nullable(), + ))); + } + Ok(Arc::new(Schema::new(fields))) + } + + async fn enrich( + &self, + batch: arrow_array::RecordBatch, + dataset: &Dataset, + task: &DuplicatePairTask, + ) -> Result { + use arrow_array::{StringArray, UInt64Array}; + let mut columns = batch.columns().to_vec(); + let n = batch.num_rows(); + if self.scope { + columns.extend([ + Arc::new(StringArray::from(vec![task.segment_id.to_string(); n])) + as arrow_array::ArrayRef, + Arc::new(UInt64Array::from(vec![task.partition_id as u64; n])), + Arc::new(UInt64Array::from(vec![dataset.version().version; n])), + ]); + } + if let Some(column) = &self.id_column { + for endpoint in 0..2 { + let ids = batch + .column(endpoint) + .as_any() + .downcast_ref::() + .expect("native row IDs"); + let projection = lance::dataset::ProjectionRequest::from_columns( + [column.as_str()], + dataset.schema(), + ); + let values = dataset.take_rows(ids.values(), projection).await?; + if values.num_rows() != n { + return plan_err!("pair row ID is missing from the pinned snapshot"); + } + columns.push(values.column(0).clone()); + } + } + Ok(arrow_array::RecordBatch::try_new( + self.schema(dataset)?, + columns, + )?) + } +} + +/// Streaming native operator over partition descriptors, never source-vector +/// rows. Assigned descriptors are consumed sequentially; distributed hosts send +/// one descriptor per worker request. No Python or per-vector ANN calls. +/// Distributed merging deliberately advertises no global row ordering. +#[derive(Debug)] +pub struct DuplicatePairsExec { + dataset: Arc, + config: DuplicatePairsConfig, + tasks: Vec, + properties: Arc, + metrics: ExecutionPlanMetricsSet, + output: DuplicatePairsOutput, +} + +impl DuplicatePairsExec { + pub fn try_new( + dataset: Arc, + config: DuplicatePairsConfig, + tasks: Vec, + ) -> Result { + if dataset.version().version != config.dataset_version + || !config.distance_threshold.is_finite() + || tasks.is_empty() + { + return plan_err!("invalid duplicate-pair snapshot, threshold, or empty task list"); + } + let unique = tasks + .iter() + .map(|t| (t.segment_id, t.partition_id)) + .collect::>(); + if unique.len() != tasks.len() { + return plan_err!("duplicate partition tasks"); + } + let properties = Arc::new(PlanProperties::new( + EquivalenceProperties::new(duplicate_pair_schema()), + Partitioning::UnknownPartitioning(1), + EmissionType::Incremental, + Boundedness::Bounded, + )); + Ok(Self { + dataset, + config, + tasks, + properties, + metrics: ExecutionPlanMetricsSet::new(), + output: DuplicatePairsOutput::default(), + }) + } + /// Configure optional output fields before dispatching this plan. + pub fn with_output(mut self, output: DuplicatePairsOutput) -> Result { + self.properties = Arc::new(PlanProperties::new( + EquivalenceProperties::new(output.schema(&self.dataset)?), + Partitioning::UnknownPartitioning(1), + EmissionType::Incremental, + Boundedness::Bounded, + )); + self.output = output; + Ok(self) + } + pub fn output(&self) -> &DuplicatePairsOutput { + &self.output + } + pub fn dataset(&self) -> &Arc { + &self.dataset + } + pub fn config(&self) -> &DuplicatePairsConfig { + &self.config + } + pub fn tasks(&self) -> &[DuplicatePairTask] { + &self.tasks + } +} + +impl DisplayAs for DuplicatePairsExec { + fn fmt_as(&self, _: DisplayFormatType, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "DuplicatePairsExec: source_version={}, column={}, threshold={}, tasks={}", + self.config.dataset_version, + self.config.column, + self.config.distance_threshold, + self.tasks.len() + ) + } +} + +impl ExecutionPlan for DuplicatePairsExec { + fn name(&self) -> &str { + "DuplicatePairsExec" + } + fn properties(&self) -> &Arc { + &self.properties + } + fn children(&self) -> Vec<&Arc> { + vec![] + } + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + if !children.is_empty() { + return plan_err!("DuplicatePairsExec has no children"); + } + Ok(self) + } + fn metrics(&self) -> Option { + Some(self.metrics.clone_inner()) + } + fn execute( + &self, + partition: usize, + context: Arc, + ) -> Result { + if partition != 0 { + return plan_err!("duplicate-pair output partition out of bounds"); + } + let output = self.output.clone(); + let tasks = self.tasks.clone(); + let dataset = self.dataset.clone(); + let config = self.config.clone(); + let output_rows = MetricBuilder::new(&self.metrics).counter("output_rows", partition); + let completed = + MetricBuilder::new(&self.metrics).counter("completed_partitions", partition); + // Beta.14 stages native codes and scores bounded tiles concurrently. + // Keep the host staging budget explicit rather than inheriting Lance's + // larger default or multiplying CPU-pool concurrency across PE tasks. + let dimension = match dataset + .schema() + .field(&config.column) + .map(|f| f.data_type()) + { + Some(DataType::FixedSizeList(_, dimension)) if dimension > 0 => dimension as usize, + _ => return plan_err!("duplicate pairs requires a fixed-size vector column"), + }; + let batch_bytes = VECTOR_BATCH_BYTES.max( + MIN_VECTOR_BATCH_ROWS * (dimension * MAX_COORDINATE_BYTES + STAGED_ROW_OVERHEAD_BYTES), + ); + // Allow staging/preparation overlap, at least three live spill batches, + // source input, and in-flight output plus the batch being encoded. + // Quantizer models and caches remain outside this admission estimate. + let budget = 2 * CODE_STAGING_BYTES + + 3 * batch_bytes + + INDEX_READ_ROWS * (dimension * MAX_COORDINATE_BYTES + STAGED_ROW_OVERHEAD_BYTES) + + (SCORING_CONCURRENCY + 1) * ANCHOR_BLOCK_ROWS * VECTOR_BATCH_ROWS * PAIR_ROW_BYTES; + let reservation = + datafusion_execution::memory_pool::MemoryConsumer::new("DuplicatePairsExec") + .register(context.memory_pool()); + reservation.try_grow(budget)?; + let reservation = Arc::new(reservation); + // Sequential local work prevents one query from reserving a kernel for + // every IVF partition at once. Distributed hosts split these descriptors + // into one-task PE requests before execution. + let stream = stream::iter(tasks) + .then(move |task| { + let output = output.clone(); + let dataset = dataset.clone(); + let config = config.clone(); + let reservation = reservation.clone(); + let output_rows = output_rows.clone(); + let completed = completed.clone(); + async move { + let reader = lance::index::vector::dedup::find_duplicate_pairs_in_partition_with_options( + dataset.clone(), + &config.column, + task.segment_id, + task.partition_id, + config.distance_threshold, + lance::index::vector::dedup::DuplicatePairsOptions::default() + .with_memory_limit(CODE_STAGING_BYTES) + .with_max_concurrency(SCORING_CONCURRENCY), + ) + .await?; + let measured = stream::try_unfold( + (reader, reservation), + move |(mut reader, reservation)| { + let output_rows = output_rows.clone(); + let completed = completed.clone(); + async move { + match reader.try_next().await? { + Some(batch) => { + output_rows.add(batch.num_rows()); + Ok::<_, DataFusionError>(Some(( + batch, + (reader, reservation), + ))) + } + None => { + completed.add(1); + Ok(None) + } + } + } + }, + ); + let schema = output.schema(&dataset)?; + let measured = measured.and_then(move |batch| { + let output = output.clone(); + let dataset = dataset.clone(); + let task = task.clone(); + async move { output.enrich(batch, &dataset, &task).await } + }); + Ok::<_, DataFusionError>(Box::pin(RecordBatchStreamAdapter::new(schema, measured)) as SendableRecordBatchStream) + } + }) + .try_flatten(); + Ok(Box::pin(RecordBatchStreamAdapter::new( + self.schema(), + stream, + ))) + } +} + +/// Inspectable immutable task manifest. Credentials are never included. +pub fn duplicate_pair_task_schema() -> SchemaRef { + Arc::new(Schema::new(vec![ + Field::new("task_id", DataType::Utf8, false), + Field::new("dataset_uri", DataType::Utf8, false), + Field::new("dataset_version", DataType::UInt64, false), + Field::new("column", DataType::Utf8, false), + Field::new("segment_id", DataType::Utf8, false), + Field::new("partition_id", DataType::UInt64, false), + Field::new("distance_threshold", DataType::Float32, false), + Field::new("operator_version", DataType::UInt32, false), + ])) +} + +pub fn duplicate_pair_task_batch( + dataset: &Dataset, + config: &DuplicatePairsConfig, + tasks: &[DuplicatePairTask], +) -> Result { + use arrow_array::{Float32Array, StringArray, UInt32Array, UInt64Array}; + use sha2::{Digest, Sha256}; + let ids = tasks + .iter() + .map(|task| { + // Length-delimited strings prevent ambiguous concatenation; threshold + // bits preserve the exact float32 value sent to the native primitive. + let bytes = serde_json::to_vec(&( + DUPLICATE_PAIRS_OPERATOR_VERSION, + dataset.uri(), + config.dataset_version, + &config.column, + task.segment_id.to_string(), + task.partition_id, + config.distance_threshold.to_bits(), + )) + .expect("task fields are serializable"); + format!("{:x}", Sha256::digest(bytes)) + }) + .collect::>(); + let n = tasks.len(); + Ok(arrow_array::RecordBatch::try_new( + duplicate_pair_task_schema(), + vec![ + Arc::new(StringArray::from(ids)), + Arc::new(StringArray::from(vec![dataset.uri(); n])), + Arc::new(UInt64Array::from(vec![config.dataset_version; n])), + Arc::new(StringArray::from(vec![config.column.as_str(); n])), + Arc::new(StringArray::from( + tasks + .iter() + .map(|t| t.segment_id.to_string()) + .collect::>(), + )), + Arc::new(UInt64Array::from( + tasks + .iter() + .map(|t| t.partition_id as u64) + .collect::>(), + )), + Arc::new(Float32Array::from(vec![config.distance_threshold; n])), + Arc::new(UInt32Array::from(vec![DUPLICATE_PAIRS_OPERATOR_VERSION; n])), + ], + )?) +} diff --git a/rust/lancedb/src/table/datafusion/udtf/vector_query.rs b/rust/lancedb/src/table/datafusion/udtf/vector_query.rs new file mode 100644 index 000000000..0d4f23d92 --- /dev/null +++ b/rust/lancedb/src/table/datafusion/udtf/vector_query.rs @@ -0,0 +1,352 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Composable index-backed pair and dedup relations. Hosts bind a snapshot once +//! during planning, distribute the pair child, and use their ordinary SQL sink. + +use std::sync::Arc; + +use arrow_schema::SchemaRef; +use async_trait::async_trait; +use datafusion_catalog::{Session, TableProvider}; +use datafusion_common::{Result, plan_err}; +use datafusion_execution::{SendableRecordBatchStream, TaskContext}; +use datafusion_expr::{Expr, TableType}; +use datafusion_physical_expr::{EquivalenceProperties, Partitioning}; +use datafusion_physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, + coalesce_partitions::CoalescePartitionsExec, + empty::EmptyExec, + execution_plan::{Boundedness, EmissionType}, + projection::ProjectionExec, + stream::RecordBatchStreamAdapter, +}; +use futures::{TryStreamExt, stream}; +use lance::dataset::scanner::{RowAddrMask, RowAddrTreeMap}; +use lance::{Dataset, index::DatasetIndexInternalExt}; +use lance_core::datatypes::BlobHandling; +use lance_index::metrics::NoOpMetricsCollector; + +use super::duplicate_pairs::{ + DuplicatePairTask, DuplicatePairsConfig, DuplicatePairsExec, DuplicatePairsOutput, + plan_duplicate_pairs, +}; +use crate::table::{NativeTableExt, datafusion::BaseTableAdapter}; + +const REPRESENTATIVE_SORT_MEMORY_BYTES: usize = 256 * 1024 * 1024; +const MATERIALIZE_BATCH_ROWS: usize = 1024; + +/// Select nonempty physical IVF scopes before dispatch, never by filtering the +/// resulting pairs. A count spans all segments; partition IDs are segment-local. +#[derive(Debug, Clone, Default)] +pub enum PartitionSelection { + #[default] + All, + First { + count: usize, + }, + Random { + count: usize, + seed: u64, + }, +} + +/// Query-only options. Snapshot binding is separate so multiple calls in one +/// statement can share the same immutable dataset. +#[derive(Debug, Clone)] +pub struct VectorQueryOptions { + pub column: String, + pub distance_threshold: f32, + pub selection: PartitionSelection, + pub output: DuplicatePairsOutput, + pub dedup: bool, +} + +impl BaseTableAdapter { + /// Open a retained version, or refresh and capture the latest manifest once. + /// No worker is allowed to independently resolve "latest". + pub async fn vector_query_snapshot(&self, version: Option) -> Result> { + let native = self.table.as_native().ok_or_else(|| { + datafusion_common::DataFusionError::Plan( + "vector queries require a native indexed table".into(), + ) + })?; + if version == Some(0) { + return plan_err!("dataset_version must be positive"); + } + let current = native + .dataset + .get() + .await + .map_err(|e| datafusion_common::DataFusionError::External(Box::new(e)))?; + Ok(match version { + Some(version) => Arc::new(current.checkout_version(version).await?), + None => { + // Do not change a shared table handle's pinned version. + let mut latest = (*current).clone(); + latest.checkout_latest().await?; + Arc::new(latest) + } + }) + } +} + +/// A table provider bound to one source snapshot. The host retains the user's +/// unbound SQL for subsequent refreshes; this provider lives only for this run. +#[derive(Debug)] +pub struct VectorQueryTable { + dataset: Arc, + config: DuplicatePairsConfig, + tasks: Vec, + options: VectorQueryOptions, + schema: SchemaRef, +} + +impl VectorQueryTable { + pub async fn try_new(dataset: Arc, options: VectorQueryOptions) -> Result { + if options.dedup + && (!matches!(options.selection, PartitionSelection::All) + || options.output.id_column.is_some()) + { + return plan_err!( + "vector_dedup requires all partitions and returns original rows; sampling and id_column are pair-query options" + ); + } + let config = DuplicatePairsConfig { + dataset_version: dataset.version().version, + column: options.column.clone(), + distance_threshold: options.distance_threshold, + }; + let mut tasks = plan_duplicate_pairs(dataset.clone(), &config).await?; + // Canonical scope order is independent of metadata enumeration order. + tasks.sort_by_key(|task| (task.segment_id, task.partition_id)); + let mut nonempty = Vec::new(); + for group in tasks.chunk_by(|a, b| a.segment_id == b.segment_id) { + let index = dataset + .open_vector_index(&config.column, &group[0].segment_id, &NoOpMetricsCollector) + .await?; + nonempty.extend( + group + .iter() + .filter(|task| index.partition_size(task.partition_id) > 0) + .cloned(), + ); + } + match options.selection { + PartitionSelection::All => {} + PartitionSelection::First { count } | PartitionSelection::Random { count, .. } + if count == 0 => + { + return plan_err!("partition count must be positive"); + } + PartitionSelection::First { count } => nonempty.truncate(count), + PartitionSelection::Random { count, seed } => { + // A single segment matches ivf_partition's SplitMix64 sampling. + // With multiple segments the canonical scope ordinal extends the + // same policy without conflating equal partition IDs. + let segments = tasks + .iter() + .map(|t| t.segment_id) + .collect::>() + .len(); + let mut ranked = nonempty + .into_iter() + .enumerate() + .map(|(ordinal, task)| { + let key = if segments == 1 { + task.partition_id as u64 + } else { + ordinal as u64 + }; + (sample_rank(seed, key), task) + }) + .collect::>(); + ranked.sort_by_key(|(rank, task)| (*rank, task.segment_id, task.partition_id)); + ranked.truncate(count); + nonempty = ranked.into_iter().map(|(_, task)| task).collect(); + nonempty.sort_by_key(|task| (task.segment_id, task.partition_id)); + } + } + let schema = if options.dedup { + let mut scanner = dataset.scan(); + scanner.blob_handling(BlobHandling::AllBinary); + scanner.schema().await? + } else { + options.output.schema(&dataset)? + }; + Ok(Self { + dataset, + config, + tasks: nonempty, + options, + schema, + }) + } +} + +fn sample_rank(seed: u64, ordinal: u64) -> u64 { + let mut value = seed.wrapping_add(ordinal).wrapping_add(0x9E3779B97F4A7C15); + value = (value ^ (value >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); + value = (value ^ (value >> 27)).wrapping_mul(0x94D049BB133111EB); + value ^ (value >> 31) +} + +#[async_trait] +impl TableProvider for VectorQueryTable { + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + fn table_type(&self) -> TableType { + TableType::View + } + async fn scan( + &self, + _state: &dyn Session, + projection: Option<&Vec>, + _filters: &[Expr], + _limit: Option, + ) -> Result> { + let output = if self.options.dedup { + DuplicatePairsOutput::default() + } else { + self.options.output.clone() + }; + let pairs: Arc = if self.tasks.is_empty() { + Arc::new(EmptyExec::new(output.schema(&self.dataset)?)) + } else { + Arc::new( + DuplicatePairsExec::try_new( + self.dataset.clone(), + self.config.clone(), + self.tasks.clone(), + )? + .with_output(output)?, + ) + }; + let plan: Arc = if self.options.dedup { + Arc::new(VectorDedupExec::new( + self.dataset.clone(), + pairs, + self.schema.clone(), + )) + } else { + pairs + }; + if let Some(indices) = projection { + let expressions: Vec<_> = indices + .iter() + .map(|&i| { + ( + Arc::new(datafusion_physical_expr::expressions::Column::new( + self.schema.field(i).name(), + i, + )) + as Arc, + self.schema.field(i).name().clone(), + ) + }) + .collect(); + Ok(Arc::new(ProjectionExec::try_new(expressions, plan)?)) + } else { + Ok(plan) + } + } +} + +/// Greedy direct representatives over the complete pair relation. Sorting spills +/// to disk; only vertex masks are retained. This has no durable checkpoint and +/// publishes nothing: failure is handled by the enclosing query/MV transaction. +#[derive(Debug)] +pub struct VectorDedupExec { + dataset: Arc, + pairs: Arc, + properties: Arc, +} + +impl VectorDedupExec { + fn new(dataset: Arc, pairs: Arc, schema: SchemaRef) -> Self { + Self { + dataset, + pairs, + properties: Arc::new(PlanProperties::new( + EquivalenceProperties::new(schema), + Partitioning::UnknownPartitioning(1), + EmissionType::Final, + Boundedness::Bounded, + )), + } + } +} + +impl DisplayAs for VectorDedupExec { + fn fmt_as(&self, _: DisplayFormatType, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "VectorDedupExec: source_version={}, policy=direct_representative", + self.dataset.version().version + ) + } +} + +impl ExecutionPlan for VectorDedupExec { + fn name(&self) -> &str { + "VectorDedupExec" + } + fn properties(&self) -> &Arc { + &self.properties + } + fn children(&self) -> Vec<&Arc> { + vec![&self.pairs] + } + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + if children.len() != 1 { + return plan_err!("VectorDedupExec requires one pair input"); + } + Ok(Arc::new(Self::new( + self.dataset.clone(), + children[0].clone(), + self.schema(), + ))) + } + fn execute( + &self, + partition: usize, + context: Arc, + ) -> Result { + if partition != 0 { + return plan_err!("invalid dedup output partition"); + } + let dataset = self.dataset.clone(); + let pairs = CoalescePartitionsExec::new(self.pairs.clone()).execute(0, context)?; + let stream = stream::once(async move { + let (removed, _) = crate::materialized_view::vector_dedup::select_representatives( + pairs, + REPRESENTATIVE_SORT_MEMORY_BYTES, + ) + .await + .map_err(|e| datafusion_common::DataFusionError::External(Box::new(e)))?; + let mut scanner = dataset.scan(); + scanner + .with_row_addr_prefilter(RowAddrMask::from_block(RowAddrTreeMap::from_iter( + removed, + ))) + .blob_handling(BlobHandling::AllBinary) + .batch_size(MATERIALIZE_BATCH_ROWS) + .strict_batch_size(true); + Ok::<_, datafusion_common::DataFusionError>( + scanner + .try_into_stream() + .await? + .map_err(datafusion_common::DataFusionError::from), + ) + }) + .try_flatten(); + Ok(Box::pin(RecordBatchStreamAdapter::new( + self.schema(), + stream, + ))) + } +} diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index 8f5585829..354eef5a5 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -1,15 +1,18 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors +use std::collections::HashSet; use std::sync::Arc; use std::time::Duration; use arrow_array::RecordBatchReader; +use arrow_schema::{DataType, Fields}; use futures::future::Either; use futures::{FutureExt, TryFutureExt}; use lance::dataset::{ MergeInsertBuilder as LanceMergeInsertBuilder, WhenMatched, WhenNotMatchedBySource, }; +use lance_datafusion::utils::StreamingWriteSource; use serde::{Deserialize, Serialize}; use crate::error::{Error, Result}; @@ -248,6 +251,59 @@ fn canonicalize_merge_filter(filter: Option) -> Result Result<()> { + let mut names = HashSet::with_capacity(input.len()); + for field in input { + let path = if parent.is_empty() { + field.name().clone() + } else { + format!("{parent}.{}", field.name()) + }; + if !names.insert(field.name()) { + return Err(Error::InvalidInput { + message: format!("merge source field '{path}' is specified more than once"), + }); + } + let target_field = target + .iter() + .find(|candidate| candidate.name() == field.name()) + .ok_or_else(|| Error::InvalidInput { + message: format!("merge source field '{path}' is not present in the table schema"), + })?; + validate_merge_source_type(field.data_type(), target_field.data_type(), &path)?; + } + Ok(()) +} + +fn validate_merge_source_type(input: &DataType, target: &DataType, path: &str) -> Result<()> { + match (input, target) { + (DataType::Struct(input), DataType::Struct(target)) => { + validate_merge_source_fields(input, target, path) + } + ( + DataType::List(input) + | DataType::LargeList(input) + | DataType::FixedSizeList(input, _) + | DataType::ListView(input) + | DataType::LargeListView(input), + DataType::List(target) + | DataType::LargeList(target) + | DataType::FixedSizeList(target, _) + | DataType::ListView(target) + | DataType::LargeListView(target), + ) + | (DataType::Map(input, _), DataType::Map(target, _)) => { + validate_merge_source_type(input.data_type(), target.data_type(), path) + } + (DataType::Dictionary(_, input), _) => validate_merge_source_type(input, target, path), + (_, DataType::Dictionary(_, target)) => validate_merge_source_type(input, target, path), + _ => Ok(()), + } +} + /// Internal implementation of the merge insert logic /// /// This logic was moved from NativeTable::merge_insert to keep table.rs clean. @@ -279,6 +335,27 @@ pub(crate) async fn execute_merge_insert( } let dataset = table.dataset.get().await?; + let schema = arrow_schema::Schema::from(dataset.schema()); + // JSON source fields must carry the stored extension metadata just as on + // append. Keep arrow.json text labelled until Lance encodes it as JSONB. + let source = if schema + .fields() + .iter() + .any(|field| lance_arrow::json::has_json_fields(field)) + { + validate_merge_source_fields(new_data.schema().fields(), schema.fields(), "")?; + let plan = Arc::new(super::datafusion::scannable_exec::ScannableExec::new( + Box::new(new_data), + None, + )); + let plan = super::datafusion::cast::cast_to_table_schema(plan, &schema)?; + datafusion_physical_plan::execute_stream( + plan, + Arc::new(datafusion_execution::TaskContext::default()), + )? + } else { + new_data.into_stream() + }; let mut builder = LanceMergeInsertBuilder::try_new(dataset.clone(), params.on)?; match ( params.when_matched_update_all, @@ -313,10 +390,7 @@ pub(crate) async fn execute_merge_insert( builder.use_index(params.use_index); let future = if let Some(timeout) = params.timeout { - let future = builder - .retry_timeout(timeout) - .try_build()? - .execute_reader(new_data); + let future = builder.retry_timeout(timeout).try_build()?.execute(source); Either::Left(tokio::time::timeout(timeout, future).map(|res| match res { Ok(Ok((new_dataset, stats))) => Ok((new_dataset, stats)), Ok(Err(e)) => Err(e.into()), @@ -326,7 +400,7 @@ pub(crate) async fn execute_merge_insert( })) } else { let job = builder.try_build()?; - Either::Right(job.execute_reader(new_data).map_err(|e| e.into())) + Either::Right(job.execute(source).map_err(|e| e.into())) }; let (new_dataset, stats) = future.await?; let version = new_dataset.manifest().version; @@ -353,6 +427,186 @@ mod tests { use crate::connect; + #[rstest::rstest] + #[case(None)] + #[case(Some("struct"))] + #[case(Some("list"))] + #[case(Some("large_list"))] + #[case(Some("fixed_size_list"))] + #[case(Some("map"))] + #[tokio::test] + async fn merge_json_rejects_ambiguous_source_fields( + #[case] nested: Option<&str>, + #[values(false, true)] duplicate: bool, + ) { + let mut target = vec![ + Field::new("id", DataType::Int32, false), + lance_arrow::json::json_field("payload", true), + ]; + let mut input = vec![ + Field::new("id", DataType::Int32, false), + Field::new("payload", DataType::Utf8, true), + ]; + let mut columns: Vec> = vec![ + Arc::new(Int32Array::from(vec![1])), + Arc::new(StringArray::from(vec![r#"{"x":1}"#])), + ]; + if let Some(nested) = nested { + let wrap = |children: Vec| { + let structure = DataType::Struct(children.into()); + let item = Arc::new(Field::new("item", structure.clone(), true)); + match nested { + "struct" => structure, + "list" => DataType::List(item), + "large_list" => DataType::LargeList(item), + "fixed_size_list" => DataType::FixedSizeList(item, 2), + "map" => DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct( + vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", structure, true), + ] + .into(), + ), + false, + )), + false, + ), + _ => unreachable!(), + } + }; + let known = Field::new("value", DataType::Int32, true); + let unexpected = if duplicate { "value" } else { "extra" }; + let target_type = wrap(vec![known.clone()]); + let input_type = wrap(vec![known, Field::new(unexpected, DataType::Int32, true)]); + columns.push(arrow_array::new_null_array(&input_type, 1)); + target.push(Field::new("details", target_type, true)); + input.push(Field::new("details", input_type, true)); + } else { + let unexpected = if duplicate { "id" } else { "extra" }; + input.push(Field::new(unexpected, DataType::Int32, true)); + columns.push(Arc::new(Int32Array::from(vec![99]))); + } + let db = connect("memory://").execute().await.unwrap(); + let table = db + .create_empty_table("ambiguous_json", Arc::new(Schema::new(target))) + .execute() + .await + .unwrap(); + let version = table.version().await.unwrap(); + let schema = Arc::new(Schema::new(input)); + let batch = RecordBatch::try_new(schema.clone(), columns).unwrap(); + let mut merge = table.merge_insert(&["id"]); + merge.when_not_matched_insert_all(); + let error = merge + .execute(Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema))) + .await + .expect_err("JSON alignment must not drop source fields"); + let expected = if duplicate { + "more than once" + } else { + "not present" + }; + assert!(error.to_string().contains(expected), "{error}"); + assert_eq!(table.version().await.unwrap(), version); + assert_eq!(table.count_rows(None).await.unwrap(), 0); + } + + #[tokio::test] + async fn merge_json_preserves_empty_extension_metadata() { + use crate::query::{ExecutableQuery, QueryBase, Select}; + use futures::TryStreamExt; + let mut stored_json = lance_arrow::json::json_field("payload", true); + let mut metadata = stored_json.metadata().clone(); + metadata.insert(lance_arrow::ARROW_EXT_META_KEY.into(), String::new()); + stored_json = stored_json.with_metadata(metadata); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + stored_json, + ])); + let db = connect("memory://").execute().await.unwrap(); + let table = db + .create_empty_table("json_merge_metadata", schema) + .execute() + .await + .unwrap(); + let input_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("payload", DataType::Utf8, true).with_metadata( + std::collections::HashMap::from([( + lance_arrow::ARROW_EXT_NAME_KEY.into(), + lance_arrow::json::ARROW_JSON_EXT_NAME.into(), + )]), + ), + ])); + let seed = RecordBatch::try_new( + input_schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1])), + Arc::new(StringArray::from(vec![r#"{"old":true}"#])), + ], + ) + .unwrap(); + table.add(seed).execute().await.unwrap(); + let batch = RecordBatch::try_new( + input_schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2])), + Arc::new(StringArray::from(vec![ + r#"{"updated":true}"#, + r#"{"inserted":true}"#, + ])), + ], + ) + .unwrap(); + let mut merge = table.merge_insert(&["id"]); + merge + .when_matched_update_all(None) + .when_not_matched_insert_all(); + let result = merge + .execute(Box::new(RecordBatchIterator::new( + vec![Ok(batch)], + input_schema, + ))) + .await + .unwrap(); + assert_eq!(result.num_updated_rows, 1); + assert_eq!(result.num_inserted_rows, 1); + let output = table + .query() + .select(Select::columns(&["payload"])) + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let values = output + .iter() + .flat_map(|b| { + b.column(0) + .as_any() + .downcast_ref::() + .unwrap() + .iter() + }) + .flatten() + .collect::>(); + assert_eq!(values.len(), 2); + assert!( + values + .iter() + .any(|v| serde_json::from_str::(v).unwrap()["updated"] == true) + ); + assert!( + values + .iter() + .any(|v| serde_json::from_str::(v).unwrap()["inserted"] == true) + ); + } + fn merge_insert_test_batches(offset: i32, age: i32) -> Box { let schema = Arc::new(Schema::new(vec![ Field::new("i", DataType::Int32, false), @@ -1251,6 +1505,30 @@ mod lsm_tests { ); } + /// A table with `id`/`text` and a primary key, ready for an LSM write spec. + async fn lsm_text_table(dir: &tempfile::TempDir) -> crate::Table { + let conn = connect(dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let table = conn + .create_table("t", id_text_reader(vec![(1, "alpha")])) + .execute() + .await + .unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); + table + } + + /// Upsert `rows` through the LSM write path. + async fn upsert_text(table: &crate::Table, rows: Vec<(i64, &str)>) { + let mut builder = table.merge_insert(&[]); + builder + .when_matched_update_all(None) + .when_not_matched_insert_all(); + builder.execute(id_text_reader(rows)).await.unwrap(); + } + /// A reader of `[id: Int64, text: Utf8]` rows. fn id_text_reader(rows: Vec<(i64, &str)>) -> Box { let schema = Arc::new(Schema::new(vec![ @@ -1270,6 +1548,165 @@ mod lsm_tests { Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema)) } + /// Under the default set, an index created while a writer is open reaches + /// that writer. + /// + /// The default maintains every index the table has, so creating one changes + /// what the table maintains without anyone touching the spec. A writer left + /// on its old configs would keep building MemTables the new index does not + /// cover. + #[tokio::test] + async fn lsm_default_set_picks_up_an_index_created_while_writing() { + use crate::index::Index; + use lance_index::scalar::FullTextSearchQuery; + + let dir = tempdir().unwrap(); + let table = lsm_text_table(&dir).await; + // The default set, on a table with no FTS index yet. + table + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap(); + + // Opens the writer, whose MemTable carries no FTS index. + upsert_text(&table, vec![(99, "zebra")]).await; + + // Created with the writer still open and never reopened. + table + .create_index(&["text"], Index::FTS(Default::default())) + .execute() + .await + .unwrap(); + + upsert_text(&table, vec![(100, "zebra stripes")]).await; + + let query = FullTextSearchQuery::new("stripes".to_string()) + .with_column("text".to_string()) + .unwrap(); + let batches = table + .query() + .full_text_search(query) + .execute() + .await + .expect("the created index reached the open writer") + .try_collect::>() + .await + .unwrap(); + let found: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(found, 1, "the row written after create_index must be found"); + } + + /// A spec is replaced by unsetting it and setting the new one. + /// + /// `unset_lsm_write_spec` drains the open writer, so the rows it held + /// survive the replacement, and the writer the next write opens builds its + /// MemTables from the new set. + #[tokio::test] + async fn lsm_spec_is_replaced_by_unsetting_it_first() { + use crate::index::Index; + use lance_index::scalar::FullTextSearchQuery; + + let dir = tempdir().unwrap(); + let table = lsm_text_table(&dir).await; + table + .create_index(&["text"], Index::FTS(Default::default())) + .execute() + .await + .unwrap(); + let fts_index = table.list_indices().await.unwrap()[0].name.clone(); + + // Installed maintaining nothing, then written to: the writer opens with + // no FTS index. + table + .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new())) + .await + .unwrap(); + upsert_text(&table, vec![(99, "zebra")]).await; + + // Setting over an installed spec is refused; unset, then set. + let err = table + .set_lsm_write_spec( + LsmWriteSpec::unsharded().with_maintained_indexes(vec![fts_index.clone()]), + ) + .await + .expect_err("an installed spec cannot be set over"); + assert!( + err.to_string().contains("already set"), + "unexpected error: {err}" + ); + table.unset_lsm_write_spec().await.unwrap(); + table + .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![fts_index])) + .await + .unwrap(); + + // A row written after the replacement is answered by the index the new + // writer maintains. + upsert_text(&table, vec![(100, "zebra stripes")]).await; + + let query = FullTextSearchQuery::new("stripes".to_string()) + .with_column("text".to_string()) + .unwrap(); + let batches = table + .query() + .full_text_search(query) + .execute() + .await + .expect("the replaced spec maintains the index") + .try_collect::>() + .await + .unwrap(); + let found: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!( + found, 1, + "the row written after the replacement must be found" + ); + } + + /// A row written before the index exists is searchable once it does. + /// + /// The MemTable holding it was built without the index. Creating the index + /// replaces the writer's configs, which seals that MemTable and waits for + /// its flush, so no resident MemTable is left that cannot answer the read. + #[tokio::test] + async fn lsm_a_row_written_before_the_index_is_searchable_after_it() { + use crate::index::Index; + use lance_index::scalar::FullTextSearchQuery; + + let dir = tempdir().unwrap(); + let table = lsm_text_table(&dir).await; + // No index yet, and the default set: maintain whatever the table has. + table + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap(); + + // Opens the writer, whose MemTable therefore carries no FTS index. + upsert_text(&table, vec![(99, "zebra")]).await; + + // Built afterwards: the spec now resolves to it, the MemTable does not. + table + .create_index(&["text"], Index::FTS(Default::default())) + .execute() + .await + .unwrap(); + + let query = FullTextSearchQuery::new("zebra".to_string()) + .with_column("text".to_string()) + .unwrap(); + let batches = table + .query() + .full_text_search(query) + .execute() + .await + .expect("no resident MemTable is left that cannot answer") + .try_collect::>() + .await + .unwrap(); + let found: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(found, 1, "the row written before the index must be found"); + } + #[tokio::test] async fn lsm_read_full_text_search() { use crate::index::Index; diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index fc925c014..0f86fdcac 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -77,21 +77,36 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) let dataset = table.dataset.get().await?; if dataset.mem_wal_index_details().await?.is_some() { return Err(Error::InvalidInput { - message: "set_lsm_write_spec: an LSM write spec is already set on this table; mutation is not supported".into(), + message: "set_lsm_write_spec: an LSM write spec is already set on this \ + table and cannot be changed" + .into(), }); } } - // Before the builder borrows the dataset clone. `list_indices` merges an - // index's segments into one entry, so the result needs no dedup. - let maintained_indexes = { - let dataset = table.dataset.get().await?; - resolve_maintained_indexes( - &dataset, - &table.list_indices().await?, - spec.maintained_indexes(), - ) - .await? + // A named set is checked against the table here, where the caller can still + // be told which names exist. An unnamed set is not resolved at all: it is an + // intent lance re-reads whenever it builds a MemTable, so an index created + // later is maintained without another call. + let maintained_indexes = match spec.maintained_indexes() { + Some(requested) => { + let indices = table.list_indices().await?; + for name in requested { + if !indices.iter().any(|index| &index.name == name) { + return Err(Error::InvalidInput { + message: format!( + "maintained index '{}' does not exist on this table; it has {}", + name, + index_name_list(&indices), + ), + }); + } + } + let dataset = table.dataset.get().await?; + validate_maintained_indexes(&dataset, requested).await?; + Some(requested.to_vec()) + } + None => None, }; table.checkout_latest().await?; @@ -138,7 +153,9 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) writer_config_defaults } }; - builder = builder.maintained_indexes(maintained_indexes); + if let Some(maintained_indexes) = maintained_indexes { + builder = builder.maintained_indexes(maintained_indexes); + } for (key, value) in writer_config_defaults { builder = builder.add_writer_config_default(key, value); } @@ -147,48 +164,6 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) Ok(()) } -/// Resolve a spec's maintained-index selection against `indices`, as reported -/// by [`Table::list_indices`](crate::Table::list_indices). -/// -/// `None` means every index on the table, snapshotted now. Lance validates -/// either selection against its shard-writer rules, so a spec that installs is -/// one the MemWAL can open. -/// -/// An unmaintainable index fails an inferred set rather than being dropped from -/// it — dropping would leave the caller believing it is maintained. -async fn resolve_maintained_indexes( - dataset: &Dataset, - indices: &[IndexConfig], - requested: Option<&[String]>, -) -> Result> { - let Some(requested) = requested else { - let all: Vec = indices.iter().map(|index| index.name.clone()).collect(); - validate_maintained_indexes(dataset, &all) - .await - .map_err(|source| Error::InvalidInput { - message: format!( - "cannot maintain every index on this table: {source}. Set \ - maintained_indexes explicitly to choose from {}", - index_name_list(indices), - ), - })?; - return Ok(all); - }; - for name in requested { - if !indices.iter().any(|index| &index.name == name) { - return Err(Error::InvalidInput { - message: format!( - "maintained index '{}' does not exist on this table; it has {}", - name, - index_name_list(indices), - ), - }); - } - } - validate_maintained_indexes(dataset, requested).await?; - Ok(requested.to_vec()) -} - /// Index names for an error message. fn index_name_list(indices: &[IndexConfig]) -> String { if indices.is_empty() { @@ -298,7 +273,9 @@ fn lsm_write_spec_from_details( }; Ok(base - .with_maintained_indexes(details.maintained_indexes.clone()) + .with_maintained_indexes( + (!details.maintain_all_indexes).then(|| details.maintained_indexes.clone()), + ) .with_writer_config_defaults(details.writer_config_defaults.clone())) } @@ -460,6 +437,55 @@ impl ShardWriterCache { Ok(entry) } + /// Install `dataset`'s maintained index set on the cached writer, if one is + /// open, so a change reaches a table that is already being written to. + /// + /// Reported rather than propagated: the index is committed durably by the + /// time this runs, and the writer this process happens to hold may be fenced + /// or poisoned -- which is not a reason to fail creating an index. Any + /// writer opened afterwards reads the committed set. + /// + /// Holds the entry's read lock across the refresh, so a concurrent close + /// waits rather than retiring the writer mid-seal. Writes take the same + /// lock first, so the order matches theirs. + pub(crate) async fn refresh_maintained_indexes(&self, dataset: &Dataset) { + let entry = { + let guard = self.slot.read().await; + guard.as_ref().map(|(_, entry)| entry.clone()) + }; + let Some(entry) = entry else { + return; + }; + let guard = entry.inner.read().await; + let Some(writer) = guard.as_ref() else { + return; + }; + // Replacing the configs seals the active memtable so the next one carries + // the new indexes. The outgoing one keeps the set it was built with, so + // its flush has to land before a read: while it is still frozen, an + // indexed read sees a resident memtable that cannot answer and is + // refused. + let sealed = match dataset.refresh_mem_wal_index_configs(writer).await { + Ok(sealed) => sealed, + Err(error) => { + log::warn!( + "the maintained index set changed but the open shard writer could \ + not be refreshed, so it keeps the set it opened with: {error}" + ); + return; + } + }; + if let Some(fence) = sealed + && let Err(error) = fence.wait().await + { + log::warn!( + "the shard writer took the new maintained index set, but the memtable \ + it sealed was not flushed, so an indexed read stays refused until \ + it is: {error}" + ); + } + } + /// Snapshot the cached writer's shard for the LSM read path: its shard id, /// authoritative in-memory manifest, and active + frozen memtable refs. /// Returns `None` when no writer is currently cached (e.g. nothing has been diff --git a/rust/lancedb/src/table/query/lsm.rs b/rust/lancedb/src/table/query/lsm.rs index 86c1fe5f2..f3e40fc7b 100644 --- a/rust/lancedb/src/table/query/lsm.rs +++ b/rust/lancedb/src/table/query/lsm.rs @@ -33,7 +33,7 @@ use datafusion_physical_plan::expressions::Column; use datafusion_physical_plan::projection::ProjectionExec; use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr}; use lance::Dataset; -use lance::dataset::mem_wal::scanner::InMemoryMemTables; +use lance::dataset::mem_wal::scanner::{InMemoryMemTableRef, InMemoryMemTables}; use lance::dataset::mem_wal::{ DatasetMemWalExt, LsmScanner, ShardManifestStore, ShardSnapshot, ShardWriterConfig, }; @@ -468,20 +468,19 @@ async fn fts_plan( .to_string(), })?; - // Without a maintained in-memory FTS index for this column, the active memtable - // arm produces an empty plan (`active_source_can_execute_fts` returns false), so - // the search silently omits un-compacted documents. Reject rather than mislead. - if !index_maintained( - dataset, - column, - &details.maintained_indexes, - "InvertedIndexDetails", - ) - .await? + // Two ways the fresh tier cannot answer: the spec maintains no FTS index for + // this column, or a resident MemTable was built before the index joined the + // set. Either leaves the memtable arm an empty plan + // (`active_source_can_execute_fts` returns false), so the search silently + // omits un-compacted documents. Reject rather than mislead. + if !index_maintained(dataset, column, details, "InvertedIndexDetails").await? + || !resident_memtables_carry(&in_memory, |memtable| { + memtable.index_store.get_fts_by_column(column).is_some() + }) { return Err(Error::NotSupported { message: format!( - "the MemWAL LSM scanner full-text search requires the FTS index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise un-compacted documents are omitted. set use_lsm(false) to read the base table only" + "the MemWAL LSM scanner full-text search requires the FTS index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes) and carried by every resident MemTable; otherwise un-compacted documents are omitted. set use_lsm(false) to read the base table only" ), }); } @@ -493,6 +492,27 @@ async fn fts_plan( Ok(scanner.create_plan().await?) } +/// Whether the spec maintains the index named `name`. +/// +/// An unnamed set names nothing and covers everything, so the stored list alone +/// cannot answer this. +fn spec_maintains(details: &MemWalIndexDetails, name: &str) -> bool { + details.maintain_all_indexes || details.maintained_indexes.iter().any(|n| n == name) +} + +/// Whether every resident MemTable carries the index an arm needs. +/// +/// The spec is intent; a MemTable built before the index joined the set +/// contributes nothing to an indexed read, so its rows would go missing. +fn resident_memtables_carry( + in_memory: &HashMap, + carries: impl Fn(&InMemoryMemTableRef) -> bool, +) -> bool { + in_memory + .values() + .all(|tables| carries(&tables.active) && tables.frozen.iter().all(&carries)) +} + /// Whether an index of `type_url_suffix` covering `column` is in the MemWAL spec's /// maintained set. Only a maintained index has its catch-up tracked (so exclusion is /// gated correctly) and its in-memory arm kept current; an unmaintained base index @@ -502,7 +522,7 @@ async fn fts_plan( async fn index_maintained( dataset: &Dataset, column: &str, - maintained: &[String], + details: &MemWalIndexDetails, type_url_suffix: &str, ) -> Result { use lance::index::DatasetIndexExt; @@ -512,7 +532,7 @@ async fn index_maintained( let indices = dataset.load_indices().await?; Ok(indices.iter().any(|idx| { idx.fields.contains(&field.id) - && maintained.iter().any(|m| m == &idx.name) + && spec_maintains(details, &idx.name) && idx .index_details .as_ref() @@ -583,9 +603,7 @@ async fn arm_maintained_index_names( }) .map(|idx| idx.name.clone()) .collect(); - if let Some(name) = - resolve_single_index(segment_names, &details.maintained_indexes, arm, &column)? - { + if let Some(name) = resolve_single_index(segment_names, details, arm, &column)? { names.push(name); } } @@ -604,7 +622,7 @@ async fn arm_maintained_index_names( /// compaction watermark). fn resolve_single_index( mut names: Vec, - maintained: &[String], + details: &MemWalIndexDetails, arm: &str, column: &str, ) -> Result> { @@ -620,7 +638,7 @@ fn resolve_single_index( Ok(names .into_iter() .next() - .filter(|name| maintained.contains(name))) + .filter(|name| spec_maintains(details, name))) } /// Drop the primary-key columns Lance appends internally for dedup when the user's @@ -688,21 +706,19 @@ async fn vector_plan( None => default_vector_column(&arrow_schema, Some(query_vector.len() as i32))?, }; - // The base arm relies on the column's vector index (`fast_search`). Unless it is - // maintained, its catch-up is untracked and exclusion falls back to the - // compaction watermark — dropping compacted SSTables the (lagging) base index has - // not re-indexed. Reject rather than silently omit rows, mirroring the FTS arm. - if !index_maintained( - dataset, - &column, - &details.maintained_indexes, - "VectorIndexDetails", - ) - .await? + // The base arm relies on the column's vector index (`fast_search`). Unmaintained, + // its catch-up is untracked and exclusion falls back to the compaction watermark, + // dropping compacted SSTables the lagging base index has not re-indexed; and a + // resident MemTable built before the index joined the set carries none either. + // Reject rather than silently omit rows, mirroring the FTS arm. + if !index_maintained(dataset, &column, details, "VectorIndexDetails").await? + || !resident_memtables_carry(&in_memory, |memtable| { + memtable.index_store.get_hnsw_by_column(&column).is_some() + }) { return Err(Error::NotSupported { message: format!( - "the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only" + "the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes) and carried by every resident MemTable; otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only" ), }); } @@ -888,7 +904,10 @@ mod tests { #[test] fn resolve_single_index_dedupes_segments() { - let maintained = vec!["fts_idx".to_string()]; + let maintained = MemWalIndexDetails { + maintained_indexes: vec!["fts_idx".to_string()], + ..Default::default() + }; // Two physical segments of ONE logical index must not count as "multiple". assert_eq!( resolve_single_index( @@ -917,4 +936,33 @@ mod tests { None ); } + + /// A spec that maintains every index names none of them, so the stored list + /// is empty while every index on the table is in fact maintained. Reading + /// the list alone would refuse the read path on the default spec. + #[test] + fn an_unnamed_set_maintains_every_index() { + let all = MemWalIndexDetails { + maintained_indexes: Vec::new(), + maintain_all_indexes: true, + ..Default::default() + }; + assert!(spec_maintains(&all, "fts_idx")); + assert_eq!( + resolve_single_index(vec!["fts_idx".to_string()], &all, "full-text", "text").unwrap(), + Some("fts_idx".to_string()), + ); + + // The same empty list without the flag maintains nothing. + let none = MemWalIndexDetails { + maintained_indexes: Vec::new(), + maintain_all_indexes: false, + ..Default::default() + }; + assert!(!spec_maintains(&none, "fts_idx")); + assert_eq!( + resolve_single_index(vec!["fts_idx".to_string()], &none, "full-text", "text").unwrap(), + None, + ); + } } diff --git a/rust/lancedb/src/table/schema_evolution.rs b/rust/lancedb/src/table/schema_evolution.rs index 4d0e2815d..4990901ae 100644 --- a/rust/lancedb/src/table/schema_evolution.rs +++ b/rust/lancedb/src/table/schema_evolution.rs @@ -807,6 +807,29 @@ mod tests { // Alter Columns Tests + #[tokio::test] + async fn test_alter_columns_rejects_missing_changes() { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!(("id", Int32, [1, 2, 3])).unwrap(); + let table = conn.create_table("test", batch).execute().await.unwrap(); + let initial_version = table.version().await.unwrap(); + let initial_schema = table.schema().await.unwrap(); + + for alterations in [ + vec![ColumnAlteration::new("id".into())], + vec![ + ColumnAlteration::new("id".into()).rename("new_id".into()), + ColumnAlteration::new("id".into()), + ], + ] { + let err = table.alter_columns(&alterations).await.unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. }), "got {err:?}"); + assert!(err.to_string().contains("path 'id'")); + assert_eq!(table.version().await.unwrap(), initial_version); + assert_eq!(table.schema().await.unwrap(), initial_schema); + } + } + #[tokio::test] async fn test_alter_column_rename() { let conn = connect("memory://").execute().await.unwrap(); diff --git a/rust/lancedb/tests/duplicate_pairs.rs b/rust/lancedb/tests/duplicate_pairs.rs new file mode 100644 index 000000000..f7d7ed3a2 --- /dev/null +++ b/rust/lancedb/tests/duplicate_pairs.rs @@ -0,0 +1,750 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +use arrow_array::{ + Array, FixedSizeListArray, Float32Array, Int64Array, RecordBatch, RecordBatchIterator, + UInt64Array, +}; +use arrow_schema::{DataType, Field, Schema}; +use datafusion::prelude::SessionContext; +use datafusion_catalog::TableProvider; +use datafusion_common::Result; +use futures::TryStreamExt; +use lance::{ + Dataset, + dataset::WriteParams, + index::{DatasetIndexExt, vector::VectorIndexParams}, +}; +use lance_index::{IndexType, vector::ivf::IvfBuildParams}; +use lance_linalg::distance::MetricType; +use lancedb::table::datafusion::{ + BaseTableAdapter, + udtf::duplicate_pairs::{ + DuplicatePairsConfig, DuplicatePairsExec, DuplicatePairsResolver, + DuplicatePairsTableFunction, duplicate_pair_task_batch, plan_duplicate_pairs, + }, +}; +use std::sync::Arc; + +#[derive(Debug)] +struct Resolver(Arc); +impl DuplicatePairsResolver for Resolver { + fn resolve( + &self, + table: &str, + config: DuplicatePairsConfig, + tasks_only: bool, + ) -> Result> { + assert_eq!(table, "source"); + Ok(Arc::new(self.0.duplicate_pairs(config, tasks_only))) + } +} + +fn pairs(batches: &[RecordBatch]) -> Vec<(u64, u64, u32)> { + let mut result = Vec::new(); + for batch in batches { + let a = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let b = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let d = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..batch.num_rows() { + result.push((a.value(i), b.value(i), d.value(i).to_bits())); + } + } + result.sort_unstable(); + result +} + +#[tokio::test] +async fn sql_pairs_match_native_snapshot_and_partition_tasks() -> anyhow::Result<()> { + for partitions in [1, 8] { + let dir = tempfile::tempdir()?; + let uri = dir + .path() + .join("source.lance") + .to_str() + .unwrap() + .to_string(); + let n = 8193; + let vectors = (0..n * 2) + .flat_map(|i| { + let i = i % n; + let x = if i == n - 1 { + 0.0 + } else if i < 3 { + i as f32 + } else { + i as f32 * 10.0 + }; + [x, 0.0] + }) + .collect::>(); + let vectors = FixedSizeListArray::try_new( + Arc::new(Field::new("item", DataType::Float32, true)), + 2, + Arc::new(Float32Array::from(vectors)), + None, + )?; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("vector", vectors.data_type().clone(), false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from_iter_values(0..(n * 2) as i64)), + Arc::new(vectors), + ], + )?; + let mut ds = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &uri, + Some(WriteParams { + max_rows_per_file: n, + ..Default::default() + }), + ) + .await?; + let fragments = ds + .fragments() + .iter() + .map(|f| f.id as u32) + .collect::>(); + assert_eq!(fragments.len(), 2); + let mut segments = Vec::new(); + for (i, fragment) in fragments.into_iter().enumerate() { + let centroids = FixedSizeListArray::try_new( + Arc::new(Field::new("item", DataType::Float32, true)), + 2, + Arc::new(Float32Array::from( + (0..partitions) + .flat_map(|p| [p as f32 * 1400.0 + i as f32 * 0.125, 0.0]) + .collect::>(), + )), + None, + )?; + let ivf = IvfBuildParams { + centroids: Some(Arc::new(centroids)), + ..IvfBuildParams::new(partitions) + }; + let params = VectorIndexParams::with_ivf_flat_params(MetricType::L2, ivf); + segments.push( + ds.create_index_builder(&["vector"], IndexType::Vector, ¶ms) + .name("vector_idx".into()) + .fragments(vec![fragment]) + .execute_uncommitted() + .await?, + ); + } + ds.commit_existing_index_segments("vector_idx", "vector", segments) + .await?; + let version = ds.version().version; + let ds = Arc::new(ds); + let conn = lancedb::connect(dir.path().to_str().unwrap()) + .execute() + .await?; + let table = conn.open_table("source").execute().await?; + let resolver = Arc::new(Resolver(Arc::new( + BaseTableAdapter::try_new(table.base_table().clone()).await?, + ))); + // More partitions than this pool could hold concurrently: local + // execution must consume scoped readers sequentially. + let runtime = Arc::new( + datafusion_execution::runtime_env::RuntimeEnvBuilder::new() + .with_memory_limit(128 * 1024 * 1024, 1.0) + .build()?, + ); + let ctx = + SessionContext::new_with_config_rt(datafusion::prelude::SessionConfig::new(), runtime); + ctx.register_udtf( + "vector_duplicate_pairs", + Arc::new(DuplicatePairsTableFunction::new(resolver.clone())), + ); + ctx.register_udtf( + "vector_duplicate_pair_tasks", + Arc::new(DuplicatePairsTableFunction::task_manifest(resolver)), + ); + for threshold in [-1.0, 0.0, 1.0] { + let config = DuplicatePairsConfig { + dataset_version: version, + column: "vector".into(), + distance_threshold: threshold, + }; + let tasks = plan_duplicate_pairs(ds.clone(), &config).await?; + assert_eq!(tasks.len(), partitions * 2); + let manifest = duplicate_pair_task_batch(&ds, &config, &tasks)?; + let sql_manifest = ctx.sql(&format!("SELECT * FROM vector_duplicate_pair_tasks('source', {version}, 'vector', {threshold})")).await?.collect().await?; + assert_eq!(sql_manifest, vec![manifest]); + let expected = + lance::index::vector::dedup::find_duplicate_pairs(ds.clone(), "vector", threshold) + .await? + .try_collect::>() + .await?; + let sql = format!( + "SELECT * FROM vector_duplicate_pairs('source', {version}, 'vector', {threshold})" + ); + let actual = ctx.sql(&sql).await?.collect().await?; + assert_eq!(pairs(&actual), pairs(&expected)); + use lancedb::table::datafusion::udtf::{ + duplicate_pairs::DuplicatePairsOutput, + vector_query::{PartitionSelection, VectorQueryOptions, VectorQueryTable}, + }; + let query_options = VectorQueryOptions { + column: "vector".into(), + distance_threshold: threshold, + selection: PartitionSelection::All, + output: DuplicatePairsOutput { + scope: true, + id_column: Some("id".into()), + }, + dedup: false, + }; + let provider = VectorQueryTable::try_new(ds.clone(), query_options.clone()).await?; + let extended = ctx.read_table(Arc::new(provider))?.collect().await?; + assert_eq!(pairs(&extended), pairs(&expected)); + for batch in &extended { + let rowids = batch + .column_by_name("row_id_a") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let ids = batch + .column_by_name("id_a") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..batch.num_rows() { + let expected_id = (rowids.value(i) >> 32) as i64 * n as i64 + + (rowids.value(i) & 0xffffffff) as i64; + assert_eq!(ids.value(i), expected_id); + } + } + let mut sampled_options = query_options.clone(); + sampled_options.selection = PartitionSelection::Random { count: 1, seed: 42 }; + let sample = ctx + .read_table(Arc::new( + VectorQueryTable::try_new(ds.clone(), sampled_options.clone()).await?, + ))? + .collect() + .await?; + let replay = ctx + .read_table(Arc::new( + VectorQueryTable::try_new(ds.clone(), sampled_options).await?, + ))? + .collect() + .await?; + assert_eq!(sample, replay); + let scopes = sample + .iter() + .flat_map(|batch| { + let segments = batch + .column_by_name("segment_id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let partitions = batch + .column_by_name("partition_id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + (0..batch.num_rows()) + .map(|i| (segments.value(i).to_string(), partitions.value(i))) + .collect::>() + }) + .collect::>(); + assert!(scopes.len() <= 1); + let mut dedup_options = query_options; + dedup_options.dedup = true; + dedup_options.output = DuplicatePairsOutput::default(); + let clean_query = ctx + .read_table(Arc::new( + VectorQueryTable::try_new(ds.clone(), dedup_options).await?, + ))? + .select(vec![datafusion::prelude::col("id")])? + .collect() + .await?; + + assert_eq!( + pairs(&actual).len(), + match threshold { + -1.0 => 0, + 0.0 => 2, + _ => 8, + } + ); + let retry = ctx.sql(&sql).await?.collect().await?; + assert_eq!(pairs(&retry), pairs(&actual)); + // The same table function can be maintained through the MV unit + // lifecycle. Staging attempts do not make any rows visible. + use lancedb::materialized_view::{ + MaterializedViewDefinition, commit_partitioned_refresh, plan_partitioned_refresh, + prepare_definition, write_refresh_partition, + }; + let view_name = format!("pairs_{}", threshold.to_bits()); + let view = prepare_definition(&table, MaterializedViewDefinition::from_sql(&sql)?) + .await? + .create(&view_name) + .await?; + assert_eq!(view.table().schema().await?.fields().len(), 3); + let plan = plan_partitioned_refresh(view.table(), Some(version)) + .await? + .unwrap(); + assert_eq!(plan.units() as usize, tasks.len()); + let plan = serde_json::from_slice(&serde_json::to_vec(&plan)?)?; + let first = write_refresh_partition(view.table(), 0, &plan).await?; + assert_eq!(view.table().count_rows(None).await?, 0); + assert!( + commit_partitioned_refresh( + view.table(), + &plan, + vec![first.clone(), first.clone()], + view.incarnation() + ) + .await + .is_err() + ); + // Lost receipt: its files remain unreferenced; the retry writes + // another attempt, of which only one receipt is ever committed. + drop(write_refresh_partition(view.table(), 0, &plan).await?); + let mut units = vec![first]; + for unit in 1..plan.units() { + units.push(write_refresh_partition(view.table(), unit, &plan).await?); + } + assert_eq!(view.table().count_rows(None).await?, 0); + let outcome = + commit_partitioned_refresh(view.table(), &plan, units.clone(), view.incarnation()) + .await?; + assert_eq!(outcome.rows_written as usize, pairs(&expected).len()); + assert!( + commit_partitioned_refresh(view.table(), &plan, units, view.incarnation()) + .await + .is_err() + ); + use lancedb::query::ExecutableQuery; + let written = view + .table() + .query() + .execute() + .await? + .try_collect::>() + .await?; + assert_eq!(pairs(&written), pairs(&expected)); + // Repeating a refresh replaces the view rather than appending. + view.refresh().execute().await?; + assert_eq!( + view.table().count_rows(None).await? as usize, + pairs(&expected).len() + ); + // The complete declarative pipeline keeps direct representatives + // and materializes original rows, including isolated source rows. + use lancedb::materialized_view::write_refresh_partition_with_inputs; + use lancedb::query::QueryBase; + let definition = MaterializedViewDefinition::from_sql(&format!( + "SELECT * FROM vector_dedup('source', {version}, 'vector', {threshold})" + ))?; + let clean = prepare_definition(&table, definition) + .await? + .create(&format!("clean_{}", threshold.to_bits())) + .await?; + let clean_plan = plan_partitioned_refresh(clean.table(), Some(version)) + .await? + .unwrap(); + assert_eq!(clean_plan.dependency_units() as usize, tasks.len()); + assert_eq!(clean_plan.units() as usize, tasks.len() + 2); + let clean_plan: lancedb::materialized_view::PartitionedRefreshPlan = + serde_json::from_slice(&serde_json::to_vec(&clean_plan)?)?; + assert!( + write_refresh_partition(clean.table(), tasks.len() as u32, &clean_plan) + .await + .is_err() + ); + let mut receipts = Vec::new(); + for unit in 0..clean_plan.units() { + let dependencies = &receipts[..receipts.len().min(tasks.len())]; + if unit == 0 || unit == tasks.len() as u32 { + // Both selection and materialization can lose a receipt. + drop( + write_refresh_partition_with_inputs( + clean.table(), + unit, + &clean_plan, + dependencies, + ) + .await?, + ); + } + let receipt = write_refresh_partition_with_inputs( + clean.table(), + unit, + &clean_plan, + dependencies, + ) + .await?; + receipts.push(serde_json::from_slice(&serde_json::to_vec(&receipt)?)?); + } + assert_eq!(clean.table().count_rows(None).await?, 0); + assert!( + commit_partitioned_refresh( + clean.table(), + &clean_plan, + receipts[..receipts.len() - 1].to_vec(), + clean.incarnation() + ) + .await + .is_err() + ); + commit_partitioned_refresh(clean.table(), &clean_plan, receipts, clean.incarnation()) + .await?; + let mut edges = pairs(&expected) + .into_iter() + .map(|(a, b, _)| (a.min(b), a.max(b))) + .collect::>(); + edges.sort_unstable(); + let mut removed = std::collections::BTreeSet::new(); + for (a, b) in edges { + if !removed.contains(&a) { + removed.insert(b); + } + } + let source_ids = ds + .scan() + .project(&["id"])? + .with_row_id() + .try_into_stream() + .await? + .try_collect::>() + .await?; + let mut expected_ids = Vec::new(); + for batch in source_ids { + let ids = batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let rowids = batch + .column_by_name("_rowid") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..batch.num_rows() { + if !removed.contains(&rowids.value(i)) { + expected_ids.push(ids.value(i)); + } + } + } + let kept = clean + .table() + .query() + .select(lancedb::query::Select::columns(&["id"])) + .execute() + .await? + .try_collect::>() + .await?; + let mut actual_ids = kept + .iter() + .flat_map(|b| { + b.column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .iter() + .copied() + }) + .collect::>(); + actual_ids.sort_unstable(); + expected_ids.sort_unstable(); + let mut query_ids = clean_query + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect::>(); + query_ids.sort_unstable(); + assert_eq!(query_ids, expected_ids); + + assert_eq!(actual_ids, expected_ids); + if threshold == 1.0 { + assert!(actual_ids.contains(&0) && actual_ids.contains(&2)); + assert!(!actual_ids.contains(&1), "A-B-C retains A and C"); + } + clean.refresh().execute().await?; + assert_eq!(clean.table().count_rows(None).await?, expected_ids.len()); + assert_eq!(table.count_rows(None).await?, n * 2); + assert!( + DuplicatePairsExec::try_new( + ds.clone(), + config.clone(), + vec![tasks[0].clone(), tasks[0].clone()] + ) + .is_err() + ); + } + // Latest no longer contains B, but pinned queries must retain the same pairs. + table.delete("id = 1").await?; + let actual = ctx + .sql(&format!( + "SELECT * FROM vector_duplicate_pairs('source', {version}, 'vector', 1.0)" + )) + .await? + .collect() + .await?; + assert_eq!(pairs(&actual).len(), 8); + assert!( + ctx.sql(&format!( + "SELECT * FROM vector_duplicate_pairs('source', {version}, 'missing', 1.0)" + )) + .await? + .collect() + .await + .is_err() + ); + } + Ok(()) +} + +#[tokio::test] +async fn dedup_materializes_blob_payloads_null_vectors_and_snapshot_deletions() -> anyhow::Result<()> +{ + use arrow_array::{LargeBinaryArray, StringArray, StructArray, types::Float32Type}; + use lancedb::materialized_view::{MaterializedViewDefinition, prepare_definition}; + for stable in [false, true] { + let dir = tempfile::tempdir()?; + let conn = lancedb::connect(dir.path().to_str().unwrap()) + .execute() + .await?; + let vectors = FixedSizeListArray::from_iter_primitive::( + [ + Some(vec![Some(0.0), Some(0.0)]), + Some(vec![Some(1.0), Some(0.0)]), + Some(vec![Some(2.0), Some(0.0)]), + None, + Some(vec![Some(30.0), Some(0.0)]), + Some(vec![Some(40.0), Some(0.0)]), + ], + 2, + ); + let blob = lancedb::blob::blob("image", true); + let DataType::Struct(fields) = blob.data_type().clone() else { + unreachable!() + }; + let images = StructArray::new( + fields, + vec![ + Arc::new(LargeBinaryArray::from_iter_values( + (0..6).map(|id| vec![id; 4096]), + )), + Arc::new(StringArray::from(vec![None::<&str>; 6])), + ], + None, + ); + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("vector", vectors.data_type().clone(), true), + blob, + ])), + vec![ + Arc::new(Int64Array::from_iter_values(0..6)), + Arc::new(vectors), + Arc::new(images), + ], + )?; + let table = conn + .create_table("source", batch) + .storage_option("new_table_enable_stable_row_ids", stable.to_string()) + .execute() + .await?; + table.delete("id = 5").await?; + table + .optimize(lancedb::table::OptimizeAction::Compact { + options: lancedb::table::CompactionOptions::default(), + remap_options: None, + }) + .await?; + let uri = table.uri().await?; + let mut ds = Dataset::open(&uri).await?; + assert!( + ds.fragments().iter().all(|f| f.id > 0), + "compaction must change physical row addresses" + ); + let centroids = FixedSizeListArray::from_iter_primitive::( + [Some(vec![Some(0.0), Some(0.0)])], + 2, + ); + let ivf = IvfBuildParams { + centroids: Some(Arc::new(centroids)), + ..IvfBuildParams::new(1) + }; + ds.create_index( + &["vector"], + IndexType::Vector, + Some("vector_idx".into()), + &VectorIndexParams::with_ivf_flat_params(MetricType::L2, ivf), + true, + ) + .await?; + let version = ds.version().version; + use lancedb::table::datafusion::udtf::{ + duplicate_pairs::DuplicatePairsOutput, + vector_query::{PartitionSelection, VectorQueryOptions, VectorQueryTable}, + }; + let options = VectorQueryOptions { + column: "vector".into(), + distance_threshold: 1.0, + selection: PartitionSelection::All, + output: DuplicatePairsOutput::default(), + dedup: true, + }; + let mapped = VectorQueryTable::try_new( + Arc::new(ds.clone()), + VectorQueryOptions { + dedup: false, + output: DuplicatePairsOutput { + scope: true, + id_column: Some("id".into()), + }, + ..options.clone() + }, + ) + .await?; + let mapped = SessionContext::new() + .read_table(Arc::new(mapped))? + .collect() + .await?; + let mut mapped_edges = Vec::new(); + for batch in mapped { + let a = batch + .column_by_name("id_a") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let b = batch + .column_by_name("id_b") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..batch.num_rows() { + mapped_edges.push((a.value(i).min(b.value(i)), a.value(i).max(b.value(i)))); + } + } + mapped_edges.sort_unstable(); + assert_eq!(mapped_edges, [(0, 1), (1, 2)]); + let provider = VectorQueryTable::try_new(Arc::new(ds.clone()), options).await?; + let query_rows = SessionContext::new() + .read_table(Arc::new(provider))? + .collect() + .await?; + let mut query_ids = query_rows + .iter() + .flat_map(|batch| { + batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect::>(); + query_ids.sort_unstable(); + assert_eq!(query_ids, [0, 2, 3, 4]); + for batch in &query_rows { + let ids = batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let images = batch + .column_by_name("image") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..batch.num_rows() { + assert_eq!(images.value(i), vec![ids.value(i) as u8; 4096]); + } + } + let definition = MaterializedViewDefinition::from_sql(&format!( + "SELECT * FROM vector_dedup('source', {version}, 'vector', 1)" + ))?; + let clean = prepare_definition(&table, definition) + .await? + .create("clean") + .await?; + // A later source delete does not change the declared result snapshot. + table.delete("id = 0").await?; + let adapter = BaseTableAdapter::try_new(table.base_table().clone()).await?; + let explicit = adapter.vector_query_snapshot(Some(version)).await?; + let latest = adapter.vector_query_snapshot(None).await?; + assert_eq!(explicit.version().version, version); + assert!(latest.version().version > version); + assert_eq!(explicit.count_rows(None).await?, 5); + assert_eq!(latest.count_rows(None).await?, 4); + clean.refresh().execute().await?; + assert_eq!(clean.table().count_rows(None).await?, 4); + let output = Dataset::open(&clean.table().uri().await?).await?; + let batches = output + .scan() + .blob_handling(lance_core::datatypes::BlobHandling::AllBinary) + .try_into_stream() + .await? + .try_collect::>() + .await?; + let mut ids = Vec::new(); + for batch in batches { + let row_ids = batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let images = batch + .column_by_name("image") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..batch.num_rows() { + let id = row_ids.value(i); + assert_eq!(images.value(i), vec![id as u8; 4096]); + ids.push(id); + } + } + ids.sort_unstable(); + assert_eq!(ids, [0, 2, 3, 4]); + assert_eq!( + table.count_rows(None).await?, + 4, + "refresh must not mutate the source" + ); + } + Ok(()) +} diff --git a/rust/lancedb/tests/first_class_function_slice2.rs b/rust/lancedb/tests/first_class_function_slice2.rs index 14ceb511d..380766e03 100644 --- a/rust/lancedb/tests/first_class_function_slice2.rs +++ b/rust/lancedb/tests/first_class_function_slice2.rs @@ -97,13 +97,16 @@ async fn local_function_catalog_operations_return_stable_not_supported() { )) .unwrap(); - let create_error = connection.create_function_async(request).await.unwrap_err(); + let create_error = connection + .create_function_async(request, &[]) + .await + .unwrap_err(); let lookup_error = connection - .get_function("normalize_score", "1") + .get_function("normalize_score", "1", &[]) .await .unwrap_err(); let drop_error = connection - .drop_function("normalize_score", "1") + .drop_function("normalize_score", "1", &[]) .await .unwrap_err(); for error in [create_error, lookup_error, drop_error] {