Compare commits

..
Author SHA1 Message Date
Lance Release 2761e6108e Bump version: 0.38.0-beta.11 → 0.38.0-beta.12 2026-08-28 00:51:15 +00:00
87 changed files with 12258 additions and 5987 deletions
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.39.0-beta.1"
current_version = "0.38.0-beta.12"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
-24
View File
@@ -44,27 +44,3 @@ updates:
python-deps:
patterns:
- "*"
# The npm ecosystem covers pnpm lockfiles. There are two separate installs:
# the bindings themselves and the examples, which have their own lockfile.
# As with cargo and pip above, only bump the lockfile — the version ranges
# in package.json are our consumers' constraints, not ours.
- package-ecosystem: npm
directory: /nodejs
schedule:
interval: weekly
versioning-strategy: lockfile-only
groups:
nodejs-deps:
patterns:
- "*"
- package-ecosystem: npm
directory: /nodejs/examples
schedule:
interval: weekly
versioning-strategy: lockfile-only
groups:
nodejs-examples-deps:
patterns:
- "*"
+4 -10
View File
@@ -29,14 +29,12 @@ jobs:
steps:
- uses: actions/setup-node@v6
with:
node-version: "24"
- uses: pnpm/action-setup@v6
with:
version: 11.1.1
node-version: "18"
# These rules are disabled because Github will always ensure there
# is a blank line between the title and the body and Github will
# word wrap the description field to ensure a reasonable max line
# length.
- run: npm install @commitlint/config-conventional
- run: >
echo 'module.exports = {
"rules": {
@@ -45,11 +43,7 @@ jobs:
"body-leading-blank": [0, "always"]
}
}' > .commitlintrc.js
- run: >
pnpm dlx
--package @commitlint/cli@21.2.2
--package @commitlint/config-conventional@21.2.2
commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
- run: npx commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
env:
COMMIT_MSG: >
${{ github.event.pull_request.title }}
@@ -60,7 +54,7 @@ jobs:
with:
script: |
const message = `**ACTION NEEDED**
Lance follows the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) for release automation.
The PR title and description are used as the merge commit message.\
+1 -1
View File
@@ -56,7 +56,7 @@ jobs:
uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0
with:
# Restricted to http(s) on purpose. Much of docs/src is generated
# API reference (the js/ tree comes from `pnpm run docs` in nodejs)
# API reference (the js/ tree comes from `npm run docs` in nodejs)
# and the hand-written pages use mkdocstrings cross-references and
# nav-relative paths that only resolve in the site mkdocs builds,
# not in this checkout, so relative links would be reported as
+3 -1
View File
@@ -55,7 +55,9 @@ jobs:
- name: Set up node
uses: actions/setup-node@v6
with:
node-version: 24
node-version: 20
cache: 'npm'
cache-dependency-path: docs/package-lock.json
- name: Install node dependencies
working-directory: nodejs
run: |
+13 -11
View File
@@ -47,8 +47,9 @@ jobs:
version: 11.1.1
- uses: actions/setup-node@v6
with:
# Build on a supported LTS; the matrix job below covers every
# Node version the library claims to support.
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October. The library itself still supports Node >= 18
# (see test matrix below).
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -83,7 +84,7 @@ jobs:
timeout-minutes: 30
strategy:
matrix:
node-version: [ "22", "24", "26" ]
node-version: [ "18", "20" ]
runs-on: "ubuntu-22.04"
defaults:
run:
@@ -100,9 +101,9 @@ jobs:
- uses: actions/setup-node@v6
name: Setup Node.js 24 for build
with:
# Build and install once on a fixed version so the generated docs
# are identical across matrix legs; the tests below then run on each
# supported Node version.
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October. Build/install runs on Node 24; tests run on the
# matrix version below using direct jest invocation.
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -151,9 +152,9 @@ jobs:
S3_TEST: "1"
# Newer @smithy/core uses dynamic ESM imports.
NODE_OPTIONS: "--experimental-vm-modules"
# Invoke the installed jest binary directly; the pnpm shim is set up
# against the build-phase Node, not the version selected above.
run: node_modules/.bin/jest --verbose
# Invoke jest directly because pnpm 11 itself requires Node 22+
# while the matrix tests on older Node versions.
run: npx jest --verbose
- name: Test examples
working-directory: ./
env:
@@ -163,7 +164,7 @@ jobs:
run: |
python ci/mock_openai.py &
cd nodejs/examples
node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose
npx jest --testEnvironment jest-environment-node-single-context --verbose
macos:
timeout-minutes: 30
# macos-15 ships a newer linker; the older macos-14 linker fails to insert
@@ -184,7 +185,8 @@ jobs:
version: 11.1.1
- uses: actions/setup-node@v6
with:
# pnpm 11 requires Node >= 22.13.
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October.
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
+8 -7
View File
@@ -168,7 +168,8 @@ jobs:
- name: Setup node
uses: actions/setup-node@v6
with:
# pnpm 11 requires Node >= 22.13.
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October.
node-version: 24
cache: pnpm
cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -250,7 +251,7 @@ jobs:
run: |
set -e
${{ matrix.settings.pre_build }}
node_modules/.bin/napi build --platform --release \
npx napi build --platform --release \
--features ${{ matrix.settings.features }} \
--target ${{ matrix.settings.target }} \
--dts ../lancedb/native.d.ts \
@@ -270,7 +271,7 @@ jobs:
- name: Build
run: |
${{ matrix.settings.pre_build }}
node_modules/.bin/napi build --platform --release \
npx napi build --platform --release \
--features ${{ matrix.settings.features }} \
--target ${{ matrix.settings.target }} \
--dts ../lancedb/native.d.ts \
@@ -338,7 +339,7 @@ jobs:
- target: aarch64-unknown-linux-gnu
host: ubuntu-2404-8x-arm64
node:
- '22'
- '20'
runs-on: ${{ matrix.settings.host }}
defaults:
run:
@@ -384,9 +385,9 @@ jobs:
- name: Move built files
run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
- name: Test bindings
# Invoke the installed jest binary directly; the pnpm shim is set up
# against the install-phase Node, not the version selected above.
run: node_modules/.bin/jest --verbose
# Invoke jest directly because pnpm 11 itself requires Node 22+
# while the matrix tests on older Node versions.
run: npx jest --verbose
publish:
name: Publish
runs-on: ubuntu-latest
@@ -0,0 +1,22 @@
name: Update package-lock.json
on:
workflow_dispatch:
permissions:
contents: read
jobs:
publish:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: main
persist-credentials: false
fetch-depth: 0
lfs: true
- uses: ./.github/workflows/update_package_lock
with:
github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
@@ -0,0 +1,22 @@
name: Update NodeJs package-lock.json
on:
workflow_dispatch:
permissions:
contents: read
jobs:
publish:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: main
persist-credentials: false
fetch-depth: 0
lfs: true
- uses: ./.github/workflows/update_package_lock_nodejs
with:
github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+1 -4
View File
@@ -20,10 +20,7 @@ repos:
hooks:
- id: local-biome-check
name: biome check
# Use the biome from nodejs/package.json rather than a separately
# pinned one: the two drifted apart and disagreed on formatting, so
# this hook rejected code that `pnpm lint` accepted.
entry: nodejs/node_modules/.bin/biome check --config-path nodejs/biome.json nodejs/
entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
language: system
types: [text]
files: "nodejs/.*"
+3 -3
View File
@@ -38,7 +38,7 @@ Before committing changes, run formatting for every language you touched. At min
* Rust changes: run `cargo fmt --all`.
* Python changes: run `ruff format .` and `ruff check .` from the repository root,
and run targeted tests through `cd python && uv run ...`.
* TypeScript changes: run the relevant `pnpm` lint, format, build, and docs commands in `nodejs`.
* TypeScript changes: run the relevant `npm`/`pnpm` lint, format, build, and docs commands in `nodejs`.
Before creating a PR, the exact value passed to `gh pr create --title` must follow
Conventional Commits, such as `fix: support nested field paths in native index creation`
@@ -101,12 +101,12 @@ Python bindings changes:
TypeScript bindings changes:
1. Add napi-rs method binding on `Table` in `nodejs/src/table.rs`.
2. Run `pnpm build` to generate TypeScript definitions.
2. Run `npm run build` to generate TypeScript definitions.
3. Add typescript method on abstract class `Table` in `nodejs/src/table.ts`.
4. Add concrete method on `LocalTable` class in `nodejs/src/native_table.ts`.
* Note: despite the name, this class is also used for remote tables.
5. Add test in `nodejs/__test__/table.test.ts`.
6. Run `pnpm run docs` to generate TypeScript documentation.
6. Run `npm run docs` to generate TypeScript documentation.
## Python API reference
Generated
+73 -76
View File
@@ -535,9 +535,9 @@ dependencies = [
[[package]]
name = "async-trait"
version = "0.1.92"
version = "0.1.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
dependencies = [
"proc-macro2",
"quote",
@@ -1443,9 +1443,9 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
[[package]]
name = "bytemuck"
version = "1.25.2"
version = "1.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
dependencies = [
"bytemuck_derive",
]
@@ -1597,9 +1597,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chacha20"
version = "0.10.2"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601"
dependencies = [
"cfg-if 1.0.4",
"cpufeatures 0.3.0",
@@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arc-swap",
"arrow",
@@ -4888,8 +4888,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4911,7 +4911,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4925,7 +4925,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -4934,8 +4934,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arrayref",
"crunchy",
@@ -4945,8 +4945,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4983,8 +4983,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arrow",
"arrow-array",
@@ -5000,7 +5000,6 @@ dependencies = [
"datafusion-functions",
"datafusion-physical-expr",
"futures",
"half",
"jsonb",
"lance-arrow",
"lance-core",
@@ -5014,8 +5013,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arrow",
"arrow-array",
@@ -5032,8 +5031,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"proc-macro2",
"quote",
@@ -5042,8 +5041,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5076,8 +5075,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5108,8 +5107,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arc-swap",
"arrow",
@@ -5173,8 +5172,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5196,8 +5195,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arrow",
"arrow-array",
@@ -5237,8 +5236,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5252,8 +5251,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arrow",
"async-trait",
@@ -5265,8 +5264,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5305,9 +5304,9 @@ dependencies = [
[[package]]
name = "lance-namespace-reqwest-client"
version = "0.11.1"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d06b1fbb5d41f93bc652b61e2872af92e8a6c5f6b4ce8839a8ecfa05365d359"
checksum = "0a030196da1c994b63a96a4f0bf5b0cfa459fe6dadc9e962320246ca328da22a"
dependencies = [
"reqwest 0.12.28",
"serde",
@@ -5319,8 +5318,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5334,8 +5333,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arrow",
"arrow-array",
@@ -5375,8 +5374,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5389,8 +5388,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
dependencies = [
"frostem",
"icu_segmenter",
@@ -5403,7 +5402,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.39.0-beta.1"
version = "0.38.0-beta.11"
dependencies = [
"ahash",
"anyhow",
@@ -5491,7 +5490,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.39.0-beta.1"
version = "0.38.0-beta.11"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5516,7 +5515,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.39.0-beta.1"
version = "0.38.0-beta.11"
dependencies = [
"arrow",
"async-trait",
@@ -5749,9 +5748,9 @@ dependencies = [
[[package]]
name = "log"
version = "0.4.34"
version = "0.4.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
[[package]]
name = "loom"
@@ -6002,9 +6001,9 @@ dependencies = [
[[package]]
name = "moka"
version = "0.12.16"
version = "0.12.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9"
checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046"
dependencies = [
"async-lock",
"crossbeam-channel",
@@ -6098,15 +6097,14 @@ dependencies = [
[[package]]
name = "napi"
version = "3.12.2"
version = "3.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58c5f4d5375213fdb7be2655e152386e82f026f9a5ba36a75556e11359aafe09"
checksum = "de33522036981030a75c231829566bc63414e08101a6f5ff4ac6cef19c8e0941"
dependencies = [
"bitflags 2.11.1",
"chrono",
"ctor 1.0.12",
"futures",
"libc",
"napi-build",
"napi-sys",
"nohash-hasher",
@@ -6118,15 +6116,15 @@ dependencies = [
[[package]]
name = "napi-build"
version = "2.4.1"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60fdf9b392c50e7c4170fa633bd909490ed7835cea4c046776d1a4dd8d2ae0ab"
checksum = "5282704fbe8d49b0cf8b08e3f33233416a528658f205c7e5ace63b582de0b11c"
[[package]]
name = "napi-derive"
version = "3.6.3"
version = "3.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fa55ea69990c90b888e9e77044410e304ce7f35de599dc6d0b5c1923d2e59af"
checksum = "4d5c9c02556ea6dc99dffd36c1ce60141411657438501a125b675776d011ce92"
dependencies = [
"convert_case",
"ctor 1.0.12",
@@ -6138,9 +6136,9 @@ dependencies = [
[[package]]
name = "napi-derive-backend"
version = "6.1.2"
version = "6.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df4056ac7c18e4438ccf0edaed4340ca0d269278c8ec19284f7b23cb039fd0ae"
checksum = "d60b5d773ad46c698c8cc2cd9fde0b283d39cbb7f71c04bee633c7bdba4423bd"
dependencies = [
"convert_case",
"proc-macro2",
@@ -8603,9 +8601,9 @@ dependencies = [
[[package]]
name = "roaring"
version = "0.11.5"
version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1"
checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9"
dependencies = [
"bytemuck",
"byteorder",
@@ -9065,9 +9063,9 @@ dependencies = [
[[package]]
name = "serde_with"
version = "3.22.0"
version = "3.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a"
checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c"
dependencies = [
"base64 0.22.1",
"bs58",
@@ -9075,7 +9073,6 @@ dependencies = [
"hex",
"indexmap 1.9.3",
"indexmap 2.14.0",
"jiff",
"schemars 0.9.0",
"schemars 1.2.1",
"serde_core",
@@ -9086,9 +9083,9 @@ dependencies = [
[[package]]
name = "serde_with_macros"
version = "3.22.0"
version = "3.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46"
checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660"
dependencies = [
"darling 0.23.0",
"proc-macro2",
@@ -10455,9 +10452,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
version = "1.26.0"
version = "1.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812"
checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
dependencies = [
"getrandom 0.4.2",
"js-sys",
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
+1 -1
View File
@@ -5,5 +5,5 @@ licenses:
cd python && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
cd python && uv sync --all-extras && uv tool run pip-licenses --python .venv/bin/python --format=markdown --with-urls --output-file=PYTHON_THIRD_PARTY_LICENSES.md
cd nodejs && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
cd nodejs && pnpm dlx license-checker@25 --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
cd nodejs && npx license-checker --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
cd java && ./mvnw license:aggregate-add-third-party -q
+6 -2
View File
@@ -12,12 +12,16 @@ done
# This updates the lockfile without building
cargo metadata --quiet > /dev/null
pushd nodejs || exit 1
npm install --package-lock-only --silent
popd
if git diff --quiet --exit-code; then
echo "No lockfile changes to commit; skipping amend."
elif $AMEND; then
git add Cargo.lock
git add Cargo.lock nodejs/package-lock.json
git commit --amend --no-edit
else
git add Cargo.lock
git add Cargo.lock nodejs/package-lock.json
git commit -m "Update lockfiles"
fi
+11 -1
View File
@@ -131,13 +131,18 @@ allow = [
"BSD-3-Clause",
"ISC",
"Unicode-3.0",
"Unicode-DFS-2016",
"Zlib",
"CC0-1.0",
"MPL-2.0",
"BSL-1.0",
"OpenSSL",
# 0BSD ("BSD Zero Clause") is effectively public domain — no attribution
# required. Pulled in by `mock_instant`.
"0BSD",
# bzip2-1.0.6 is the permissive upstream bzip2 license (BSD-like). Pulled
# in by `libbz2-rs-sys`, the pure-Rust bzip2 implementation.
"bzip2-1.0.6",
# CDLA-Permissive-2.0 is a permissive data license used by `webpki-roots`
# for the Mozilla CA root bundle. Data-only, distribution-compatible.
"CDLA-Permissive-2.0",
@@ -145,7 +150,12 @@ allow = [
confidence-threshold = 0.8
# Per-crate license exceptions: allow a license for a specific crate only,
# rather than globally via the `allow` list above.
exceptions = []
exceptions = [
# CDDL-1.0 (copyleft) is pulled in only as a dev/profiling dependency via
# `inferno` -> `pprof` -> `lance-testing`; it is a test dependency that we
# do not distribute, so scope the allowance to `inferno` alone.
{ allow = ["CDDL-1.0"], crate = "inferno" },
]
# Crates whose license cannot be determined from Cargo metadata but whose
# license we've manually confirmed from upstream. Keep this list minimal.
[[licenses.clarify]]
+8 -11
View File
@@ -47,24 +47,22 @@ pytest -vv python/tests/docs
### Checking typescript examples
The examples depend on `@lancedb/lancedb` at `file:../dist`, so the package must be
built before running the tests. This uses pnpm; see the
[Typescript contributing guide](../nodejs/CONTRIBUTING.md) for the toolchain setup.
The `@lancedb/lancedb` package must be built before running the tests:
```shell
pushd nodejs
pnpm install
pnpm build
npm ci
npm run build
popd
```
Then you can run the examples by going to the `nodejs/examples` directory, which is a
separate pnpm package with its own lockfile:
Then you can run the examples by going to the `nodejs/examples` directory and
running the tests like a normal npm package:
```shell
pushd nodejs/examples
pnpm install
pnpm test
npm ci
npm test
popd
```
@@ -86,7 +84,6 @@ The new files should be checked into the repository.
```shell
pushd nodejs
# `pnpm docs` would invoke pnpm's built-in `docs` command, not the script.
pnpm run docs
npm run docs
popd
```
-9
View File
@@ -446,15 +446,6 @@ paths:
properties:
column:
type: string
name:
type: string
description: Optional name for the created index.
replace:
type: boolean
default: true
description: |
Whether to replace an existing index with the same resolved
name. Defaults to true.
metric_type:
type: string
nullable: false
+135
View File
@@ -0,0 +1,135 @@
{
"name": "lancedb-docs-test",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "lancedb-docs-test",
"version": "1.0.0",
"license": "Apache 2",
"dependencies": {
"apache-arrow": "file:../node/node_modules/apache-arrow",
"vectordb": "file:../node"
},
"devDependencies": {
"@types/node": "^20.11.8",
"typescript": "^5.3.3"
}
},
"../node": {
"name": "vectordb",
"version": "0.21.2-beta.0",
"cpu": [
"x64",
"arm64"
],
"license": "Apache-2.0",
"os": [
"darwin",
"linux",
"win32"
],
"dependencies": {
"@neon-rs/load": "^0.0.74",
"axios": "^1.4.0"
},
"devDependencies": {
"@neon-rs/cli": "^0.0.160",
"@types/chai": "^4.3.4",
"@types/chai-as-promised": "^7.1.5",
"@types/mocha": "^10.0.1",
"@types/node": "^18.16.2",
"@types/sinon": "^10.0.15",
"@types/temp": "^0.9.1",
"@types/uuid": "^9.0.3",
"@typescript-eslint/eslint-plugin": "^5.59.1",
"apache-arrow-old": "npm:apache-arrow@13.0.0",
"cargo-cp-artifact": "^0.1",
"chai": "^4.3.7",
"chai-as-promised": "^7.1.1",
"eslint": "^8.39.0",
"eslint-config-standard-with-typescript": "^34.0.1",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-n": "^15.7.0",
"eslint-plugin-promise": "^6.1.1",
"mocha": "^10.2.0",
"openai": "^4.24.1",
"sinon": "^15.1.0",
"temp": "^0.9.4",
"ts-node": "^10.9.1",
"ts-node-dev": "^2.0.0",
"typedoc": "^0.24.7",
"typedoc-plugin-markdown": "^3.15.3",
"typescript": "^5.1.0",
"uuid": "^9.0.0"
},
"optionalDependencies": {
"@lancedb/vectordb-darwin-arm64": "0.21.2-beta.0",
"@lancedb/vectordb-darwin-x64": "0.21.2-beta.0",
"@lancedb/vectordb-linux-arm64-gnu": "0.21.2-beta.0",
"@lancedb/vectordb-linux-x64-gnu": "0.21.2-beta.0",
"@lancedb/vectordb-win32-x64-msvc": "0.21.2-beta.0"
},
"peerDependencies": {
"@apache-arrow/ts": "^14.0.2",
"apache-arrow": "^14.0.2"
}
},
"../node/node_modules/apache-arrow": {
"version": "14.0.2",
"license": "Apache-2.0",
"dependencies": {
"@types/command-line-args": "5.2.0",
"@types/command-line-usage": "5.0.2",
"@types/node": "20.3.0",
"@types/pad-left": "2.1.1",
"command-line-args": "5.2.1",
"command-line-usage": "7.0.1",
"flatbuffers": "23.5.26",
"json-bignum": "^0.0.3",
"pad-left": "^2.1.0",
"tslib": "^2.5.3"
},
"bin": {
"arrow2csv": "bin/arrow2csv.js"
}
},
"node_modules/@types/node": {
"version": "20.11.8",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.8.tgz",
"integrity": "sha512-i7omyekpPTNdv4Jb/Rgqg0RU8YqLcNsI12quKSDkRXNfx7Wxdm6HhK1awT3xTgEkgxPn3bvnSpiEAc7a7Lpyow==",
"dev": true,
"dependencies": {
"undici-types": "~5.26.4"
}
},
"node_modules/apache-arrow": {
"resolved": "../node/node_modules/apache-arrow",
"link": true
},
"node_modules/typescript": {
"version": "5.3.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz",
"integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==",
"dev": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=14.17"
}
},
"node_modules/undici-types": {
"version": "5.26.5",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
"dev": true
},
"node_modules/vectordb": {
"resolved": "../node",
"link": true
}
}
}
+20
View File
@@ -0,0 +1,20 @@
{
"name": "lancedb-docs-test",
"version": "1.0.0",
"description": "auto-generated tests from doc",
"author": "dev@lancedb.com",
"license": "Apache 2",
"dependencies": {
"apache-arrow": "file:../node/node_modules/apache-arrow",
"vectordb": "file:../node"
},
"scripts": {
"build": "tsc -b && cd ../node && npm run build-release",
"example": "npm run build && node",
"test": "npm run build && ls dist/*.js | xargs -n 1 node"
},
"devDependencies": {
"@types/node": "^20.11.8",
"typescript": "^5.3.3"
}
}
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.39.0-beta.1</version>
<version>0.38.0-beta.12</version>
</dependency>
```
-8
View File
@@ -676,17 +676,9 @@ List all the versions of the table
abstract mergeInsert(on): MergeInsertBuilder
```
Create a [MergeInsertBuilder](MergeInsertBuilder.md), which combines new data with the
existing table in a single transaction — inserting, updating and deleting
rows depending on how they match.
#### Parameters
* **on**: `string` \| `string`[]
The column, or columns, to match source rows against target
rows on. Typically a key or id column. Several columns match on the
composite key: a source row updates a target row only when it agrees on
every one of them.
#### Returns
+1 -1
View File
@@ -50,7 +50,7 @@ If this is false, and another index already exists on the same columns
and the same name, then an error will be returned. This is true even if
that index is out of date.
The default is false
The default is true
***
@@ -50,16 +50,6 @@ projections: [string, string][];
***
### sourceNamespace
```ts
sourceNamespace: string[];
```
Namespace holding the source table; empty is the root namespace.
***
### sourceTable
```ts
+17
View File
@@ -0,0 +1,17 @@
{
"include": [
"src/*.ts",
],
"compilerOptions": {
"target": "es2022",
"module": "nodenext",
"declaration": true,
"outDir": "./dist",
"strict": true,
"allowJs": true,
"resolveJsonModule": true,
},
"exclude": [
"./dist/*",
]
}
+1 -1
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.39.0-beta.1</version>
<version>0.38.0-beta.12</version>
<relativePath>../pom.xml</relativePath>
</parent>
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.39.0-beta.1</version>
<version>0.38.0-beta.12</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>12.0.0-beta.11</lance-core.version>
<lance-core.version>12.0.0-beta.2</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.39.0-beta.1"
version = "0.38.0-beta.12"
publish = false
license.workspace = true
description.workspace = true
-22
View File
@@ -48,28 +48,6 @@ describe("materialized views", () => {
expect(definitionFromMetadata(safe, "v").limit).toBe(42);
});
it("reads the namespaced select kind and refuses unknown kinds", () => {
// "namespaced_select" is the namespaced form of "select": same shape, a
// separate kind so readers that predate it refuse instead of resolving
// the source at the root.
const namespaced = new Map([
[
DEFINITION_META_KEY,
'{"kind":"namespaced_select","source_table":"people","source_namespace":["ns"]}',
],
]);
const definition = definitionFromMetadata(namespaced, "v");
expect(definition.sourceTable).toBe("people");
expect(definition.sourceNamespace).toEqual(["ns"]);
const unknown = new Map([
[DEFINITION_META_KEY, '{"kind":"select_v3","source_table":"people"}'],
]);
expect(() => definitionFromMetadata(unknown, "v")).toThrow(
/cannot refresh/,
);
});
it("creates, refreshes and queries a view", async () => {
const view = await db.createMaterializedView("adults", "people", {
select: ["name", ["shout", "upper(name)"]],
+2 -2
View File
@@ -5,8 +5,8 @@ import packageJson = require("../package.json");
describe("package metadata", () => {
it("requires Node.js type declarations compatible with the runtime", () => {
expect(packageJson.engines.node).toBe(">= 22");
expect(packageJson.peerDependencies["@types/node"]).toBe(">=22");
expect(packageJson.engines.node).toBe(">= 18");
expect(packageJson.peerDependencies["@types/node"]).toBe(">=18");
expect(packageJson.peerDependenciesMeta["@types/node"]).toEqual({
optional: true,
});
+2 -9
View File
@@ -3,7 +3,6 @@
import * as http from "http";
import { RequestListener } from "http";
import packageJson = require("../package.json");
import {
ClientConfig,
Connection,
@@ -71,13 +70,7 @@ async function withMockDatabase(
try {
await callback(db);
} finally {
// `close()` alone leaves the port bound until keep-alive sockets drain, so
// a single failing test would cascade into EADDRINUSE for every test after
// it. Destroy the connections and wait for the port to actually be free.
await new Promise<void>((resolve) => {
server.closeAllConnections();
server.close(() => resolve());
});
server.close();
}
}
@@ -138,7 +131,7 @@ describe("remote connection", () => {
(req, res) => {
expect(req.headers["x-api-key"]).toEqual("fake");
expect(req.headers["user-agent"]).toEqual(
`LanceDB-Node-Client/${packageJson.version}`,
`LanceDB-Node-Client/${process.env.npm_package_version}`,
);
const body = JSON.stringify({ tables: [] });
+6 -35
View File
@@ -737,12 +737,11 @@ it("should query documents with LangChain PDF metadata", async () => {
describe("merge insert", () => {
let tmpDir: tmp.DirResult;
let conn: Connection;
let table: Table;
beforeEach(async () => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
conn = await connect(tmpDir.name);
const conn = await connect(tmpDir.name);
table = await conn.createTable("some_table", [
{ a: 1, b: "a" },
@@ -780,38 +779,6 @@ describe("merge insert", () => {
expect(result.map((row) => ({ ...row }))).toEqual(expected);
});
test("upsert on a composite key", async () => {
const composite = await conn.createTable("composite", [
{ shard: "a", id: 1, val: "x" },
{ shard: "a", id: 2, val: "y" },
{ shard: "b", id: 1, val: "z" },
]);
// ("a", 1) matches an existing row and updates it. ("b", 2) agrees with an
// existing row on each key column separately but on neither pair, so it is
// an insert.
const mergeInsertRes = await composite
.mergeInsert(["shard", "id"])
.whenMatchedUpdateAll()
.whenNotMatchedInsertAll()
.execute([
{ shard: "a", id: 1, val: "X" },
{ shard: "b", id: 2, val: "W" },
]);
expect(mergeInsertRes.numUpdatedRows).toBe(1);
expect(mergeInsertRes.numInsertedRows).toBe(1);
const result = (await composite.toArrow())
.toArray()
.sort((a, b) => a.shard.localeCompare(b.shard) || a.id - b.id);
expect(result.map((row) => ({ ...row }))).toEqual([
{ shard: "a", id: 1, val: "X" },
{ shard: "a", id: 2, val: "y" },
{ shard: "b", id: 1, val: "z" },
{ shard: "b", id: 2, val: "W" },
]);
});
test("conditional update", async () => {
const newData = [
{ a: 2, b: "x" },
@@ -1526,7 +1493,11 @@ describe("When creating an index", () => {
it("should allow me to replace (or not) an existing index", async () => {
await tbl.createIndex("id");
await expect(tbl.createIndex("id")).rejects.toThrow("already exists");
// Default is replace=true
await tbl.createIndex("id");
await expect(tbl.createIndex("id", { replace: false })).rejects.toThrow(
"already exists",
);
await tbl.createIndex("id", { replace: true });
});
+1 -2
View File
@@ -8,8 +8,7 @@
"//1": "--experimental-vm-modules is needed to run jest with sentence-transformers",
"//2": "--testEnvironment is needed to run jest with sentence-transformers",
"//3": "See: https://github.com/huggingface/transformers.js/issues/57",
"//4": "jest is invoked by its JS entry, not node_modules/.bin/jest: under pnpm that path is a shell shim, which `node` cannot execute",
"test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testEnvironment jest-environment-node-single-context --verbose",
"test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
"lint": "biome check *.ts && biome format *.ts",
"lint-ci": "biome ci .",
"lint-fix": "biome check --write *.ts && pnpm format",
+1 -1
View File
@@ -841,7 +841,7 @@ export interface IndexOptions {
* and the same name, then an error will be returned. This is true even if
* that index is out of date.
*
* The default is false
* The default is true
*/
replace?: boolean;
+1 -5
View File
@@ -19,8 +19,6 @@ export interface MaterializedViewDefinition {
limit?: number;
/** Source columns the projections and filter read. */
inputs: string[];
/** Namespace holding the source table; empty is the root namespace. */
sourceNamespace: string[];
}
/**
@@ -80,8 +78,7 @@ export function definitionFromMetadata(
}
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
const value: any = JSON.parse(raw);
// "namespaced_select" keeps older readers from resolving the source at root.
if (value.kind !== "select" && value.kind !== "namespaced_select") {
if (value.kind !== "select") {
throw new Error(
`materialized view '${name}' is defined by '${value.kind}', which this ` +
"version of lancedb cannot refresh",
@@ -106,7 +103,6 @@ export function definitionFromMetadata(
filter: value.filter ?? undefined,
limit,
inputs: value.inputs ?? [],
sourceNamespace: value.source_namespace ?? [],
};
}
-10
View File
@@ -919,16 +919,6 @@ export abstract class Table {
/** Return the table as an arrow table */
abstract toArrow(): Promise<ArrowTable>;
/**
* Create a {@link MergeInsertBuilder}, which combines new data with the
* existing table in a single transaction — inserting, updating and deleting
* rows depending on how they match.
*
* @param on - The column, or columns, to match source rows against target
* rows on. Typically a key or id column. Several columns match on the
* composite key: a source row updates a target row only when it agrees on
* every one of them.
*/
abstract mergeInsert(on: string | string[]): MergeInsertBuilder;
/** List all the stats of a specified index
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.39.0-beta.1",
"version": "0.38.0-beta.12",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.39.0-beta.1",
"version": "0.38.0-beta.12",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.39.0-beta.1",
"version": "0.38.0-beta.12",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.39.0-beta.1",
"version": "0.38.0-beta.12",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.39.0-beta.1",
"version": "0.38.0-beta.12",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.39.0-beta.1",
"version": "0.38.0-beta.12",
"os": [
"win32"
],
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.39.0-beta.1",
"version": "0.38.0-beta.12",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
+11106
View File
File diff suppressed because it is too large Load Diff
+5 -5
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.39.0-beta.1",
"version": "0.38.0-beta.12",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
@@ -44,7 +44,7 @@
"@biomejs/biome": "^1.7.3",
"@jest/globals": "^29.7.0",
"@napi-rs/cli": "3.7.0",
"@opentelemetry/sdk-metrics": "^2.10.0",
"@opentelemetry/sdk-metrics": "^1.30.0",
"@types/axios": "^0.14.0",
"@types/jest": "^29.1.2",
"@types/node": "22.7.4",
@@ -56,7 +56,7 @@
"eslint": "^8.57.0",
"jest": "^29.7.0",
"shx": "^0.3.4",
"tmp": "^0.2.7",
"tmp": "^0.2.3",
"ts-jest": "^29.1.2",
"typedoc": "0.26.4",
"typedoc-plugin-markdown": "4.2.1",
@@ -67,7 +67,7 @@
"timeout": "3m"
},
"engines": {
"node": ">= 22"
"node": ">= 18"
},
"packageManager": "pnpm@11.1.1",
"cpu": ["x64", "arm64"],
@@ -101,7 +101,7 @@
"openai": "4.29.2"
},
"peerDependencies": {
"@types/node": ">=22",
"@types/node": ">=18",
"apache-arrow": ">=15.0.0 <=18.1.0"
},
"peerDependenciesMeta": {
+436 -592
View File
File diff suppressed because it is too large Load Diff
-38
View File
@@ -16,41 +16,3 @@ allowBuilds:
onnxruntime-node: true
protobufjs: true
sharp: true
minimumReleaseAgeExclude:
- protobufjs@7.5.8
- tmp@0.2.6
- form-data@4.0.6
- tar@7.5.16
- markdown-it@14.1.2
- linkify-it@5.0.1
- js-yaml@3.15.0
- js-yaml@4.1.2
- protobufjs@7.6.1
- protobufjs@7.6.3
- '@babel/core@7.29.1'
- axios@1.18.0
- brace-expansion@2.1.2
- brace-expansion@1.1.16
- js-yaml@4.3.0
- tar@7.5.18
- tar@7.5.19
- tar@7.5.17
- protobufjs@7.6.5
- linkify-it@5.0.2
- sharp@0.35.0
- brace-expansion@1.1.17
- brace-expansion@2.1.3
- brace-expansion@2.1.4
- brace-expansion@1.1.18
- js-yaml@3.15.1
- js-yaml@4.3.1
- tar@7.5.21
- '@opentelemetry/core@2.8.0'
# @huggingface/transformers pins sharp ^0.33.5 and no released version has moved
# past ^0.34.5, all of which inherit the libvips CVEs in GHSA-f88m-g3jw-g9cj.
# Force the patched line. sharp is only reached by transformers' image pipeline,
# which LanceDB's text embedding function never uses.
overrides:
sharp: ^0.35.4
+1 -5
View File
@@ -664,11 +664,7 @@ impl JsFullTextQuery {
}
fn parse_fts_query(query: Object) -> napi::Result<FullTextSearchQuery> {
// `&JsFullTextQuery` recovers a native class reference through napi's borrow-tracked
// path, which is only usable from generated `#[napi]` argument conversion. This is a
// manual lookup on a nested `Object` property instead, so use `ClassInstance`, which
// unwraps the class without requiring a borrow scope.
if let Ok(Some(query)) = query.get::<ClassInstance<JsFullTextQuery>>("query") {
if let Ok(Some(query)) = query.get::<&JsFullTextQuery>("query") {
Ok(FullTextSearchQuery::new_query(query.inner.clone()))
} else if let Ok(Some(query_text)) = query.get::<String>("query") {
let mut query_text = query_text;
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.39.0-beta.1"
version = "0.38.0-beta.12"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
-3
View File
@@ -150,8 +150,6 @@ class Connection(object):
def job(self, job_id: str) -> Job: ...
async def create_function_async(self, request_json: str) -> Job: ...
async def get_function(self, name: str, version: str) -> str: ...
async def list_functions(self) -> List[str]: ...
async def drop_function(self, name: str, version: str) -> bool: ...
async def list_jobs(self) -> List[JobInfo]: ...
async def get_job(self, job_id: str) -> Optional[JobDescription]: ...
async def cancel_job(self, job_id: str) -> bool: ...
@@ -608,7 +606,6 @@ class FullTextQuery:
class PyQueryRequest:
limit: Optional[int]
offset: Optional[int]
take_offsets: Optional[List[int]]
filter: Optional[Union[str, bytes]]
full_text_search: Optional[FullTextQuery]
select: Optional[Union[str, List[str]]]
-51
View File
@@ -712,34 +712,6 @@ class DBConnection(EnforceOverrides):
"Function catalog operations are not supported for this connection type"
)
def list_functions(self) -> List[FunctionVersion]:
"""List every published immutable Function version.
Results are ordered by Function name then version. Local connections
raise ``NotImplementedError``.
Examples
--------
List the identities available to use in Function-backed columns:
```python
[(function.name, function.version) for function in db.list_functions()]
```
"""
raise NotImplementedError(
"Function catalog operations are not supported for this connection type"
)
def drop_function(self, name: str, *, version: str) -> bool:
"""Drop one exact immutable Function version from the remote catalog.
Returns True when the version changed to Dropped and False for an
idempotent replay. Local connections raise NotImplementedError.
"""
raise NotImplementedError(
"Function catalog operations are not supported for this connection type"
)
def job(self, job_id: str) -> Job:
"""A [Job][lancedb.job.Job] handle for a server-side job by id.
@@ -1441,14 +1413,6 @@ class LanceDBConnection(DBConnection):
def get_function(self, name: str, *, version: str) -> FunctionVersion:
return LOOP.run(self._conn.get_function(name, version=version))
@override
def list_functions(self) -> List[FunctionVersion]:
return LOOP.run(self._conn.list_functions())
@override
def drop_function(self, name: str, *, version: str) -> bool:
return LOOP.run(self._conn.drop_function(name, version=version))
@override
def list_jobs(self) -> List[JobInfo]:
"""List server-side jobs across the database's tables."""
@@ -2279,21 +2243,6 @@ class AsyncConnection(object):
"""Open one exact immutable Function version from the remote catalog."""
return FunctionVersion.from_json(await self._inner.get_function(name, version))
async def list_functions(self) -> List[FunctionVersion]:
"""List every published immutable Function version.
Results are ordered by Function name then version. Local connections
raise ``NotImplementedError``.
"""
return [
FunctionVersion.from_json(value)
for value in await self._inner.list_functions()
]
async def drop_function(self, name: str, *, version: str) -> bool:
"""Drop one exact immutable Function version from the remote catalog."""
return await self._inner.drop_function(name, version)
async def list_jobs(self) -> List[JobInfo]:
"""List server-side jobs across the database's tables."""
return await self._inner.list_jobs()
+20 -327
View File
@@ -49,25 +49,11 @@ from pydantic import (
model_validator,
)
from .schema import is_blob_v2_field as _is_blob_v2_field
_Int32 = conint(strict=True, ge=-(2**31), le=2**31 - 1)
_UInt32 = conint(strict=True, ge=0, le=2**32 - 1)
_UInt64 = conint(strict=True, ge=0, le=2**64 - 1)
def _validate_gpu_wire_marker(value: Any) -> bool:
if value is not True:
raise ValueError("runtime.gpu must be true")
return True
def _normalize_gpu_marker(value: bool) -> Optional[bool]:
if not isinstance(value, bool):
raise ValueError("gpu must be a boolean")
return True if value else None
class _FrozenDict(dict):
def _immutable(self, *args, **kwargs):
raise TypeError("remote canonical values are immutable")
@@ -253,23 +239,6 @@ class PythonRuntimeSpec(_RemoteValue):
python_version: Optional[str] = None
environment: Optional[PythonEnvironmentSpec] = None
env: Optional[Mapping[str, str]] = None
gpu: Optional[bool] = None
@model_validator(mode="before")
@classmethod
def _discard_unknown_runtime_payload(cls, value):
if isinstance(value, Mapping):
kind = value.get("kind")
if isinstance(kind, str) and kind not in {"python", "python_v2"}:
return {"kind": kind}
return value
@field_validator("gpu", mode="before")
@classmethod
def _validate_gpu_marker(cls, value):
if value is None:
return None
return _validate_gpu_wire_marker(value)
@model_validator(mode="after")
def _validate_runtime_kind(self):
@@ -278,28 +247,18 @@ class PythonRuntimeSpec(_RemoteValue):
raise ValueError("python runtime requires python_version")
if self.environment is None:
raise ValueError("python runtime requires environment")
if self.gpu is not None:
raise ValueError("python runtime with gpu requires kind='python_v2'")
elif self.kind == "python_v2":
if self.python_version is None:
raise ValueError("python_v2 runtime requires python_version")
if self.environment is None:
raise ValueError("python_v2 runtime requires environment")
if self.gpu is None:
raise ValueError("python_v2 runtime requires gpu")
else:
object.__setattr__(self, "python_version", None)
object.__setattr__(self, "environment", None)
object.__setattr__(self, "env", None)
object.__setattr__(self, "gpu", None)
return self
class FunctionVersion(_RemoteValue):
"""An exact immutable Function version returned by Enterprise.
The GPU execution requirement is part of this identity. CPU and memory sizing,
priority, concurrency, and retry policy belong to the execution platform.
Scheduling resources, priority, concurrency, and retry policy belong to
the submitting Job and are not part of this identity.
"""
name: str
@@ -520,13 +479,6 @@ class RefreshColumnResult(_RemoteValue):
_FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$")
_FUNCTION_BLOB_V2_TYPE = "blob_v2"
_ARROW_EXTENSION_NAME_KEY = "ARROW:extension:name"
_BLOB_V2_EXTENSION_NAME = "lance.blob.v2"
_NESTED_BLOB_COLLECTION_ERROR = (
"unsupported Arrow type for Function signature: Blob v2 fields nested under "
"collection types are not supported"
)
_GRAMMAR_PRIMITIVES = (
@@ -543,7 +495,6 @@ _GRAMMAR_PRIMITIVES = (
(pa.float32(), "float32"),
(pa.float64(), "float64"),
(pa.string(), "utf8"),
(pa.large_string(), "large_utf8"),
(pa.binary(), "binary"),
(pa.date32(), "date32"),
(pa.date64(), "date64"),
@@ -551,258 +502,31 @@ _GRAMMAR_PRIMITIVES = (
def _canonical_arrow_type(data_type: pa.DataType) -> str:
"""The compact Function grammar, or canonical exact JSON for nested types."""
grammar = _grammar_arrow_type(data_type)
if grammar is not None:
return grammar
exact = _exact_arrow_type(data_type)
return json.dumps(exact, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
def _grammar_arrow_type(data_type: pa.DataType) -> Optional[str]:
"""The server's V1 Function type grammar. Anything outside it is rejected
here rather than at registration."""
for candidate, name in _GRAMMAR_PRIMITIVES:
if data_type == candidate:
return name
if pa.types.is_list(data_type) or pa.types.is_large_list(data_type):
item = _grammar_list_item(data_type)
if item is None:
return None
prefix = "list" if pa.types.is_list(data_type) else "large_list"
return f"{prefix}<{item}>"
return f"{prefix}<{_canonical_list_item(data_type)}>"
if pa.types.is_fixed_size_list(data_type) and data_type.list_size > 0:
item = _grammar_list_item(data_type)
if item is not None:
return f"fixed_size_list<{item}, {data_type.list_size}>"
return None
return (
f"fixed_size_list<{_canonical_list_item(data_type)}, {data_type.list_size}>"
)
raise TypeError(f"unsupported Arrow type for Function signature: {data_type}")
def _grammar_list_item(data_type: pa.DataType) -> Optional[str]:
def _canonical_list_item(data_type: pa.DataType) -> str:
"""The grammar names only the item type; it always means a non-nullable
child called `item`, so other child properties require exact JSON."""
child called `item`, so any other child metadata cannot be represented."""
child = data_type.value_field
if child.name != "item" or child.nullable or child.metadata:
return None
return _grammar_arrow_type(child.type)
def _validate_exact_arrow_field(field: pa.Field) -> None:
if not field.name:
raise TypeError(
"unsupported Arrow type for Function signature: field names "
"must not be empty"
"unsupported Arrow type for Function signature: list items must be a "
f"non-nullable field named 'item', got {child}"
)
if _is_blob_v2_field(field):
if not _has_supported_blob_v2_layout(field):
raise TypeError(
"unsupported Arrow type for Function signature: lance.blob.v2 "
f"requires a supported Blob storage layout, got {field}"
)
metadata = {
(key.decode() if isinstance(key, bytes) else key): (
value.decode() if isinstance(value, bytes) else value
)
for key, value in (field.metadata or {}).items()
}
if metadata and metadata != {
_ARROW_EXTENSION_NAME_KEY: _BLOB_V2_EXTENSION_NAME
}:
raise TypeError(
"unsupported Arrow type for Function signature: lance.blob.v2 "
"field metadata must contain only its canonical extension marker"
)
elif field.metadata:
raise TypeError(
"unsupported Arrow type for Function signature: field metadata "
f"is not supported, got {field}"
)
def _has_supported_blob_v2_layout(field: pa.Field) -> bool:
data_type = field.type
if isinstance(data_type, pa.ExtensionType):
data_type = data_type.storage_type
if not pa.types.is_struct(data_type):
return False
fields = tuple(data_type)
def matches(spec, compare_nullable) -> bool:
return len(fields) == len(spec) and all(
actual.name == name
and actual.type == expected_type
and (not check_nullable or actual.nullable == nullable)
for actual, (name, expected_type, nullable), check_nullable in zip(
fields, spec, compare_nullable
)
)
logical_minimal = (
("data", pa.large_binary(), True),
("uri", pa.utf8(), True),
)
logical_full = logical_minimal + (
("position", pa.uint64(), True),
("size", pa.uint64(), True),
)
prepared = (
("kind", pa.uint8(), True),
("data", pa.large_binary(), True),
("uri", pa.utf8(), True),
("blob_id", pa.uint32(), True),
("blob_size", pa.uint64(), True),
("position", pa.uint64(), True),
)
descriptor = (
("kind", pa.uint8(), False),
("position", pa.uint64(), False),
("size", pa.uint64(), False),
("blob_id", pa.uint32(), False),
("blob_uri", pa.utf8(), False),
)
return (
matches(logical_minimal, (True, True))
or matches(logical_full, (True, True, False, False))
or matches(prepared, (True,) * len(prepared))
or matches(descriptor, (False,) * len(descriptor))
)
def _canonical_arrow_field(field: pa.Field) -> str:
_validate_exact_arrow_field(field)
if _is_blob_v2_field(field):
return _FUNCTION_BLOB_V2_TYPE
return _canonical_arrow_type(field.type)
def _blob_storage_type(field: pa.Field) -> pa.DataType:
data_type = field.type
if isinstance(data_type, pa.ExtensionType):
return data_type.storage_type
return data_type
def _exact_blob_storage_type(field: pa.Field) -> dict[str, Any]:
storage = _blob_storage_type(field)
if not pa.types.is_struct(storage):
raise TypeError(
"unsupported Arrow type for Function signature: lance.blob.v2 "
"requires struct storage"
)
return {
"type": "struct",
"fields": [
{
"name": child.name,
"nullable": child.nullable,
"type": (
{"type": "large_binary"}
if pa.types.is_large_binary(child.type)
else _exact_arrow_type(child.type)
),
}
for child in storage
],
}
def _data_type_has_blob_v2(data_type: pa.DataType) -> bool:
if pa.types.is_struct(data_type):
return any(
_is_blob_v2_field(field) or _data_type_has_blob_v2(field.type)
for field in data_type
)
if (
pa.types.is_list(data_type)
or pa.types.is_large_list(data_type)
or pa.types.is_fixed_size_list(data_type)
):
field = data_type.value_field
return _is_blob_v2_field(field) or _data_type_has_blob_v2(field.type)
if pa.types.is_map(data_type):
return any(
_is_blob_v2_field(field) or _data_type_has_blob_v2(field.type)
for field in (data_type.key_field, data_type.item_field)
)
return False
def _exact_arrow_field(
field: pa.Field, *, inside_collection: bool = False
) -> dict[str, Any]:
_validate_exact_arrow_field(field)
if _is_blob_v2_field(field):
if inside_collection:
raise TypeError(_NESTED_BLOB_COLLECTION_ERROR)
return {
"name": field.name,
"nullable": field.nullable,
"type": _exact_blob_storage_type(field),
"metadata": {
_ARROW_EXTENSION_NAME_KEY: _BLOB_V2_EXTENSION_NAME,
},
}
value = {
"name": field.name,
"nullable": field.nullable,
"type": _exact_arrow_type(field.type, inside_collection=inside_collection),
}
return value
def _exact_arrow_type(
data_type: pa.DataType, *, inside_collection: bool = False
) -> dict[str, Any]:
for candidate, name in _GRAMMAR_PRIMITIVES:
if data_type == candidate:
return {"type": name}
if pa.types.is_struct(data_type):
fields = list(data_type)
names = [field.name for field in fields]
if not fields or len(set(names)) != len(names):
raise TypeError(
"unsupported Arrow type for Function signature: structs must have "
"non-empty, uniquely named fields"
)
return {
"type": "struct",
"fields": [
_exact_arrow_field(field, inside_collection=inside_collection)
for field in fields
],
}
if (
pa.types.is_list(data_type)
or pa.types.is_large_list(data_type)
or pa.types.is_fixed_size_list(data_type)
):
if pa.types.is_fixed_size_list(data_type):
if data_type.value_field.name != "item":
raise TypeError(
"unsupported Arrow type for Function signature: fixed-size list "
"items must be named 'item'"
)
if data_type.list_size <= 0:
raise TypeError(
f"unsupported Arrow type for Function signature: {data_type}"
)
value: dict[str, Any] = {
"type": (
"list"
if pa.types.is_list(data_type)
else "large_list"
if pa.types.is_large_list(data_type)
else "fixed_size_list"
),
"fields": [
_exact_arrow_field(data_type.value_field, inside_collection=True)
],
}
if pa.types.is_fixed_size_list(data_type):
value["length"] = data_type.list_size
return value
if pa.types.is_map(data_type) and _data_type_has_blob_v2(data_type):
raise TypeError(_NESTED_BLOB_COLLECTION_ERROR)
raise TypeError(f"unsupported Arrow type for Function signature: {data_type}")
return _canonical_arrow_type(child.type)
def _list_of(item: pa.DataType) -> pa.DataType:
@@ -876,15 +600,8 @@ def _callable_parameters(function: Callable[..., Any]) -> tuple[inspect.Paramete
def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutput:
if isinstance(output, pa.Schema):
if output.metadata:
raise TypeError("Function output schema metadata is not supported")
fields = tuple(output)
elif (
isinstance(output, pa.Field)
and not _is_blob_v2_field(output)
and pa.types.is_struct(output.type)
):
_validate_exact_arrow_field(output)
elif isinstance(output, pa.Field) and pa.types.is_struct(output.type):
if output.nullable:
raise ValueError("Function output must be non-nullable")
fields = tuple(output.type)
@@ -900,12 +617,11 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
raise TypeError(
"output_schema must be a PyArrow DataType, Field, or Schema"
)
_validate_exact_arrow_field(field)
if field.nullable:
raise ValueError("Function output must be non-nullable")
return FunctionOutput(
kind="scalar",
arrow_type=_canonical_arrow_field(field),
arrow_type=_canonical_arrow_type(field.type),
nullable=False,
)
@@ -913,8 +629,6 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
raise ValueError("named-struct Function output must contain at least one field")
if any(field.nullable for field in fields):
raise ValueError("Function output fields must be non-nullable")
for field in fields:
_validate_exact_arrow_field(field)
names = [field.name for field in fields]
if len(set(names)) != len(names):
raise ValueError("Function output field names must be unique")
@@ -923,7 +637,7 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
fields=tuple(
FunctionResultField(
name=field.name,
arrow_type=_canonical_arrow_field(field),
arrow_type=_canonical_arrow_type(field.type),
nullable=False,
)
for field in fields
@@ -943,10 +657,6 @@ def _infer_signature(
if input_schema is not None:
if not isinstance(input_schema, pa.Schema):
raise TypeError("input_schema must be a PyArrow Schema")
if input_schema.metadata:
raise TypeError("Function input schema metadata is not supported")
for field in input_schema:
_validate_exact_arrow_field(field)
expected = tuple(parameter.name for parameter in parameters)
actual = tuple(input_schema.names)
if actual != expected:
@@ -957,7 +667,7 @@ def _infer_signature(
inputs = tuple(
FunctionParameter(
name=field.name,
arrow_type=_canonical_arrow_field(field),
arrow_type=_canonical_arrow_type(field.type),
nullable=field.nullable,
)
for field in input_schema
@@ -980,9 +690,7 @@ def _infer_signature(
inputs.append(
FunctionParameter(
name=parameter.name,
arrow_type=_canonical_arrow_field(
pa.field(parameter.name, data_type, nullable=nullable)
),
arrow_type=_canonical_arrow_type(data_type),
nullable=nullable,
)
)
@@ -1202,7 +910,6 @@ class UdfDefinition:
pip: tuple[str, ...],
env: Mapping[str, str],
python_version: Optional[str],
gpu: bool = False,
conda: tuple[str, ...] = (),
conda_channels: tuple[str, ...] = (),
):
@@ -1231,14 +938,12 @@ class UdfDefinition:
signature = _infer_signature(function, input_schema, output_schema)
source = _package_source(function)
digest = f"sha256:{hashlib.sha256(source).hexdigest()}"
gpu_marker = _normalize_gpu_marker(gpu)
runtime = PythonRuntimeSpec(
kind="python_v2" if gpu_marker is not None else "python",
kind="python",
python_version=python_version
or f"{sys.version_info.major}.{sys.version_info.minor}",
environment=environment_spec,
env=environment,
gpu=gpu_marker,
)
self._function = function
self._request = FunctionRegistrationRequest(
@@ -1284,7 +989,6 @@ def udf(
pip: tuple[str, ...] | list[str] = (),
env: Optional[Mapping[str, str]] = None,
python_version: Optional[str] = None,
gpu: bool = False,
conda: tuple[str, ...] | list[str] = (),
conda_channels: tuple[str, ...] | list[str] = (),
) -> Callable[[Callable[..., Any]], UdfDefinition]: ...
@@ -1299,7 +1003,6 @@ def udf(
pip: tuple[str, ...] | list[str] = (),
env: Optional[Mapping[str, str]] = None,
python_version: Optional[str] = None,
gpu: bool = False,
conda: tuple[str, ...] | list[str] = (),
conda_channels: tuple[str, ...] | list[str] = (),
):
@@ -1332,10 +1035,6 @@ def udf(
Environment variables included in the Function definition.
python_version : str, optional
Remote Python major/minor version. Defaults to the client version.
gpu : bool, default False
Whether every remote execution requires a GPU. The execution platform
selects one compatible GPU for each worker. The requirement is part of
the immutable Function version.
The packaged artifact is a snapshot: the function source plus exactly
the module-level names it references (modules as imports, importable
@@ -1360,11 +1059,6 @@ def udf(
... return value * 2
>>> score(1.5)
3.0
>>> @udf(pip=["cupy-cuda12x"], gpu=True)
... def gpu_score(value: int) -> int:
... return value * 2
>>> gpu_score.registration_request.runtime.gpu
True
"""
def decorate(target: Callable[..., Any]) -> UdfDefinition:
@@ -1376,7 +1070,6 @@ def udf(
pip=tuple(pip),
env={} if env is None else env,
python_version=python_version,
gpu=gpu,
conda=tuple(conda),
conda_channels=tuple(conda_channels),
)
+1 -5
View File
@@ -42,8 +42,6 @@ class MaterializedViewDefinition:
"""Cap on the number of rows the view holds."""
inputs: List[str] = field(default_factory=list)
"""Source columns the projections and filter read."""
source_namespace: List[str] = field(default_factory=list)
"""Namespace holding the source table; empty is the root namespace."""
def _definition_from_schema(
@@ -55,8 +53,7 @@ def _definition_from_schema(
raise ValueError(f"Table '{name}' is not a materialized view")
value = json.loads(raw)
kind = value.get("kind")
# "namespaced_select" keeps older readers from resolving the source at root.
if kind not in ("select", "namespaced_select"):
if kind != "select":
raise NotImplementedError(
f"materialized view '{name}' is defined by '{kind}', which this "
"version of lancedb cannot refresh"
@@ -69,7 +66,6 @@ def _definition_from_schema(
filter=value.get("filter"),
limit=value.get("limit"),
inputs=value.get("inputs", []),
source_namespace=value.get("source_namespace", []),
)
-6
View File
@@ -109,7 +109,6 @@ def _query_is_plain_scan(query: Query) -> bool:
return (
query.vector is None
and query.full_text_query is None
and query.take_offsets is None
and not query.postfilter
and not query.order_by
)
@@ -805,10 +804,6 @@ class Query(pydantic.BaseModel):
# offset to start fetching results from
offset: Optional[int] = None
# Dataset offsets whose duplicate occurrences must be restored after lookup.
# This is populated when a take query is converted to this serializable form.
take_offsets: Optional[List[int]] = None
# if true, will only search the indexed data
fast_search: Optional[bool] = None
@@ -830,7 +825,6 @@ class Query(pydantic.BaseModel):
query = cls()
query.limit = req.limit
query.offset = req.offset
query.take_offsets = req.take_offsets
query.filter = req.filter
query.full_text_query = req.full_text_search
query.columns = req.select
-8
View File
@@ -749,14 +749,6 @@ class RemoteDBConnection(DBConnection):
def get_function(self, name: str, *, version: str) -> FunctionVersion:
return LOOP.run(self._conn.get_function(name, version=version))
@override
def list_functions(self) -> List[FunctionVersion]:
return LOOP.run(self._conn.list_functions())
@override
def drop_function(self, name: str, *, version: str) -> bool:
return LOOP.run(self._conn.drop_function(name, version=version))
@override
def list_jobs(self) -> List["JobInfo"]:
"""List server-side jobs across the database's tables."""
+13 -14
View File
@@ -67,15 +67,7 @@ from ..query import (
LanceTakeQueryBuilder,
LanceVectorQueryBuilder,
)
from ..table import (
AsyncTable,
BlobMode,
Branches,
IndexStatistics,
Query,
Table,
Tags,
)
from ..table import AsyncTable, BlobMode, Branches, IndexStatistics, Query, Table, Tags
from ..types import BaseTokenizerType
@@ -402,7 +394,6 @@ class RemoteTable(Table):
/,
*,
config: IndexConfigType,
replace: bool = ...,
wait_timeout: Optional[timedelta] = ...,
name: Optional[str] = ...,
train: bool = ...,
@@ -417,7 +408,7 @@ class RemoteTable(Table):
index_cache_size: Optional[int] = ...,
num_partitions: Optional[int] = ...,
num_sub_vectors: Optional[int] = ...,
replace: bool = ...,
replace: Optional[bool] = ...,
accelerator: Optional[str] = ...,
index_type: Literal[
"VECTOR", "IVF_FLAT", "IVF_SQ", "IVF_PQ", "IVF_HNSW_SQ", "IVF_HNSW_PQ"
@@ -436,7 +427,7 @@ class RemoteTable(Table):
index_cache_size: Optional[int] = None,
num_partitions: Optional[int] = None,
num_sub_vectors: Optional[int] = None,
replace: bool = False,
replace: Optional[bool] = None,
accelerator: Optional[str] = None,
index_type="vector",
wait_timeout: Optional[timedelta] = None,
@@ -480,6 +471,7 @@ class RemoteTable(Table):
vector_column_name,
accelerator,
index_cache_size,
replace,
)
if is_legacy:
@@ -503,6 +495,12 @@ class RemoteTable(Table):
"If you have 100M+ vectors to index,"
"please contact us at contact@lancedb.com"
)
if replace is not None:
logging.warning(
"replace is not supported on LanceDB cloud."
"Existing indexes will always be replaced."
)
idx_type = index_type.upper()
if idx_type == "VECTOR" or idx_type == "IVF_PQ":
config = IvfPq(
@@ -542,7 +540,6 @@ class RemoteTable(Table):
LOOP.run(
self._table.create_index(
column,
replace=replace,
config=config,
wait_timeout=wait_timeout,
name=name,
@@ -555,7 +552,7 @@ class RemoteTable(Table):
column: str,
*,
config: IndexConfigType,
replace: bool = False,
replace: Optional[bool] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
train: bool = True,
@@ -587,6 +584,7 @@ class RemoteTable(Table):
vector_column_name: str,
accelerator: Optional[str],
index_cache_size: Optional[int],
replace: Optional[bool],
) -> bool:
"""Detect if this is a legacy create_index call."""
if config is not None:
@@ -598,6 +596,7 @@ class RemoteTable(Table):
num_sub_vectors,
accelerator,
index_cache_size,
replace,
)
):
return True
+24 -54
View File
@@ -1132,7 +1132,7 @@ class Table(ABC):
num_partitions: Optional[int] = None,
num_sub_vectors: Optional[int] = None,
vector_column_name: str = VECTOR_COLUMN_NAME,
replace: bool = False,
replace: bool = True,
accelerator: Optional[str] = None,
index_cache_size: Optional[int] = None,
*,
@@ -1166,7 +1166,7 @@ class Table(ABC):
The index configuration object. If provided, uses the new unified API.
Can be one of: IvfFlat, IvfPq, IvfSq, IvfRq, HnswPq, HnswSq,
BTree, Bitmap, LabelList, Fm, FTS.
replace : bool, default False
replace : bool, default True
Whether to replace an existing index on this column.
wait_timeout : timedelta, optional
Timeout to wait for async indexing to complete.
@@ -1198,7 +1198,7 @@ class Table(ABC):
column: str,
*,
config: IndexConfigType,
replace: bool = False,
replace: Optional[bool] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
train: bool = True,
@@ -1260,7 +1260,7 @@ class Table(ABC):
self,
column: str,
*,
replace: bool = False,
replace: bool = True,
index_type: ScalarIndexType = "BTREE",
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
@@ -1272,7 +1272,7 @@ class Table(ABC):
column : str
The column to be indexed. Must be a boolean, integer, float,
or string column.
replace : bool, default False
replace : bool, default True
Replace the existing index if it exists.
index_type: Literal["BTREE", "BITMAP", "LABEL_LIST"], default "BTREE"
The type of index to create.
@@ -1547,9 +1547,7 @@ class Table(ABC):
on: Union[str, Iterable[str]]
A column (or columns) to join on. This is how records from the
source table and target table are matched. Typically this is some
kind of key or id column. Passing several columns matches on the
composite key: a source row updates a target row only when it
agrees on every one of them.
kind of key or id column.
Examples
--------
@@ -1680,9 +1678,9 @@ class Table(ABC):
Offsets are mostly useful for sampling as the set of all valid offsets is easily
known in advance to be [0, len(table)).
No guarantees are made regarding the order in which results are returned.
Repeated offsets produce repeated rows, which makes this method suitable for
sampling with replacement.
No guarantees are made regarding the order in which results are returned. If
you desire an output order that matches the order of the given offsets, you will
need to add the row offset column to the output and align it yourself.
Parameters
----------
@@ -2167,11 +2165,9 @@ class Table(ABC):
Function columns are supported only on LanceDB Cloud and
Enterprise.
computed: Dict[str, str], optional
A mapping from output column names to SQL expressions derives each
output field from its expression. A direct projection of a Blob v2
field inherits Blob v2 semantics; other expressions derive their
ordinary Arrow type. Mapping order is declaration and dependency
order.
A map of column name to a SQL expression defining the column. The
column's type and inputs are derived from the expression, so no
data type is supplied.
Unlike ``transforms``, the expression is stored rather than
evaluated now: the column is committed with no values, and rows get
@@ -2988,7 +2984,7 @@ class LanceTable(Table):
num_partitions: Optional[int] = None,
num_sub_vectors: Optional[int] = None,
vector_column_name: str = VECTOR_COLUMN_NAME,
replace: bool = False,
replace: bool = True,
accelerator: Optional[str] = None,
index_cache_size: Optional[int] = None,
num_bits: int = 8,
@@ -3030,7 +3026,7 @@ class LanceTable(Table):
The index configuration object. If provided, uses the new unified API.
Can be one of: IvfFlat, IvfPq, IvfSq, IvfRq, HnswPq, HnswSq,
BTree, Bitmap, LabelList, Fm, FTS.
replace : bool, default False
replace : bool, default True
Whether to replace an existing index on this column.
wait_timeout : timedelta, optional
Timeout to wait for async indexing to complete.
@@ -3169,7 +3165,7 @@ class LanceTable(Table):
column: str,
*,
config: IndexConfigType,
replace: bool = False,
replace: Optional[bool] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
train: bool = True,
@@ -3424,7 +3420,7 @@ class LanceTable(Table):
self,
column: str,
*,
replace: bool = False,
replace: bool = True,
index_type: ScalarIndexType = "BTREE",
name: Optional[str] = None,
):
@@ -4092,7 +4088,6 @@ class LanceTable(Table):
)
and not self._route_pushdown_to_rust
and self.current_branch() is None
and query.take_offsets is None
):
from lancedb.namespace import _execute_server_side_query
@@ -5313,7 +5308,7 @@ class AsyncTable:
self,
column: str,
*,
replace: bool = False,
replace: Optional[bool] = None,
config: Optional[
Union[
IvfFlat,
@@ -5344,14 +5339,14 @@ class AsyncTable:
----------
column: str
The column to index.
replace: bool, default False
replace: bool, default True
Whether to replace the existing index
If this is false, and another index already exists on the same columns
and the same name, then an error will be returned. This is true even if
that index is out of date.
The default is False
The default is True
config: default None
For advanced configuration you can specify the type of index you would
like to create. You can also specify index-specific parameters when
@@ -5409,7 +5404,7 @@ class AsyncTable:
self,
column: str,
*,
replace: bool = False,
replace: Optional[bool] = None,
config: Optional[
Union[
IvfFlat,
@@ -5703,9 +5698,7 @@ class AsyncTable:
on: Union[str, Iterable[str]]
A column (or columns) to join on. This is how records from the
source table and target table are matched. Typically this is some
kind of key or id column. Passing several columns matches on the
composite key: a source row updates a target row only when it
agrees on every one of them.
kind of key or id column.
Examples
--------
@@ -5988,23 +5981,7 @@ class AsyncTable:
def _sync_query_to_async(
self, query: Query
) -> (
AsyncHybridQuery
| AsyncFTSQuery
| AsyncVectorQuery
| AsyncQuery
| AsyncTakeQuery
):
if query.take_offsets is not None:
take_query = self.take_offsets(query.take_offsets)
if query.columns:
take_query = take_query.select(query.columns)
if query.use_lsm is not None:
take_query = take_query.use_lsm(query.use_lsm)
if query.with_row_id:
take_query = take_query.with_row_id()
return take_query
) -> AsyncHybridQuery | AsyncFTSQuery | AsyncVectorQuery | AsyncQuery:
async_query = self.query()
if query.limit is not None:
async_query = async_query.limit(query.limit)
@@ -6069,7 +6046,6 @@ class AsyncTable:
self._namespace_client, self._pushdown_operations
)
and not self._route_pushdown_to_rust
and query.take_offsets is None
):
from lancedb.namespace import _execute_server_side_query
@@ -6292,11 +6268,8 @@ class AsyncTable:
Function columns are supported only on LanceDB Cloud and
Enterprise.
computed: Dict[str, str], optional
A mapping from output column names to SQL expressions derives each
output field from its expression. A direct projection of a Blob v2
field inherits Blob v2 semantics; other expressions derive their
ordinary Arrow type. Mapping order is declaration and dependency
order.
A map of column name to a SQL expression defining the column. The
column's type and inputs are derived from the expression.
Unlike ``transforms``, the expression is stored rather than
evaluated now: the column is committed with no values, and rows get
@@ -6567,9 +6540,6 @@ class AsyncTable:
Offsets are mostly useful for sampling as the set of all valid offsets is easily
known in advance to be [0, len(table)).
No guarantees are made regarding the order in which results are returned.
Repeated offsets produce repeated rows.
Parameters
----------
offsets: list[int]
@@ -12,8 +12,6 @@ from datetime import date
import http.server
import json
from pathlib import Path
import subprocess
import sys
import threading
from typing import Optional
@@ -21,13 +19,7 @@ import pyarrow as pa
import pytest
import lancedb
from lancedb.functions import (
PythonRuntimeSpec,
UdfDefinition,
_canonical_arrow_type,
_GRAMMAR_PRIMITIVES,
udf,
)
from lancedb.functions import UdfDefinition, udf
THRESHOLD = 20
_CACHE = None
@@ -69,80 +61,6 @@ def test_scalar_udf_matches_shared_registration_golden_and_remains_callable():
}
def _main_udf_source(
*, threshold: int = 20, input_annotation: str = "int", comparison: str = ">="
) -> str:
return (
"from __future__ import annotations\n"
"from lancedb.functions import udf\n"
f"THRESHOLD = {threshold}\n"
"\n"
"@udf\n"
f"def label(value: {input_annotation}) -> str:\n"
f" return 'big' if value {comparison} THRESHOLD else 'small'\n"
"\n"
"assert label.__module__ == '__main__'\n"
"print(label.registration_request.to_canonical_json())\n"
)
def _run_main_udf(path: Path, source: str) -> dict:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(source)
result = subprocess.run(
[sys.executable, str(path)],
check=True,
capture_output=True,
text=True,
)
return json.loads(result.stdout)
def test_main_udf_registration_identity_is_stable_across_processes_and_paths(
tmp_path,
):
source = _main_udf_source()
original_path = tmp_path / "original" / "job.py"
moved_path = tmp_path / "moved" / "renamed_job.py"
original_runs = [_run_main_udf(original_path, source) for _ in range(2)]
moved_run = _run_main_udf(moved_path, source)
assert len({run["artifact"]["digest"] for run in [*original_runs, moved_run]}) == 1
assert all(
run["signature"] == original_runs[0]["signature"]
for run in [original_runs[1], moved_run]
)
assert original_runs[0] == original_runs[1] == moved_run
body_change = _run_main_udf(
tmp_path / "changes" / "body.py", _main_udf_source(comparison=">")
)
global_change = _run_main_udf(
tmp_path / "changes" / "global.py", _main_udf_source(threshold=21)
)
annotation_change = _run_main_udf(
tmp_path / "changes" / "annotation.py",
_main_udf_source(input_annotation="float"),
)
baseline = original_runs[0]
assert baseline["signature"] == body_change["signature"]
assert baseline["signature"] == global_change["signature"]
assert baseline["signature"] != annotation_change["signature"]
assert (
len(
{
baseline["artifact"]["digest"],
body_change["artifact"]["digest"],
global_change["artifact"]["digest"],
annotation_change["artifact"]["digest"],
}
)
== 4
)
def _run_packaged(definition, *args):
"""Execute the shipped artifact in a fresh namespace, as a worker would."""
source = base64.b64decode(definition.registration_request.artifact.content.data)
@@ -171,58 +89,6 @@ def test_udf_conda_environment():
udf(name="channels", conda_channels=["conda-forge"])(lambda value: value)
def test_udf_gpu_marker_uses_gpu_runtime():
@udf(pip=["cupy-cuda12x"], gpu=True)
def double_on_gpu(value: int) -> int:
return value * 2
request = json.loads(double_on_gpu.registration_request.to_canonical_json())
assert request["runtime"]["kind"] == "python_v2"
assert request["runtime"]["gpu"] is True
@udf(pip=["pyarrow"])
def cpu_function(value: int) -> int:
return value
cpu_runtime = json.loads(cpu_function.registration_request.to_canonical_json())[
"runtime"
]
assert cpu_runtime["kind"] == "python"
assert "gpu" not in cpu_runtime
def identity(value: int) -> int:
return value
for invalid in [None, 0, 1, -1, 1.5, "", "true", "1", "H100"]:
with pytest.raises(ValueError, match="gpu must be a boolean"):
udf(name="invalid_gpu", gpu=invalid)(identity)
base_runtime = {
"kind": "python_v2",
"python_version": "3.12",
"environment": {"kind": "pip"},
}
runtime = PythonRuntimeSpec.model_validate({**base_runtime, "gpu": True})
assert runtime.gpu is True
for invalid in [False, 1, 0, "", "true", "1", "H100"]:
with pytest.raises(ValueError, match="runtime.gpu must be true"):
PythonRuntimeSpec.model_validate({**base_runtime, "gpu": invalid})
def test_unknown_runtime_discards_payload_before_known_field_validation():
for payload in [
{"kind": "python_v3", "gpu": {"model": "H100"}},
{"kind": "python_v3", "resources": []},
{
"kind": "python_v3",
"environment": {"kind": []},
"python_version": 3.15,
},
]:
runtime = PythonRuntimeSpec.model_validate(payload)
assert runtime.to_canonical_json() == '{"kind":"python_v3"}'
def test_udf_packages_attribute_access_and_body_imports():
@udf
def word_norm(body: str) -> float:
@@ -302,7 +168,9 @@ def test_udf_resolves_module_globals_before_builtins(tmp_path):
udf(module.uses_callable_shadow)
def test_canonical_arrow_type_prefers_the_compact_grammar():
def test_canonical_arrow_type_is_exactly_the_grammar():
from lancedb.functions import _GRAMMAR_PRIMITIVES, _canonical_arrow_type
golden = json.loads(
(
Path(__file__).parents[3]
@@ -313,19 +181,14 @@ def test_canonical_arrow_type_prefers_the_compact_grammar():
case["arrow_type"] for case in golden["valid"] if "<" not in case["arrow_type"]
]
assert [name for _, name in _GRAMMAR_PRIMITIVES] == primitives
assert _canonical_arrow_type(pa.list_(pa.field("item", pa.float32(), False))) == (
"list<float32>"
)
assert (
_canonical_arrow_type(pa.large_list(pa.field("item", pa.float32(), False)))
== "large_list<float32>"
)
for outside in [
pa.timestamp("us"),
pa.decimal128(10, 2),
pa.large_string(),
pa.large_binary(),
pa.binary(4),
pa.duration("s"),
pa.struct([pa.field("a", pa.int32())]),
pa.list_(pa.float32(), 0),
pa.list_(pa.timestamp("us")),
]:
@@ -515,27 +378,14 @@ def test_udf_recursion_versus_a_rebound_module_name(tmp_path):
udf(raw_fact)
def test_canonical_arrow_type_uses_exact_json_for_list_child_properties():
nullable = pa.list_(pa.float32())
assert json.loads(_canonical_arrow_type(nullable)) == {
"type": "list",
"fields": [
{
"name": "item",
"nullable": True,
"type": {"type": "float32"},
}
],
}
named = pa.list_(pa.field("custom", pa.float32(), nullable=False))
assert json.loads(_canonical_arrow_type(named))["fields"][0]["name"] == "custom"
def test_canonical_arrow_type_rejects_unrepresentable_list_children():
from lancedb.functions import _canonical_arrow_type
for outside in [
pa.list_(pa.float32()), # pyarrow default: nullable child
pa.list_(pa.field("custom", pa.float32(), nullable=False)),
pa.list_(pa.field("item", pa.float32(), nullable=False, metadata={"k": "v"})),
pa.list_(pa.field("item", pa.float32(), nullable=False), 0),
pa.list_(
pa.field("item", pa.float32(), nullable=False, metadata={"k": "v"}), 3
),
pa.list_(pa.field("custom", pa.float32(), nullable=False), 3),
]:
with pytest.raises(TypeError, match="unsupported Arrow type"):
_canonical_arrow_type(outside)
@@ -545,29 +395,6 @@ def test_canonical_arrow_type_uses_exact_json_for_list_child_properties():
)
== "fixed_size_list<float32, 3>"
)
fixed = json.loads(_canonical_arrow_type(pa.list_(pa.float32(), 3)))
assert fixed == {
"type": "fixed_size_list",
"fields": [
{
"name": "item",
"nullable": True,
"type": {"type": "float32"},
}
],
"length": 3,
}
large = json.loads(_canonical_arrow_type(pa.large_list(pa.float32())))
assert large["type"] == "large_list"
assert large["fields"][0]["nullable"] is True
for invalid_struct in [
pa.struct([]),
pa.struct([pa.field("a", pa.int32()), pa.field("a", pa.int64())]),
pa.struct([pa.field("", pa.int32())]),
]:
with pytest.raises(TypeError, match="unsupported Arrow type"):
_canonical_arrow_type(invalid_struct)
def _calls_missing(value: int) -> int:
@@ -605,7 +432,6 @@ def _arrow_type_from_golden(spec: dict) -> pa.DataType:
"null": pa.null(),
"bool": pa.bool_(),
"utf8": pa.string(),
"large_utf8": pa.large_string(),
"binary": pa.binary(),
"float16": pa.float16(),
"float32": pa.float32(),
@@ -622,6 +448,8 @@ def test_arrow_type_grammar_matches_the_shared_golden():
/ "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json"
).read_text()
)
from lancedb.functions import _canonical_arrow_type
emitted = {
case["arrow_type"]: _canonical_arrow_type(_arrow_type_from_golden(case["json"]))
for case in golden["valid"]
@@ -654,395 +482,6 @@ def test_explicit_arrow_schema_is_deterministic():
assert signature.output.nullable is False
def test_blob_fields_use_the_scalar_function_semantic_type():
@udf(
input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
output_schema=lancedb.blob("result", nullable=False),
)
def copy_blob(image):
return image
signature = copy_blob.registration_request.signature
assert signature.inputs[0].arrow_type == "blob_v2"
assert signature.output.kind == "scalar"
assert signature.output.arrow_type == "blob_v2"
def test_whole_named_struct_function_can_include_a_blob_result_field():
@udf(
input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
output_schema=pa.field(
"payload",
pa.struct(
[
pa.field("mime_type", pa.string(), nullable=False),
lancedb.blob("image", nullable=False),
]
),
nullable=False,
),
)
def inspect_blob(image):
return {"mime_type": "image/png", "image": image}
output = inspect_blob.registration_request.signature.output
assert output.kind == "named_struct"
assert [(field.name, field.arrow_type) for field in output.fields] == [
("mime_type", "utf8"),
("image", "blob_v2"),
]
def test_struct_blob_signature_fields_preserve_exact_metadata_and_nullability():
nested_input = pa.field(
"payload",
pa.struct(
[
pa.field("mime_type", pa.string(), nullable=False),
pa.field(
"nested",
pa.struct([lancedb.blob("image", nullable=True)]),
nullable=True,
),
]
),
nullable=True,
)
nested_output = pa.field(
"result",
pa.struct(
[
pa.field("mime_type", pa.string(), nullable=False),
pa.field(
"nested",
pa.struct([lancedb.blob("image", nullable=True)]),
nullable=False,
),
]
),
nullable=False,
)
@udf(input_schema=pa.schema([nested_input]), output_schema=nested_output)
def copy_payload(payload):
return payload
signature = copy_payload.registration_request.signature
input_type = json.loads(signature.inputs[0].arrow_type)
assert input_type["fields"][1]["nullable"] is True
input_blob = input_type["fields"][1]["type"]["fields"][0]
assert input_blob["nullable"] is True
assert input_blob["metadata"] == {"ARROW:extension:name": "lance.blob.v2"}
assert signature.output.kind == "named_struct"
nested_result = next(
field for field in signature.output.fields if field.name == "nested"
)
output_type = json.loads(nested_result.arrow_type)
output_blob = output_type["fields"][0]
assert output_blob["nullable"] is True
assert output_blob["metadata"] == {"ARROW:extension:name": "lance.blob.v2"}
def test_struct_blob_signature_supports_multiple_struct_levels():
recursive = pa.field(
"value",
pa.struct(
[
pa.field(
"level_1",
pa.struct(
[
pa.field(
"level_2",
pa.struct([lancedb.blob("image", nullable=False)]),
nullable=False,
)
]
),
nullable=False,
)
]
),
nullable=False,
)
@udf(
input_schema=pa.schema([recursive]),
output_schema=pa.field("size", pa.int64(), nullable=False),
)
def blob_size(value):
return len(value["level_1"]["level_2"]["image"])
encoded = json.loads(blob_size.registration_request.signature.inputs[0].arrow_type)
blob = encoded["fields"][0]["type"]["fields"][0]["type"]["fields"][0]
assert blob["metadata"]["ARROW:extension:name"] == "lance.blob.v2"
@pytest.mark.parametrize(
"data_type",
[
pa.list_(lancedb.blob("item", nullable=False)),
pa.large_list(lancedb.blob("item", nullable=False)),
pa.list_(lancedb.blob("item", nullable=False), 2),
pa.map_(pa.string(), lancedb.blob("value", nullable=False).type),
],
)
def test_blob_signature_rejects_collection_ancestors(data_type):
with pytest.raises(
TypeError,
match="Blob v2 fields nested under collection types are not supported",
):
@udf(
input_schema=pa.schema([pa.field("value", data_type, nullable=False)]),
output_schema=pa.field("size", pa.int64(), nullable=False),
)
def blob_size(value):
return len(value)
def test_blob_signature_rejects_collection_below_a_struct():
nested = pa.field(
"value",
pa.struct(
[
pa.field(
"images",
pa.list_(lancedb.blob("item", nullable=False)),
nullable=False,
)
]
),
nullable=False,
)
with pytest.raises(
TypeError,
match="Blob v2 fields nested under collection types are not supported",
):
@udf(
input_schema=pa.schema([nested]),
output_schema=pa.field("size", pa.int64(), nullable=False),
)
def blob_size(value):
return len(value["images"])
def test_named_struct_function_can_include_a_blob_result_field():
@udf(
input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
output_schema=pa.schema(
[
lancedb.blob("thumbnail", nullable=False),
pa.field("width", pa.int32(), nullable=False),
]
),
)
def inspect_blob(image):
return {"thumbnail": image, "width": 1}
output = inspect_blob.registration_request.signature.output
assert output.kind == "named_struct"
assert [(field.name, field.arrow_type) for field in output.fields] == [
("thumbnail", "blob_v2"),
("width", "int32"),
]
def test_metadata_marked_blob_field_uses_the_semantic_type():
extension = lancedb.blob("image", nullable=False).type
storage = (
extension.storage_type if isinstance(extension, pa.ExtensionType) else extension
)
metadata_blob = pa.field(
"image",
storage,
nullable=False,
metadata={"ARROW:extension:name": "lance.blob.v2"},
)
@udf(
input_schema=pa.schema([metadata_blob]),
output_schema=pa.field("size", pa.int64(), nullable=False),
)
def blob_size(image):
return len(image)
assert blob_size.registration_request.signature.inputs[0].arrow_type == "blob_v2"
def test_blob_marker_rejects_invalid_storage_layout():
malformed = pa.field(
"image",
pa.int64(),
nullable=False,
metadata={"ARROW:extension:name": "lance.blob.v2"},
)
with pytest.raises(TypeError, match="requires a supported Blob storage layout"):
@udf(
input_schema=pa.schema([malformed]),
output_schema=pa.field("size", pa.int64(), nullable=False),
)
def blob_size(image):
return len(image)
def test_nested_non_blob_extension_is_not_silently_unwrapped():
class TestExtension(pa.ExtensionType):
def __init__(self):
super().__init__(pa.int64(), "test.function.extension")
def __arrow_ext_serialize__(self):
return b""
@classmethod
def __arrow_ext_deserialize__(cls, storage_type, serialized):
return cls()
nested = pa.field(
"value",
pa.struct([pa.field("extended", TestExtension(), nullable=False)]),
nullable=False,
)
with pytest.raises(TypeError, match="unsupported Arrow type"):
@udf(
input_schema=pa.schema([nested]),
output_schema=pa.field("result", pa.int64(), nullable=False),
)
def extension_value(value):
return value["extended"]
def test_explicit_large_utf8_schemas_use_the_canonical_function_name():
input_schema = pa.schema([pa.field("text", pa.large_string(), nullable=True)])
output_schema = pa.field("result", pa.large_string(), nullable=False)
@udf(input_schema=input_schema, output_schema=output_schema)
def preserve(text):
return text
signature = preserve.registration_request.signature
assert signature.inputs[0].arrow_type == "large_utf8"
assert signature.inputs[0].nullable is True
assert signature.output.arrow_type == "large_utf8"
assert signature.output.nullable is False
nested = pa.struct([pa.field("text", pa.large_string(), nullable=True)])
assert json.loads(_canonical_arrow_type(nested)) == {
"type": "struct",
"fields": [
{
"name": "text",
"nullable": True,
"type": {"type": "large_utf8"},
}
],
}
def test_nested_struct_output_uses_canonical_exact_json():
token = pa.struct(
[
pa.field("position", pa.int32(), nullable=False),
pa.field("value", pa.string(), nullable=False),
pa.field("length", pa.int32(), nullable=False),
]
)
analysis = pa.struct(
[
pa.field("normalized_text", pa.string(), nullable=False),
pa.field("has_content", pa.bool_(), nullable=False),
pa.field(
"metrics",
pa.struct(
[
pa.field("character_count", pa.int64(), nullable=False),
pa.field("word_count", pa.int32(), nullable=False),
pa.field("average_word_length", pa.float64(), nullable=False),
]
),
nullable=False,
),
pa.field(
"diagnostics",
pa.struct(
[
pa.field("status", pa.string(), nullable=False),
pa.field(
"normalization",
pa.struct(
[
pa.field("changed", pa.bool_(), nullable=False),
pa.field(
"original_length", pa.int64(), nullable=False
),
]
),
nullable=False,
),
]
),
nullable=False,
),
pa.field(
"token_preview",
pa.list_(pa.field("item", token, nullable=False)),
nullable=False,
),
]
)
@udf(
input_schema=pa.schema([pa.field("text", pa.string(), nullable=False)]),
output_schema=pa.field("analysis", analysis, nullable=False),
)
def analyze(text):
return {"normalized_text": text}
output = analyze.registration_request.signature.output
assert output.kind == "named_struct"
assert [field.name for field in output.fields] == [
"normalized_text",
"has_content",
"metrics",
"diagnostics",
"token_preview",
]
metrics = json.loads(output.fields[2].arrow_type)
assert metrics == {
"type": "struct",
"fields": [
{
"name": "character_count",
"nullable": False,
"type": {"type": "int64"},
},
{
"name": "word_count",
"nullable": False,
"type": {"type": "int32"},
},
{
"name": "average_word_length",
"nullable": False,
"type": {"type": "float64"},
},
],
}
preview = json.loads(output.fields[4].arrow_type)
assert preview["type"] == "list"
assert preview["fields"][0]["type"]["type"] == "struct"
assert [field["name"] for field in preview["fields"][0]["type"]["fields"]] == [
"position",
"value",
"length",
]
def test_annotation_and_explicit_schema_validation_fail_closed():
with pytest.raises(TypeError, match="missing Function annotations"):
@@ -1086,72 +525,6 @@ def test_annotation_and_explicit_schema_validation_fail_closed():
def nullable_explicit(value):
return value
for invalid_field in [
pa.field("", pa.int32(), nullable=False),
pa.field("result", pa.int32(), nullable=False, metadata={"k": "v"}),
]:
with pytest.raises(TypeError, match="unsupported Arrow type"):
@udf(
input_schema=pa.schema([pa.field("value", pa.int64())]),
output_schema=pa.schema([invalid_field]),
)
def invalid_explicit_field(value):
return value
with pytest.raises(TypeError, match="unsupported Arrow type"):
@udf(
input_schema=pa.schema(
[pa.field("value", pa.int64(), metadata={"k": "v"})]
),
output_schema=pa.int64(),
)
def input_field_metadata(value):
return value
with pytest.raises(TypeError, match="unsupported Arrow type"):
@udf(
input_schema=pa.schema([pa.field("value", pa.int64())]),
output_schema=pa.field(
"result", pa.int64(), nullable=False, metadata={"k": "v"}
),
)
def scalar_output_field_metadata(value):
return value
struct_type = pa.struct([pa.field("value", pa.int64(), nullable=False)])
with pytest.raises(TypeError, match="unsupported Arrow type"):
@udf(
input_schema=pa.schema([pa.field("value", pa.int64())]),
output_schema=pa.field(
"result", struct_type, nullable=False, metadata={"k": "v"}
),
)
def struct_output_field_metadata(value):
return {"value": value}
for input_schema, output_schema in [
(
pa.schema([pa.field("value", pa.int64())], metadata={"k": "v"}),
pa.int64(),
),
(
pa.schema([pa.field("value", pa.int64())]),
pa.schema(
[pa.field("result", pa.int64(), nullable=False)],
metadata={"k": "v"},
),
),
]:
with pytest.raises(TypeError, match="schema metadata"):
@udf(input_schema=input_schema, output_schema=output_schema)
def schema_metadata(value):
return value
def test_local_function_catalog_operations_are_not_supported(tmp_path):
db = lancedb.connect(tmp_path)
@@ -1162,10 +535,6 @@ def test_local_function_catalog_operations_are_not_supported(tmp_path):
db.create_function_async(normalize_score)
with pytest.raises(NotImplementedError, match=message):
db.get_function("normalize_score", version="fv_exact")
with pytest.raises(NotImplementedError, match=message):
db.list_functions()
with pytest.raises(NotImplementedError, match=message):
db.drop_function("normalize_score", version="fv_exact")
@contextlib.contextmanager
@@ -1211,28 +580,6 @@ def _mock_remote_function_catalog():
"version": "fv_exact",
}
response = state["version"]
elif self.path == "/v1/functions/list":
assert body["include_definition"] is True
if "page_token" not in body:
response = {
"functions": [
{
"name": "normalize_score",
"version": "fv_exact",
"definition": state["version"],
}
],
"page_token": "next",
}
else:
assert body["page_token"] == "next"
response = {"functions": []}
elif self.path == "/v1/functions/drop":
assert body == {
"name": "normalize_score",
"version": "fv_exact",
}
response = {"dropped": True}
else:
status = 404
response = {"error": "not found"}
@@ -1291,83 +638,3 @@ def test_blocking_remote_registration_returns_function_version():
"/v1/functions/create",
"/v1/jobs/describe",
]
def test_remote_list_functions_paginates_and_returns_typed_versions():
with _mock_remote_function_catalog() as (host, state):
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
created = db.create_function(normalize_score)
state["requests"].clear()
functions = db.list_functions()
assert functions == [created]
assert state["requests"] == [
("/v1/functions/list", {"include_definition": True}),
(
"/v1/functions/list",
{"include_definition": True, "page_token": "next"},
),
]
@pytest.mark.asyncio
async def test_async_remote_list_functions_returns_typed_versions():
with _mock_remote_function_catalog() as (host, state):
db = await lancedb.connect_async(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
registration = await db.create_function_async(normalize_score)
created = await registration.wait()
state["requests"].clear()
functions = await db.list_functions()
assert functions == [created]
assert [path for path, _ in state["requests"]] == [
"/v1/functions/list",
"/v1/functions/list",
]
def test_remote_drop_function_sends_exact_version():
with _mock_remote_function_catalog() as (host, state):
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
assert db.drop_function("normalize_score", version="fv_exact") is True
assert state["requests"] == [
(
"/v1/functions/drop",
{"name": "normalize_score", "version": "fv_exact"},
)
]
@pytest.mark.asyncio
async def test_async_remote_drop_function_sends_exact_version():
with _mock_remote_function_catalog() as (host, state):
db = await lancedb.connect_async(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
assert await db.drop_function("normalize_score", version="fv_exact") is True
assert state["requests"] == [
(
"/v1/functions/drop",
{"name": "normalize_score", "version": "fv_exact"},
)
]
+2 -10
View File
@@ -97,9 +97,6 @@ async def test_create_index_async_returns_done_job(some_table: AsyncTable):
async def test_create_scalar_index(some_table: AsyncTable):
# Can create
await some_table.create_index("id")
# Can't recreate by default
with pytest.raises(RuntimeError, match="already exists"):
await some_table.create_index("id")
# Can recreate if replace=True
await some_table.create_index("id", replace=True)
indices = await some_table.list_indices()
@@ -113,7 +110,7 @@ async def test_create_scalar_index(some_table: AsyncTable):
with pytest.raises(RuntimeError, match="already exists"):
await some_table.create_index("id", replace=False)
# can also specify index type
await some_table.create_index("id", config=BTree(), replace=True)
await some_table.create_index("id", config=BTree())
await some_table.drop_index("id_idx")
indices = await some_table.list_indices()
@@ -354,18 +351,13 @@ async def test_full_text_search_index(some_table: AsyncTable):
async def test_create_vector_index(some_table: AsyncTable):
# Can create
await some_table.create_index("vector")
# Can't recreate by default
with pytest.raises(RuntimeError, match="already exists"):
await some_table.create_index("vector")
# Can recreate if replace=True
await some_table.create_index("vector", replace=True)
# Can't recreate if replace=False
with pytest.raises(RuntimeError, match="already exists"):
await some_table.create_index("vector", replace=False)
# Can also specify index type
await some_table.create_index(
"vector", config=IvfPq(num_partitions=100), replace=True
)
await some_table.create_index("vector", config=IvfPq(num_partitions=100))
indices = await some_table.list_indices()
assert len(indices) == 1
assert indices[0].index_type == "IvfPq"
@@ -266,38 +266,3 @@ async def test_async_namespace_connection_materialized_views(tmp_path):
handle._route_pushdown_to_rust == through_namespace._route_pushdown_to_rust
)
assert handle._namespace_path == through_namespace._namespace_path
def test_namespaced_select_kind_is_read_and_unknown_kinds_are_refused():
import json
import pyarrow as pa
from lancedb.materialized_view import _definition_from_schema
def schema_with(definition: dict) -> pa.Schema:
return pa.schema([pa.field("id", pa.int32())]).with_metadata(
{b"mv.definition": json.dumps(definition).encode()}
)
# "namespaced_select" is the namespaced form of "select": same shape,
# a separate kind so readers that predate it refuse instead of
# resolving the source at the root.
definition = _definition_from_schema(
schema_with(
{
"kind": "namespaced_select",
"source_table": "people",
"source_namespace": ["ns"],
"projections": [{"output": "name", "expression": "name"}],
}
),
"v",
)
assert definition.source_table == "people"
assert definition.source_namespace == ["ns"]
with pytest.raises(NotImplementedError, match="cannot refresh"):
_definition_from_schema(
schema_with({"kind": "select_v3", "source_table": "people"}), "v"
)
-15
View File
@@ -1923,21 +1923,6 @@ def test_take_queries(tmp_path):
17,
]
# Duplicate offsets are occurrences, not set members. Ordering is unspecified.
assert sorted(table.take_offsets([5, 2, 5, 17]).to_pandas()["idx"].to_list()) == [
2,
5,
5,
17,
]
# Converting a take builder to its serializable query representation must
# retain occurrence metadata and execute with the same multiplicity.
query = table.take_offsets([5, 2, 5, 17]).select(["idx"]).to_query_object()
assert query.take_offsets == [5, 2, 5, 17]
converted = table._execute_query(query).read_all()
assert sorted(converted["idx"].to_pylist()) == [2, 5, 5, 17]
# Take by row id
assert list(
sorted(table.take_row_ids([5, 2, 17]).to_pandas()["idx"].to_list())
+6 -39
View File
@@ -479,49 +479,24 @@ def test_remote_permutation_is_picklable():
match = re.search(
r"_rowoffset\s+in\s+\((.*?)\)", body["filter"], re.IGNORECASE
)
offsets = list(
dict.fromkeys(int(o.strip()) for o in match.group(1).split(","))
)
offsets = [int(o.strip()) for o in match.group(1).split(",")]
else:
offsets = list(range(len(rows)))
columns = body.get("columns") or ["a"]
table = pa.table(
{
column: (
[rows[offset] for offset in offsets]
if column == "a"
else offsets
)
for column in columns
}
)
table = pa.table({"a": [rows[offset] for offset in offsets]})
request.send_response(200)
request.send_header("Content-Type", "application/vnd.apache.arrow.file")
request.end_headers()
with pa.ipc.new_file(request.wfile, schema=table.schema) as writer:
writer.write_table(table, max_chunksize=2)
writer.write_table(table)
else:
request.send_response(404)
request.end_headers()
with mock_lancedb_connection(handler) as db:
table = db.open_table("test")
assert table.take_offsets([0, 2, 0, 4]).to_list() == [
{"a": 0},
{"a": 0},
{"a": 2},
{"a": 4},
]
permutation = Permutation.identity(table)
permutation = Permutation.identity(db.open_table("test"))
restored = pickle.loads(pickle.dumps(permutation))
assert restored.__getitems__([0, 2, 0, 4]) == [
{"a": 0},
{"a": 2},
{"a": 0},
{"a": 4},
]
assert restored.__getitems__([0, 2, 4]) == [{"a": 0}, {"a": 2}, {"a": 4}]
def test_create_table_exist_ok():
@@ -820,13 +795,11 @@ def test_table_create_indices():
scalar_req = received_requests[0]
assert "name" in scalar_req
assert scalar_req["name"] == "custom_scalar_idx"
assert scalar_req["replace"] is False
# Check FTS index request has custom name
fts_req = received_requests[1]
assert "name" in fts_req
assert fts_req["name"] == "custom_fts_idx"
assert fts_req["replace"] is False
assert fts_req["block_size"] == 256
assert fts_req["custom_stop_words"] == ["cloud"]
@@ -834,7 +807,6 @@ def test_table_create_indices():
vector_req = received_requests[2]
assert "name" in vector_req
assert vector_req["name"] == "custom_vector_idx"
assert vector_req["replace"] is False
table.wait_for_index(["custom_scalar_idx"], timedelta(seconds=2))
table.wait_for_index(
@@ -1107,9 +1079,6 @@ def test_remote_create_index_new_api():
table.create_index("text", config=FTS(block_size=256))
# IvfRq via new API
table.create_index("vector", config=IvfRq(distance_type="l2"))
table.create_index(
"vector", config=IvfPq(distance_type="l2"), replace=False
)
# Legacy index_type="IVF_RQ" routes to IvfRq config under the hood.
with pytest.warns(DeprecationWarning, match="create_index"):
@@ -1119,17 +1088,15 @@ def test_remote_create_index_new_api():
num_partitions=8,
)
assert len(received_requests) == 6
assert len(received_requests) == 5
assert [req["column"] for req in received_requests] == [
"vector",
"category",
"text",
"vector",
"vector",
"vector",
]
assert received_requests[2]["block_size"] == 256
assert received_requests[4]["replace"] is False
def test_table_wait_for_index_timeout():
+8 -68
View File
@@ -1600,7 +1600,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
)
mock_create_index.assert_called_with(
"my_vector",
replace=False,
replace=True,
config=expected_config,
wait_timeout=None,
name=None,
@@ -1620,7 +1620,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
)
mock_create_index.assert_called_with(
"my_vector",
replace=False,
replace=True,
config=expected_config,
wait_timeout=None,
name=None,
@@ -1646,7 +1646,7 @@ def test_create_index_name_and_train_parameters(
expected_config = IvfPq() # Default config
mock_create_index.assert_called_with(
"vector",
replace=False,
replace=True,
config=expected_config,
wait_timeout=None,
name="my_custom_index",
@@ -1657,7 +1657,7 @@ def test_create_index_name_and_train_parameters(
table.create_index(vector_column_name="vector", train=False)
mock_create_index.assert_called_with(
"vector",
replace=False,
replace=True,
config=expected_config,
wait_timeout=None,
name=None,
@@ -1668,7 +1668,7 @@ def test_create_index_name_and_train_parameters(
table.create_index(vector_column_name="vector", name="my_index_name", train=True)
mock_create_index.assert_called_with(
"vector",
replace=False,
replace=True,
config=expected_config,
wait_timeout=None,
name="my_index_name",
@@ -1705,7 +1705,7 @@ def test_create_index_new_api(mock_create_index, mem_db: DBConnection):
table.create_index("vector", config=IvfPq(distance_type="l2"))
mock_create_index.assert_called_with(
"vector",
replace=False,
replace=True,
config=IvfPq(distance_type="l2"),
wait_timeout=None,
name=None,
@@ -1716,7 +1716,7 @@ def test_create_index_new_api(mock_create_index, mem_db: DBConnection):
table.create_index("category", config=BTree())
mock_create_index.assert_called_with(
"category",
replace=False,
replace=True,
config=BTree(),
wait_timeout=None,
name=None,
@@ -1727,7 +1727,7 @@ def test_create_index_new_api(mock_create_index, mem_db: DBConnection):
table.create_index("text", config=FTS(with_position=True))
mock_create_index.assert_called_with(
"text",
replace=False,
replace=True,
config=FTS(with_position=True),
wait_timeout=None,
name=None,
@@ -2682,43 +2682,6 @@ def test_merge_insert(mem_db: DBConnection):
)
def test_merge_insert_composite_key(mem_db: DBConnection):
table = mem_db.create_table(
"my_table",
data=pa.table(
{
"shard": ["a", "a", "b"],
"id": [1, 2, 1],
"val": ["x", "y", "z"],
}
),
)
# ("a", 1) matches an existing row and updates it. ("b", 2) agrees with an
# existing row on each key column separately but on neither pair, so it is
# an insert.
new_data = pa.table({"shard": ["a", "b"], "id": [1, 2], "val": ["X", "W"]})
res = (
table.merge_insert(["shard", "id"])
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(new_data)
)
assert res.num_updated_rows == 1
assert res.num_inserted_rows == 1
expected = pa.table(
{
"shard": ["a", "a", "b", "b"],
"id": [1, 2, 1, 2],
"val": ["X", "y", "z", "W"],
}
)
assert table.to_arrow().sort_by([("shard", "ascending"), ("id", "ascending")]) == (
expected
)
def test_merge_insert_nullable_pandas_into_pydantic_schema(mem_db: DBConnection):
# Regression test for https://github.com/lancedb/lancedb/issues/2366
pd = pytest.importorskip("pandas")
@@ -4124,29 +4087,6 @@ def test_computed_column_rejects_transforms_and_computed_together(tmp_path):
table.add_columns({"a": "x + 1"}, computed={"b": "x * 2"})
def test_computed_column_blob_projection_inherits_semantics(tmp_path):
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
db = lancedb.connect(tmp_path)
table = db.create_table("computed_column_blob", schema=schema)
table.add(
[
{"id": 1, "image": b"hello"},
{"id": 2, "image": b""},
{"id": 3, "image": None},
]
)
table.add_columns(computed={"image_copy": "image", "second_copy": "image_copy"})
assert table.refresh_column("image_copy").rows_filled == 2
assert table.refresh_column("second_copy").rows_filled == 2
assert table.blob_columns() == ["image", "image_copy", "second_copy"]
hits = table.search().with_row_id(True).limit(10).to_arrow()
rows = sorted(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
copied = table.fetch_blobs("second_copy", [row_id for _, row_id in rows])
assert copied.to_pylist() == [b"hello", b"", None]
@pytest.mark.asyncio
async def test_computed_column_async(tmp_path):
db = await lancedb.connect_async(tmp_path)
-24
View File
@@ -629,30 +629,6 @@ impl Connection {
})
}
pub fn list_functions(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
inner
.list_functions()
.await
.infer_error()?
.into_iter()
.map(|function| function.to_canonical_json().infer_error())
.collect::<PyResult<Vec<_>>>()
})
}
pub fn drop_function(
self_: PyRef<'_, Self>,
name: String,
version: String,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
inner.drop_function(name, version).await.infer_error()
})
}
pub fn list_jobs(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
-3
View File
@@ -323,7 +323,6 @@ impl<'py> IntoPyObject<'py> for PyQueryVectors {
pub struct PyQueryRequest {
pub limit: Option<usize>,
pub offset: Option<usize>,
pub take_offsets: Option<Vec<u64>>,
pub filter: Option<PyQueryFilter>,
pub full_text_search: Option<PyLanceDB<FtsQuery>>,
pub select: PySelect,
@@ -354,7 +353,6 @@ impl From<AnyQuery> for PyQueryRequest {
AnyQuery::Query(query_request) => Self {
limit: query_request.limit,
offset: query_request.offset,
take_offsets: query_request.take_offsets,
filter: query_request.filter.map(PyQueryFilter),
full_text_search: query_request
.full_text_search
@@ -383,7 +381,6 @@ impl From<AnyQuery> for PyQueryRequest {
AnyQuery::VectorQuery(vector_query) => Self {
limit: vector_query.base.limit,
offset: vector_query.base.offset,
take_offsets: vector_query.base.take_offsets,
filter: vector_query.base.filter.map(PyQueryFilter),
full_text_search: None,
select_source_columns: PySelect::source_columns(&vector_query.base.select),
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb"
version = "0.39.0-beta.1"
version = "0.38.0-beta.12"
edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true
-37
View File
@@ -523,43 +523,6 @@ impl Connection {
.await
}
/// List every published immutable Function version in the remote catalog.
///
/// Results are ordered by Function name then version. The client walks all
/// server pages before returning. Local databases return
/// [`Error::NotSupported`].
///
/// # Example
///
/// ```no_run
/// # async fn list_functions(
/// # connection: &lancedb::Connection,
/// # ) -> Result<(), Box<dyn std::error::Error>> {
/// for function in connection.list_functions().await? {
/// println!("{} {}", function.name(), function.version());
/// }
/// # Ok(())
/// # }
/// ```
pub async fn list_functions(&self) -> Result<Vec<crate::function::FunctionVersion>> {
self.internal.list_functions().await
}
/// Drop one exact immutable Function version from the remote catalog.
///
/// Returns `true` when the server appended a Dropped transition and
/// `false` for an idempotent replay. Local databases return
/// [`Error::NotSupported`].
pub async fn drop_function(
&self,
name: impl AsRef<str>,
version: impl AsRef<str>,
) -> Result<bool> {
self.internal
.drop_function(name.as_ref(), version.as_ref())
.await
}
/// Rename a table in the database.
///
/// This is only supported in LanceDB Cloud.
-8
View File
@@ -307,14 +307,6 @@ pub trait Database:
) -> Result<crate::function::FunctionVersion> {
function_catalog_not_supported()
}
/// List every published immutable Function version in the remote catalog.
async fn list_functions(&self) -> Result<Vec<crate::function::FunctionVersion>> {
function_catalog_not_supported()
}
/// Drop one exact immutable Function version from the remote catalog.
async fn drop_function(&self, _name: &str, _version: &str) -> Result<bool> {
function_catalog_not_supported()
}
/// A [`crate::job::Job`] handle for a server-side job by id, suitable for
/// waiting on or cancelling the job. The handle is constructed without a
/// server round trip; an unknown id surfaces when the handle is used.
+3 -10
View File
@@ -539,7 +539,9 @@ impl Database for LanceNamespaceDatabase {
self.namespace
.drop_table(drop_request)
.await
.map_err(|e| map_namespace_lance_error(e, name))?;
.map_err(|e| Error::Runtime {
message: format!("Failed to drop table: {}", e),
})?;
Ok(())
}
@@ -1493,15 +1495,6 @@ mod tests {
.expect("Failed to list tables");
assert!(!table_names_after.contains(&"drop_test".to_string()));
let error = conn
.drop_table("drop_test", &["test_ns".into()])
.await
.expect_err("dropping a missing table should fail");
assert!(
matches!(error, Error::TableNotFound { ref name, .. } if name == "drop_test"),
"expected TableNotFound, got: {error:?}"
);
// Verify: Cannot open dropped table
let open_result = conn.open_table("drop_test").execute().await;
assert!(open_result.is_err());
@@ -31,7 +31,7 @@ use lance::io::RecordBatchStream;
use lance_arrow::RecordBatchExt;
use lance_core::ROW_ID;
use lance_core::error::LanceOptionExt;
use std::collections::{HashMap, HashSet};
use std::collections::HashMap;
use std::sync::Arc;
/// Reads a permutation of a source table based on row IDs stored in a separate table
@@ -234,14 +234,7 @@ impl PermutationReader {
.expect_ok()?
.values();
let mut unique_row_ids = HashSet::with_capacity(num_rows);
let in_list: Vec<Expr> = row_ids
.iter()
.copied()
.filter(|row_id| unique_row_ids.insert(*row_id))
.map(lit)
.collect();
let num_unique_row_ids = unique_row_ids.len();
let in_list: Vec<Expr> = row_ids.iter().map(|id| lit(*id)).collect();
let base_query = QueryRequest {
filter: Some(QueryFilter::Datafusion(col(ROW_ID).in_list(in_list, false))),
@@ -254,7 +247,7 @@ impl PermutationReader {
.query(
&AnyQuery::Query(base_query),
QueryExecutionOptions {
max_batch_length: num_unique_row_ids as u32,
max_batch_length: num_rows as u32,
..Default::default()
},
)
@@ -269,9 +262,9 @@ impl PermutationReader {
});
}
if batches.iter().map(|b| b.num_rows()).sum::<usize>() != num_unique_row_ids {
if batches.iter().map(|b| b.num_rows()).sum::<usize>() != num_rows {
return Err(Error::InvalidInput {
message: "Base table returned a different number of rows than the number of unique row IDs"
message: "Base table returned different number of rows than the number of row IDs"
.to_string(),
});
}
@@ -511,7 +504,6 @@ impl PermutationReader {
let table = Table::from(self.base_table.clone());
let batches = table
.take_offsets(offsets.to_vec())
.preserve_order()
.select(selection.clone())
.execute()
.await?
@@ -811,10 +803,10 @@ mod tests {
.unwrap();
// Take offsets in reverse order and verify returned rows match that order
let offsets = vec![5, 3, 5, 1, 0];
let offsets = vec![5, 3, 1, 0];
let batch = reader.take_offsets(&offsets, Select::All).await.unwrap();
assert_eq!(batch.num_rows(), 5);
assert_eq!(batch.num_rows(), 4);
let idx_values = batch
.column(0)
@@ -828,52 +820,6 @@ mod tests {
assert_eq!(idx_values, expected);
}
#[tokio::test]
async fn test_take_offsets_preserves_repeated_rows_in_permutation() {
let base_table = lance_datagen::gen_batch()
.col("idx", lance_datagen::array::step::<Int32Type>())
.into_mem_table("tbl", RowCount::from(5), BatchCount::from(1))
.await;
let base_row_ids = collect_column::<UInt64Type>(&base_table, "_rowid").await;
let permutation_row_ids = vec![
base_row_ids[3],
base_row_ids[1],
base_row_ids[3],
base_row_ids[2],
];
let permutation_batch = RecordBatch::try_new(
Arc::new(Schema::new(vec![
Field::new("row_id", DataType::UInt64, false),
Field::new(SPLIT_ID_COLUMN, DataType::UInt64, false),
])),
vec![
Arc::new(UInt64Array::from(permutation_row_ids)),
Arc::new(UInt64Array::from(vec![0; 4])),
],
)
.unwrap();
let permutation_table = virtual_table("row_ids", &permutation_batch).await;
let reader = PermutationReader::try_from_tables(
base_table.base_table().clone(),
permutation_table.base_table().clone(),
0,
)
.await
.unwrap();
let batch = reader
.take_offsets(&[0, 1, 2, 3], Select::All)
.await
.unwrap();
let idx_values = batch
.column(0)
.as_primitive::<Int32Type>()
.values()
.to_vec();
assert_eq!(idx_values, vec![3, 1, 3, 2]);
}
#[tokio::test]
async fn test_take_offsets_with_column_selection() {
let (base_table, row_ids_table, row_ids) = setup_permutation_tables(10).await;
@@ -937,17 +883,17 @@ mod tests {
.unwrap();
// With no permutation table, take_offsets uses the base table directly
let offsets = vec![0, 2, 0, 4, 6];
let offsets = vec![0, 2, 4, 6];
let batch = reader.take_offsets(&offsets, Select::All).await.unwrap();
assert_eq!(batch.num_rows(), 5);
assert_eq!(batch.num_rows(), 4);
let idx_values = batch
.column(0)
.as_primitive::<Int32Type>()
.values()
.to_vec();
assert_eq!(idx_values, vec![0, 2, 0, 4, 6]);
assert_eq!(idx_values, vec![0, 2, 4, 6]);
}
#[tokio::test]
+27 -153
View File
@@ -15,9 +15,6 @@ use serde_json::Value;
use crate::{Error, Result};
/// Semantic Function type for a Blob v2 value.
pub const FUNCTION_BLOB_V2_TYPE: &str = "blob_v2";
fn invalid_json(error: impl std::fmt::Display) -> Error {
Error::InvalidInput {
message: format!("invalid remote Function JSON: {error}"),
@@ -210,33 +207,6 @@ pub enum PythonRuntimeSpec {
environment: PythonEnvironmentSpec,
env: BTreeMap<String, String>,
},
/// The GPU-enabled Sophon-managed Python runtime.
///
/// # Examples
///
/// ```
/// use std::collections::BTreeMap;
/// use lancedb::function::{PythonEnvironmentSpec, PythonRuntimeSpec};
///
/// let runtime = PythonRuntimeSpec::PythonV2 {
/// python_version: "3.12".to_string(),
/// environment: PythonEnvironmentSpec {
/// kind: "pip".to_string(),
/// packages: vec!["cupy-cuda12x".to_string()],
/// channels: Vec::new(),
/// path: None,
/// modules: Vec::new(),
/// image: None,
/// },
/// env: BTreeMap::new(),
/// };
/// assert!(runtime.requires_gpu());
/// ```
PythonV2 {
python_version: String,
environment: PythonEnvironmentSpec,
env: BTreeMap<String, String>,
},
/// A runtime kind introduced by a newer server.
///
/// Unknown payload fields are intentionally not retained because the
@@ -249,27 +219,22 @@ impl PythonRuntimeSpec {
pub fn kind(&self) -> &str {
match self {
Self::Python { .. } => "python",
Self::PythonV2 { .. } => "python_v2",
Self::Unrecognized { kind } => kind,
}
}
/// The Python version for a known Python runtime, or `None` for an unknown kind.
/// The Python version for the V1 runtime, or `None` for an unknown kind.
pub fn python_version(&self) -> Option<&str> {
match self {
Self::Python { python_version, .. } | Self::PythonV2 { python_version, .. } => {
Some(python_version)
}
Self::Python { python_version, .. } => Some(python_version),
Self::Unrecognized { .. } => None,
}
}
/// The Python environment for a known Python runtime, or `None` for an unknown kind.
/// The Python environment for the V1 runtime, or `None` for an unknown kind.
pub fn environment(&self) -> Option<&PythonEnvironmentSpec> {
match self {
Self::Python { environment, .. } | Self::PythonV2 { environment, .. } => {
Some(environment)
}
Self::Python { environment, .. } => Some(environment),
Self::Unrecognized { .. } => None,
}
}
@@ -277,73 +242,38 @@ impl PythonRuntimeSpec {
/// Environment variables, or `None` for an unknown kind.
pub fn env(&self) -> Option<&BTreeMap<String, String>> {
match self {
Self::Python { env, .. } | Self::PythonV2 { env, .. } => Some(env),
Self::Python { env, .. } => Some(env),
Self::Unrecognized { .. } => None,
}
}
/// Whether the runtime requires a GPU selected by the execution platform.
pub fn requires_gpu(&self) -> bool {
matches!(self, Self::PythonV2 { .. })
}
}
#[derive(Deserialize)]
struct PythonRuntimeV1Wire {
python_version: String,
environment: PythonEnvironmentSpec,
struct PythonRuntimeWire {
kind: String,
#[serde(default)]
python_version: Option<String>,
#[serde(default)]
environment: Option<PythonEnvironmentSpec>,
#[serde(default)]
env: BTreeMap<String, String>,
#[serde(default)]
gpu: Option<Value>,
}
#[derive(Deserialize)]
struct PythonRuntimeV2Wire {
python_version: String,
environment: PythonEnvironmentSpec,
#[serde(default)]
env: BTreeMap<String, String>,
gpu: bool,
}
impl<'de> Deserialize<'de> for PythonRuntimeSpec {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
let value = Value::deserialize(deserializer)?;
let kind = value
.get("kind")
.ok_or_else(|| de::Error::missing_field("kind"))?
.as_str()
.ok_or_else(|| de::Error::custom("runtime.kind must be a string"))?
.to_string();
match kind.as_str() {
"python" => {
let wire: PythonRuntimeV1Wire =
serde_json::from_value(value).map_err(de::Error::custom)?;
if wire.gpu.is_some() {
return Err(de::Error::custom(
"python runtime with gpu requires kind='python_v2'",
));
}
Ok(Self::Python {
python_version: wire.python_version,
environment: wire.environment,
env: wire.env,
})
}
"python_v2" => {
let wire: PythonRuntimeV2Wire =
serde_json::from_value(value).map_err(de::Error::custom)?;
if !wire.gpu {
return Err(de::Error::custom("runtime.gpu must be true"));
}
Ok(Self::PythonV2 {
python_version: wire.python_version,
environment: wire.environment,
env: wire.env,
})
}
_ => Ok(Self::Unrecognized { kind }),
let wire = PythonRuntimeWire::deserialize(deserializer)?;
if wire.kind == "python" {
Ok(Self::Python {
python_version: wire
.python_version
.ok_or_else(|| de::Error::missing_field("python_version"))?,
environment: wire
.environment
.ok_or_else(|| de::Error::missing_field("environment"))?,
env: wire.env,
})
} else {
Ok(Self::Unrecognized { kind: wire.kind })
}
}
}
@@ -357,8 +287,6 @@ impl Serialize for PythonRuntimeSpec {
environment: &'a PythonEnvironmentSpec,
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
env: &'a BTreeMap<String, String>,
#[serde(skip_serializing_if = "Option::is_none")]
gpu: Option<bool>,
}
#[derive(Serialize)]
@@ -376,19 +304,6 @@ impl Serialize for PythonRuntimeSpec {
python_version,
environment,
env,
gpu: None,
}
.serialize(serializer),
Self::PythonV2 {
python_version,
environment,
env,
} => PythonRuntimeRef {
kind: "python_v2",
python_version,
environment,
env,
gpu: Some(true),
}
.serialize(serializer),
Self::Unrecognized { kind } => UnrecognizedRuntimeRef { kind }.serialize(serializer),
@@ -398,8 +313,8 @@ impl Serialize for PythonRuntimeSpec {
/// Immutable Function version returned by the Enterprise catalog.
///
/// The GPU execution requirement is part of this identity. CPU and memory sizing,
/// priority, concurrency, and retry policy belong to the execution platform.
/// Scheduling resources, priority, concurrency, and retry policy belong to
/// the submitting Job and are not part of this identity.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FunctionVersion {
name: String,
@@ -674,7 +589,7 @@ impl_json!(RefreshColumnResult);
#[cfg(test)]
mod conda_environment_tests {
use super::{PythonEnvironmentSpec, PythonRuntimeSpec};
use super::PythonEnvironmentSpec;
#[test]
fn conda_channels_round_trip_and_pip_stays_bare() {
@@ -693,45 +608,4 @@ mod conda_environment_tests {
serde_json::from_str(r#"{"kind":"pip","packages":["numpy"]}"#).unwrap();
assert!(!serde_json::to_string(&pip).unwrap().contains("channels"));
}
#[test]
fn gpu_python_runtime_marker_round_trips_and_validates() {
let runtime: PythonRuntimeSpec = serde_json::from_str(
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":true}"#,
)
.unwrap();
assert_eq!(runtime.kind(), "python_v2");
assert!(runtime.requires_gpu());
assert_eq!(
super::canonical_json(&runtime).unwrap(),
r#"{"environment":{"kind":"pip"},"gpu":true,"kind":"python_v2","python_version":"3.12"}"#
);
for invalid in [
r#"{"kind":"python","python_version":"3.12","environment":{"kind":"pip"},"gpu":true}"#,
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"}}"#,
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":1}"#,
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":false}"#,
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":"true"}"#,
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":"H100"}"#,
] {
assert!(serde_json::from_str::<PythonRuntimeSpec>(invalid).is_err());
}
}
#[test]
fn unknown_runtime_discards_payload_before_known_field_validation() {
for encoded in [
r#"{"kind":"python_v3","gpu":{"model":"H100"}}"#,
r#"{"kind":"python_v3","resources":[]}"#,
r#"{"kind":"python_v3","python_version":3.15,"environment":{"kind":[]}}"#,
] {
let runtime: PythonRuntimeSpec = serde_json::from_str(encoded).unwrap();
assert_eq!(runtime.kind(), "python_v3");
assert_eq!(
super::canonical_json(&runtime).unwrap(),
r#"{"kind":"python_v3"}"#
);
}
}
}
+2 -2
View File
@@ -200,14 +200,14 @@ impl IndexBuilder {
parent,
index,
columns,
replace: false,
replace: true,
train: true,
wait_timeout: None,
name: None,
}
}
/// Whether to replace the existing index, the default is `false`.
/// Whether to replace the existing index, the default is `true`.
///
/// If this is false, and another index already exists on the same columns
/// and the same name, then an error will be returned. This is true even if
+17 -35
View File
@@ -40,7 +40,6 @@ impl TerminalResult {
}
}
#[cfg(feature = "remote")]
pub(crate) fn remote(value: Option<Value>, request_id: String) -> Self {
Self {
value,
@@ -48,46 +47,30 @@ impl TerminalResult {
}
}
#[cfg(feature = "remote")]
pub(crate) fn value(&self) -> Option<&Value> {
self.value.as_ref()
}
fn decode<T: DeserializeOwned>(self) -> Result<T> {
let value = self.value.ok_or_else(|| {
#[cfg(feature = "remote")]
if let Some(request_id) = &self.request_id {
return Error::Http {
source: "successful typed job response did not contain a result".into(),
request_id: request_id.clone(),
status_code: None,
};
}
Error::Runtime {
let value = self.value.ok_or_else(|| match &self.request_id {
Some(request_id) => Error::Http {
source: "successful typed job response did not contain a result".into(),
request_id: request_id.clone(),
status_code: None,
},
None => Error::Runtime {
message: "successful typed job did not contain a result".to_string(),
}
},
})?;
serde_json::from_value(value).map_err(|error| {
#[cfg(feature = "remote")]
{
match self.request_id {
Some(request_id) => Error::Http {
source: format!("failed to parse typed job result: {error}").into(),
request_id,
status_code: None,
},
None => Error::Runtime {
message: format!("failed to parse typed job result: {error}"),
},
}
}
#[cfg(not(feature = "remote"))]
{
let _ = self.request_id;
Error::Runtime {
message: format!("failed to parse typed job result: {error}"),
}
}
serde_json::from_value(value).map_err(|error| match self.request_id {
Some(request_id) => Error::Http {
source: format!("failed to parse typed job result: {error}").into(),
request_id,
status_code: None,
},
None => Error::Runtime {
message: format!("failed to parse typed job result: {error}"),
},
})
}
}
@@ -134,7 +117,6 @@ impl Job<()> {
}
}
#[cfg(feature = "remote")]
pub(crate) fn new(handle: Box<dyn JobHandle>) -> Self {
Self {
inner: JobInner::Handle {
+47 -207
View File
@@ -74,15 +74,8 @@ const EMBEDDING_FUNCTIONS_META_KEY: &str = "embedding_functions";
const COLUMN_DEFINITIONS_META_KEY: &str = "lancedb::column_definitions";
/// Value of the definition's `kind` tag for the projected `select` form.
/// Reserved for root-namespace sources; see [`NAMESPACED_SELECT_KIND`].
pub const SELECT_KIND: &str = "select";
/// The `select` form over a namespaced source: its own kind, because released
/// readers drop unknown fields and resolve a `select` source at the root, so
/// this routes them to the [`MaterializedViewKind::Unrecognized`] refusal
/// instead of a wrong-table refresh.
pub const NAMESPACED_SELECT_KIND: &str = "namespaced_select";
/// Which view outputs each source column is projected to directly. A column
/// may be projected more than once, so each carries every name the view gives
/// it, in projection order.
@@ -102,10 +95,6 @@ pub struct ViewProjection {
pub struct MaterializedViewDefinition {
/// Name of the source table, in the same database as the view.
pub source_table: String,
/// Namespace path holding the source table; empty is the root namespace.
/// A definition written before namespaced sources reads as root.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub source_namespace: Vec<String>,
/// The projected output columns, in view schema order.
pub projections: Vec<ViewProjection>,
/// SQL predicate selecting the source rows the view holds.
@@ -140,12 +129,7 @@ pub(crate) fn definition_to_metadata(definition: &MaterializedViewDefinition) ->
let mut value = serde_json::to_value(definition).map_err(|e| Error::Runtime {
message: format!("failed to serialize view definition: {e}"),
})?;
let kind = if definition.source_namespace.is_empty() {
SELECT_KIND
} else {
NAMESPACED_SELECT_KIND
};
value["kind"] = serde_json::Value::String(kind.to_string());
value["kind"] = serde_json::Value::String(SELECT_KIND.to_string());
Ok(value.to_string())
}
@@ -166,21 +150,12 @@ pub fn materialized_view_kind(
.get("kind")
.and_then(|k| k.as_str())
.ok_or_else(|| unreadable(&"missing kind tag"))?;
if kind != SELECT_KIND && kind != NAMESPACED_SELECT_KIND {
if kind != SELECT_KIND {
return Ok(Some(MaterializedViewKind::Unrecognized {
kind: kind.to_string(),
}));
}
let kind = kind.to_string();
let definition: MaterializedViewDefinition =
serde_json::from_value(value).map_err(|e| unreadable(&e))?;
// No correct writer produces a kind that disagrees with its namespace.
if (kind == SELECT_KIND) != definition.source_namespace.is_empty() {
return Err(unreadable(&format!(
"kind '{kind}' does not match its source namespace {:?}",
definition.source_namespace
)));
}
let definition = serde_json::from_value(value).map_err(|e| unreadable(&e))?;
Ok(Some(MaterializedViewKind::Select(definition)))
}
@@ -191,7 +166,6 @@ pub fn materialized_view_kind(
pub(crate) fn plan(
source_schema: SchemaRef,
source_table: &str,
source_namespace: &[String],
projections: &[(String, String)],
filter: Option<&str>,
limit: Option<u64>,
@@ -345,7 +319,6 @@ pub(crate) fn plan(
let definition = MaterializedViewDefinition {
source_table: source_table.to_string(),
source_namespace: source_namespace.to_vec(),
projections: projections
.into_iter()
.map(|(output, expression)| ViewProjection { output, expression })
@@ -629,7 +602,7 @@ pub struct PreparedDeclaration {
definition: MaterializedViewDefinition,
/// The source's own database: the only place
/// [`PreparedDeclaration::create`] will put the view, because refresh
/// resolves the recorded source coordinate through the view's database.
/// resolves the recorded source name through the view's database.
database: Arc<dyn Database>,
}
@@ -649,21 +622,10 @@ impl PreparedDeclaration {
/// Create the view table and verify it, consuming the declaration.
///
/// The view goes at the root of the source's own database, where refresh
/// resolves the recorded source coordinate. Stable row ids are requested
/// at both levels and verified rather than trusted; nothing is rolled
/// back on failure.
/// The view goes in the source's own database, where refresh resolves the
/// recorded source name. Stable row ids are requested at both levels and
/// verified rather than trusted; nothing is rolled back on failure.
pub async fn create(self, name: &str) -> Result<MaterializedView> {
self.create_in(&[], name).await
}
/// Create the view in `namespace_path`, empty for the root namespace.
/// Otherwise [`PreparedDeclaration::create`].
pub async fn create_in(
self,
namespace_path: &[String],
name: &str,
) -> Result<MaterializedView> {
let empty: Vec<std::result::Result<arrow_array::RecordBatch, arrow_schema::ArrowError>> =
vec![];
// Minted here, not at preparation: a declaration can be cloned and
@@ -678,7 +640,6 @@ impl PreparedDeclaration {
let reader: Box<dyn arrow_array::RecordBatchReader + Send> =
Box::new(arrow_array::RecordBatchIterator::new(empty, schema));
let mut request = CreateTableRequest::new(name.to_string(), Box::new(reader));
request.namespace_path = namespace_path.to_vec();
let write_params = request
.write_options
.lance_write_params
@@ -719,8 +680,8 @@ impl PreparedDeclaration {
/// Validate a view declaration against its live source and hold what its
/// creation needs. The declaration is canonicalized through the coordinate a
/// refresh will resolve -- name and namespace both -- so a handle that does
/// not resolve back to itself is rejected. Same creation-time checks as
/// refresh will resolve, so a handle that does not resolve back to itself is
/// rejected, as is a namespaced source. Same creation-time checks as
/// [`Connection::create_materialized_view`].
///
/// ```no_run
@@ -749,9 +710,17 @@ pub async fn prepare_declaration(
message: "materialized views are supported only on local databases".into(),
});
};
// Refresh resolves the source at exactly this coordinate, so the
// definition records the namespace alongside the name.
let source_namespace = source.namespace().to_vec();
// The definition records the source by bare name; any other source
// form would be recorded as a name its refresh cannot resolve.
if !source.namespace().is_empty() {
return Err(Error::NotSupported {
message: format!(
"a namespaced source cannot be recorded in a view definition; \
'{}' must be a root-namespace table",
source.name()
),
});
}
let database = source
.database_opt()
.ok_or_else(|| Error::InvalidInput {
@@ -765,7 +734,7 @@ pub async fn prepare_declaration(
let resolved = database
.open_table(OpenTableRequest {
name: source.name().to_string(),
namespace_path: source_namespace.clone(),
namespace_path: vec![],
index_cache_size: None,
lance_read_params: None,
location: None,
@@ -811,7 +780,6 @@ pub async fn prepare_declaration(
let (definition, mut fields, lineage) = plan(
source_schema.clone(),
resolved.name(),
&source_namespace,
projections,
filter,
limit,
@@ -871,9 +839,7 @@ fn ensure_local(connection: &Connection) -> Result<()> {
pub struct CreateMaterializedViewBuilder {
connection: Connection,
name: String,
namespace: Vec<String>,
source: String,
source_namespace: Vec<String>,
projections: Vec<(String, String)>,
filter: Option<String>,
limit: Option<u64>,
@@ -884,28 +850,13 @@ impl CreateMaterializedViewBuilder {
Self {
connection,
name,
namespace: Vec::new(),
source,
source_namespace: Vec::new(),
projections: Vec::new(),
filter: None,
limit: None,
}
}
/// The namespace to create the view in. Defaults to the root namespace.
pub fn namespace(mut self, namespace_path: Vec<String>) -> Self {
self.namespace = namespace_path;
self
}
/// The namespace holding the source table; recorded in the definition
/// for refresh to resolve. Defaults to the root namespace.
pub fn source_namespace(mut self, namespace_path: Vec<String>) -> Self {
self.source_namespace = namespace_path;
self
}
/// The view's columns, as `(name, SQL expression)` pairs. Not calling
/// this selects every source column, expanded at creation time.
pub fn select(
@@ -936,12 +887,7 @@ impl CreateMaterializedViewBuilder {
/// provenance across compaction, and cannot be enabled later.
pub async fn execute(self) -> Result<MaterializedView> {
ensure_local(&self.connection)?;
let source = self
.connection
.open_table(&self.source)
.namespace(self.source_namespace.clone())
.execute()
.await?;
let source = self.connection.open_table(&self.source).execute().await?;
let prepared = prepare_declaration(
&source,
&self.projections,
@@ -949,7 +895,7 @@ impl CreateMaterializedViewBuilder {
self.limit,
)
.await?;
prepared.create_in(&self.namespace, &self.name).await
prepared.create(&self.name).await
}
}
@@ -1206,7 +1152,6 @@ mod tests {
view.definition(),
&MaterializedViewDefinition {
source_table: "people".into(),
source_namespace: Vec::new(),
projections: vec![
ViewProjection {
output: "name".into(),
@@ -2138,138 +2083,33 @@ mod tests {
.await
.unwrap_err();
assert!(err.to_string().contains("custom_loc"), "{err}");
}
/// A view declared over a namespaced source records that namespace, and
/// refresh resolves the source through it -- the coordinate round-trips.
#[tokio::test]
async fn a_namespaced_source_round_trips_through_refresh() {
use lance_namespace::models::CreateNamespaceRequest;
let tmp = tempfile::tempdir().unwrap();
let mut properties = std::collections::HashMap::new();
properties.insert("root".to_string(), tmp.path().to_str().unwrap().to_string());
let conn = crate::connect_namespace("dir", properties)
.execute()
.await
.unwrap();
conn.create_namespace(CreateNamespaceRequest {
id: Some(vec!["ns".into()]),
..Default::default()
})
// A namespaced source cannot be recorded in the definition: the
// bare name refresh resolves would reach a different table or none.
let namespaced = crate::table::NativeTable::create(
"memory://ns_src",
"ns_src",
vec!["ns".to_string()],
Box::new(arrow_array::RecordBatchIterator::new(
vec![],
std::sync::Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
"id",
arrow_schema::DataType::Int32,
true,
)])),
)) as Box<dyn arrow_array::RecordBatchReader + Send>,
None,
None,
None,
None,
std::collections::HashSet::new(),
)
.await
.unwrap();
let batch = record_batch!(
("name", Utf8, ["ada", "grace", "alan"]),
("age", Int32, [36, 85, 41])
)
.unwrap();
conn.create_table("people", batch)
.namespace(vec!["ns".to_string()])
.write_options(stable_row_ids())
.execute()
let namespaced = Table::new(std::sync::Arc::new(namespaced), conn.database().clone());
let err = prepare_declaration(&namespaced, &[], None, None)
.await
.unwrap();
// A decoy of the same name at the root: resolving the source at the
// wrong namespace materializes one row here instead of three.
let decoy = record_batch!(("name", Utf8, ["mallory"]), ("age", Int32, [42])).unwrap();
conn.create_table("people", decoy)
.write_options(stable_row_ids())
.execute()
.await
.unwrap();
let view = conn
.create_materialized_view("adults", "people")
.namespace(vec!["ns".to_string()])
.source_namespace(vec!["ns".to_string()])
.select([("name", "name")])
.only_if("age >= 18")
.execute()
.await
.unwrap();
assert_eq!(view.definition().source_table, "people");
assert_eq!(view.definition().source_namespace, vec!["ns".to_string()]);
assert_eq!(view.table().namespace(), &["ns"]);
// Refresh resolves the source at the recorded namespace, not at root.
let result = view.refresh().execute().await.unwrap();
assert_eq!(result.rows_written, 3);
}
/// A definition stored before namespaced sources existed carries no
/// namespace key and must read as the root namespace.
#[test]
fn a_definition_without_a_namespace_reads_as_root() {
let stored =
r#"{"source_table":"people","projections":[{"output":"name","expression":"name"}]}"#;
let definition: MaterializedViewDefinition = serde_json::from_str(stored).unwrap();
assert!(definition.source_namespace.is_empty());
}
fn definition(source_namespace: Vec<String>) -> MaterializedViewDefinition {
MaterializedViewDefinition {
source_table: "people".to_string(),
source_namespace,
projections: vec![ViewProjection {
output: "name".to_string(),
expression: "name".to_string(),
}],
filter: None,
limit: None,
inputs: vec!["name".to_string()],
}
}
/// A root definition keeps the pre-namespace `select` form byte-stably;
/// a namespaced one moves off `select`, which sends pre-namespace readers
/// to the `Unrecognized` refusal instead of a root resolve.
#[test]
fn a_namespaced_definition_is_refused_by_the_pre_namespace_reader() {
let root = definition_to_metadata(&definition(Vec::new())).unwrap();
let root: serde_json::Value = serde_json::from_str(&root).unwrap();
assert_eq!(root["kind"], "select");
assert!(
root.get("source_namespace").is_none(),
"a root definition must not grow new keys: {root}"
);
let stored = definition_to_metadata(&definition(vec!["ns".to_string()])).unwrap();
let value: serde_json::Value = serde_json::from_str(&stored).unwrap();
// The pre-namespace discriminator is `kind == "select"`; anything
// else lands in its Unrecognized refusal rather than in a root open.
assert_eq!(value["kind"], "namespaced_select");
// The current reader round-trips the coordinate.
let metadata = HashMap::from([(DEFINITION_META_KEY.to_string(), stored)]);
match materialized_view_kind(&metadata).unwrap() {
Some(MaterializedViewKind::Select(read)) => {
assert_eq!(read.source_namespace, vec!["ns".to_string()])
}
other => panic!("expected the namespaced select form, got {other:?}"),
}
}
/// A kind that disagrees with its namespace is an error, not a view:
/// under `select` it is the shape old readers would resolve at the root.
#[test]
fn a_kind_namespace_mismatch_is_refused() {
for (kind, namespace) in [
(SELECT_KIND, vec!["ns".to_string()]),
(NAMESPACED_SELECT_KIND, Vec::new()),
] {
let mut value = serde_json::to_value(definition(namespace)).unwrap();
value["kind"] = serde_json::Value::String(kind.to_string());
let metadata = HashMap::from([(DEFINITION_META_KEY.to_string(), value.to_string())]);
let err = materialized_view_kind(&metadata).unwrap_err();
assert!(
err.to_string()
.contains("does not match its source namespace"),
"kind '{kind}': {err}"
);
}
.unwrap_err();
assert!(err.to_string().contains("namespaced source"), "{err}");
}
}
@@ -170,7 +170,6 @@ pub(crate) async fn execute_refresh(
let (replanned, mut planned_fields, _renames) = super::plan(
source_schema,
&definition.source_table,
&definition.source_namespace,
&projections,
definition.filter.as_deref(),
definition.limit,
@@ -591,7 +590,7 @@ async fn open_source(view: &Table, definition: &MaterializedViewDefinition) -> R
let source = database
.open_table(OpenTableRequest {
name: definition.source_table.clone(),
namespace_path: definition.source_namespace.clone(),
namespace_path: Vec::new(),
index_cache_size: None,
lance_read_params: None,
location: None,
@@ -2920,7 +2919,6 @@ mod tests {
let replacement = crate::materialized_view::MaterializedViewDefinition {
source_table: "src".into(),
source_namespace: Vec::new(),
projections: vec![
crate::materialized_view::ViewProjection {
output: "x".into(),
@@ -2960,7 +2958,6 @@ mod tests {
let narrower = crate::materialized_view::MaterializedViewDefinition {
source_table: "src".into(),
source_namespace: Vec::new(),
projections: vec![crate::materialized_view::ViewProjection {
output: "x".into(),
expression: "x".into(),
+5 -835
View File
@@ -1,37 +1,21 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::collections::{HashMap, HashSet};
use std::pin::Pin;
use std::sync::Arc;
use std::{future::Future, time::Duration};
use arrow::compute::concat_batches;
use arrow_array::{
Array, Float16Array, Float32Array, Float64Array, RecordBatch, UInt64Array,
cast::AsArray,
make_array,
types::{Int64Type, UInt64Type},
};
use arrow_array::{Array, Float16Array, Float32Array, Float64Array, RecordBatch, make_array};
use arrow_schema::{DataType, SchemaRef};
use datafusion_common::{DataFusionError, Result as DataFusionResult};
use datafusion_execution::TaskContext;
use datafusion_expr::{Expr, col, lit};
use datafusion_physical_expr::{EquivalenceProperties, Partitioning};
use datafusion_physical_plan::{
DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
coalesce_partitions::CoalescePartitionsExec,
execution_plan::{Boundedness, EmissionType},
limit::GlobalLimitExec,
stream::RecordBatchStreamAdapter,
};
use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt, stream, try_join};
use datafusion_physical_plan::ExecutionPlan;
use futures::{FutureExt, TryFutureExt, TryStreamExt, stream, try_join};
use half::f16;
/// Re-export Lance ColumnOrdering type for use in query ordering
pub use lance::dataset::scanner::ColumnOrdering;
use lance::dataset::{ROW_ID, scanner::DatasetRecordBatchStream};
use lance_arrow::RecordBatchExt;
use lance_datafusion::exec::{execute_plan, format_plan as format_analyzed_plan};
use lance_datafusion::exec::execute_plan;
use lance_index::scalar::FullTextSearchQuery;
use lance_index::scalar::inverted::SCORE_COL;
use lance_index::vector::DIST_COL;
@@ -841,14 +825,6 @@ pub struct QueryRequest {
/// Offset of the query.
pub offset: Option<usize>,
/// Dataset offsets whose occurrence multiplicity must be restored after
/// executing the physical lookup represented by this request.
///
/// This is client-side execution metadata used when a [`TakeQuery`] is
/// converted into a request. It is not sent to remote services.
#[doc(hidden)]
pub take_offsets: Option<Vec<u64>>,
/// Apply filter to the returned rows.
pub filter: Option<QueryFilter>,
@@ -917,7 +893,6 @@ impl Default for QueryRequest {
Self {
limit: None,
offset: None,
take_offsets: None,
filter: None,
filter_error: None,
full_text_search: None,
@@ -1554,302 +1529,6 @@ impl HasQuery for VectorQuery {
}
}
fn take_occurrences(offsets: &[u64]) -> HashMap<u64, usize> {
let mut occurrences = HashMap::with_capacity(offsets.len());
for offset in offsets {
*occurrences.entry(*offset).or_insert(0) += 1;
}
occurrences
}
fn restore_take_batch_with_occurrences(
batch: RecordBatch,
offsets: &[u64],
occurrences: &HashMap<u64, usize>,
ordering_column: &str,
drop_ordering_column: bool,
preserve_order: bool,
) -> Result<RecordBatch> {
let actual_offsets = batch
.column_by_name(ordering_column)
.ok_or_else(|| Error::Schema {
message: format!(
"take query result did not include ordering column '{ordering_column}'"
),
})?;
let actual_offsets = match actual_offsets.data_type() {
DataType::UInt64 => actual_offsets
.as_primitive::<UInt64Type>()
.values()
.to_vec(),
DataType::Int64 => actual_offsets
.as_primitive::<Int64Type>()
.values()
.iter()
.map(|offset| {
u64::try_from(*offset).map_err(|_| Error::Schema {
message: format!(
"take query ordering column '{ordering_column}' contained a negative offset"
),
})
})
.collect::<Result<Vec<_>>>()?,
data_type => {
return Err(Error::Schema {
message: format!(
"take query ordering column '{ordering_column}' had unsupported type {data_type}"
),
});
}
};
let mut desired_order = Vec::with_capacity(offsets.len());
if preserve_order {
let ordering = actual_offsets
.iter()
.copied()
.enumerate()
.map(|(index, offset)| (offset, index as u64))
.collect::<HashMap<_, _>>();
// Missing offsets retain the filter-based behavior of returning no row.
desired_order.extend(
offsets
.iter()
.filter_map(|offset| ordering.get(offset).copied()),
);
} else {
// Public take queries do not guarantee output order. Preserve the lookup's
// existing order and only restore the multiplicity of each matching row.
for (index, offset) in actual_offsets.iter().enumerate() {
if let Some(count) = occurrences.get(offset) {
desired_order.extend(std::iter::repeat_n(index as u64, *count));
}
}
}
let mut ordered_batch = if desired_order.len() == batch.num_rows()
&& desired_order
.iter()
.enumerate()
.all(|(index, desired)| *desired == index as u64)
{
batch
} else {
arrow_select::take::take_record_batch(&batch, &UInt64Array::from(desired_order))?
};
if drop_ordering_column {
ordered_batch = ordered_batch.drop_column(ordering_column)?;
}
Ok(ordered_batch)
}
#[cfg(test)]
fn restore_take_batch(
batch: RecordBatch,
offsets: &[u64],
ordering_column: &str,
drop_ordering_column: bool,
preserve_order: bool,
) -> Result<RecordBatch> {
restore_take_batch_with_occurrences(
batch,
offsets,
&take_occurrences(offsets),
ordering_column,
drop_ordering_column,
preserve_order,
)
}
/// Restores the logical offset occurrence sequence above the physical lookup plan.
///
/// The lookup plan returns each matching row at most once. For ordinary unordered
/// takes this operator expands each input batch incrementally and preserves the
/// lookup's partitioning. The explicitly ordered reader path collects one coalesced
/// input before restoring requested order. Pagination must remain above this operator
/// so it applies to occurrences.
#[derive(Debug)]
struct TakeRestoreExec {
input: Arc<dyn ExecutionPlan>,
offsets: Vec<u64>,
occurrences: Arc<HashMap<u64, usize>>,
ordering_column: String,
drop_ordering_column: bool,
preserve_order: bool,
schema: SchemaRef,
properties: Arc<PlanProperties>,
}
impl TakeRestoreExec {
fn try_new(
input: Arc<dyn ExecutionPlan>,
offsets: Vec<u64>,
ordering_column: String,
drop_ordering_column: bool,
preserve_order: bool,
) -> Result<Self> {
let schema = if drop_ordering_column {
RecordBatch::new_empty(input.schema())
.drop_column(&ordering_column)?
.schema()
} else {
input.schema()
};
let partition_count = if preserve_order {
1
} else {
input.output_partitioning().partition_count()
};
let emission_type = if preserve_order {
EmissionType::Final
} else {
EmissionType::Incremental
};
let properties = Arc::new(PlanProperties::new(
EquivalenceProperties::new(schema.clone()),
Partitioning::UnknownPartitioning(partition_count),
emission_type,
Boundedness::Bounded,
));
Ok(Self {
input,
occurrences: Arc::new(take_occurrences(&offsets)),
offsets,
ordering_column,
drop_ordering_column,
preserve_order,
schema,
properties,
})
}
}
impl DisplayAs for TakeRestoreExec {
fn fmt_as(
&self,
_display_type: DisplayFormatType,
formatter: &mut std::fmt::Formatter<'_>,
) -> std::fmt::Result {
write!(
formatter,
"TakeRestoreExec: occurrences={}",
self.offsets.len()
)
}
}
impl ExecutionPlan for TakeRestoreExec {
fn name(&self) -> &str {
"TakeRestoreExec"
}
fn properties(&self) -> &Arc<PlanProperties> {
&self.properties
}
fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
vec![&self.input]
}
fn maintains_input_order(&self) -> Vec<bool> {
vec![!self.preserve_order]
}
fn benefits_from_input_partitioning(&self) -> Vec<bool> {
vec![false]
}
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
) -> DataFusionResult<Arc<dyn ExecutionPlan>> {
if children.len() != 1 {
return Err(DataFusionError::Internal(format!(
"TakeRestoreExec expected one child, got {}",
children.len()
)));
}
let child = children.into_iter().next().unwrap();
let plan = Self::try_new(
child,
self.offsets.clone(),
self.ordering_column.clone(),
self.drop_ordering_column,
self.preserve_order,
)
.map_err(|error| DataFusionError::External(Box::new(error)))?;
Ok(Arc::new(plan))
}
fn execute(
&self,
partition: usize,
context: Arc<TaskContext>,
) -> DataFusionResult<datafusion_physical_plan::SendableRecordBatchStream> {
let partition_count = self.input.output_partitioning().partition_count();
if partition >= partition_count || (self.preserve_order && partition != 0) {
return Err(DataFusionError::Internal(format!(
"TakeRestoreExec cannot execute partition {partition}; input has {partition_count} partitions"
)));
}
let input = self.input.execute(partition, context)?;
let output_schema = self.schema.clone();
let offsets = self.offsets.clone();
let occurrences = self.occurrences.clone();
let ordering_column = self.ordering_column.clone();
let drop_ordering_column = self.drop_ordering_column;
let preserve_order = self.preserve_order;
let stream: Pin<Box<dyn futures::Stream<Item = DataFusionResult<RecordBatch>> + Send>> =
if preserve_order {
let input_schema = input.schema();
Box::pin(stream::once(async move {
let batches = input.try_collect::<Vec<_>>().await?;
let batch = if batches.is_empty() {
RecordBatch::new_empty(input_schema.clone())
} else {
concat_batches(&input_schema, &batches)?
};
restore_take_batch_with_occurrences(
batch,
&offsets,
&occurrences,
&ordering_column,
drop_ordering_column,
true,
)
.map_err(|error| DataFusionError::External(Box::new(error)))
}))
} else {
Box::pin(input.map(move |batch| {
batch.and_then(|batch| {
restore_take_batch_with_occurrences(
batch,
&offsets,
&occurrences,
&ordering_column,
drop_ordering_column,
false,
)
.map_err(|error| DataFusionError::External(Box::new(error)))
})
}))
};
Ok(Box::pin(RecordBatchStreamAdapter::new(
output_schema,
stream,
)))
}
fn supports_limit_pushdown(&self) -> bool {
false
}
}
/// A builder for LanceDB take queries.
///
/// See [`crate::Table::query`] for more details on queries
@@ -1866,8 +1545,6 @@ impl ExecutionPlan for TakeRestoreExec {
pub struct TakeQuery {
parent: Arc<dyn BaseTable>,
request: QueryRequest,
offsets: Option<Vec<u64>>,
preserve_order: bool,
}
impl TakeQuery {
@@ -1875,24 +1552,15 @@ impl TakeQuery {
///
/// See [`crate::Table::take_offsets`] for more details.
pub fn from_offsets(parent: Arc<dyn BaseTable>, offsets: Vec<u64>) -> Self {
let mut seen = HashSet::with_capacity(offsets.len());
let in_list: Vec<Expr> = offsets
.iter()
.copied()
.filter(|offset| seen.insert(*offset))
.map(lit)
.collect();
let in_list: Vec<Expr> = offsets.iter().map(|o| lit(*o)).collect();
Self {
parent,
request: QueryRequest {
filter: Some(QueryFilter::Datafusion(
col("_rowoffset").in_list(in_list, false),
)),
take_offsets: Some(offsets.clone()),
..Default::default()
},
offsets: Some(offsets),
preserve_order: false,
}
}
@@ -1907,181 +1575,9 @@ impl TakeQuery {
filter: Some(QueryFilter::Datafusion(col(ROW_ID).in_list(in_list, false))),
..Default::default()
},
offsets: None,
preserve_order: false,
}
}
/// Preserve the requested offset order when restoring duplicate occurrences.
///
/// This is reserved for readers whose API explicitly guarantees ordering.
pub(crate) fn preserve_order(mut self) -> Self {
debug_assert!(self.offsets.is_some());
self.preserve_order = true;
self
}
async fn request_with_row_offset(
parent: &dyn BaseTable,
request: &QueryRequest,
) -> Result<(QueryRequest, String, bool)> {
const ROW_OFFSET: &str = "_rowoffset";
const INTERNAL_ROW_OFFSET: &str = "__lancedb_take_row_offset";
let mut request = request.clone();
// The physical lookup must not recursively restore occurrences. The
// wrapper above this request owns that logical operation.
request.take_offsets = None;
let (ordering_column, drop_ordering_column) = match &mut request.select {
Select::All => {
let mut columns = parent
.schema()
.await?
.fields()
.iter()
.map(|field| field.name().clone())
.collect::<Vec<_>>();
columns.push(ROW_OFFSET.to_string());
request.select = Select::Columns(columns);
(ROW_OFFSET.to_string(), true)
}
Select::Columns(columns) => {
if columns.iter().any(|column| column == ROW_OFFSET) {
(ROW_OFFSET.to_string(), false)
} else {
columns.push(ROW_OFFSET.to_string());
(ROW_OFFSET.to_string(), true)
}
}
Select::Dynamic(columns) => {
let mut ordering_column = INTERNAL_ROW_OFFSET.to_string();
while columns.iter().any(|(name, _)| name == &ordering_column) {
ordering_column.push('_');
}
columns.push((ordering_column.clone(), ROW_OFFSET.to_string()));
(ordering_column, true)
}
Select::Expr(columns) => {
let mut ordering_column = INTERNAL_ROW_OFFSET.to_string();
while columns.iter().any(|(name, _)| name == &ordering_column) {
ordering_column.push('_');
}
columns.push((ordering_column.clone(), col(ROW_OFFSET)));
(ordering_column, true)
}
};
Ok((request, ordering_column, drop_ordering_column))
}
async fn prepare_offsets_lookup(
parent: &dyn BaseTable,
request: &QueryRequest,
) -> Result<(QueryRequest, String, bool, usize, Option<usize>)> {
let (mut request, ordering_column, drop_ordering_column) =
Self::request_with_row_offset(parent, request).await?;
// The lookup operates on distinct physical rows. Pagination is a logical
// operation over occurrences and must be applied only after restoration.
let output_offset = request.offset.take().unwrap_or_default();
let output_limit = request.limit.take();
Ok((
request,
ordering_column,
drop_ordering_column,
output_offset,
output_limit,
))
}
fn wrap_offsets_plan(
lookup: Arc<dyn ExecutionPlan>,
offsets: &[u64],
ordering_column: String,
drop_ordering_column: bool,
output_offset: usize,
output_limit: Option<usize>,
preserve_order: bool,
) -> Result<Arc<dyn ExecutionPlan>> {
let lookup = if preserve_order {
Arc::new(CoalescePartitionsExec::new(lookup)) as Arc<dyn ExecutionPlan>
} else {
lookup
};
let restored: Arc<dyn ExecutionPlan> = Arc::new(TakeRestoreExec::try_new(
lookup,
offsets.to_vec(),
ordering_column,
drop_ordering_column,
preserve_order,
)?);
if output_offset > 0 || output_limit.is_some() {
Ok(Arc::new(GlobalLimitExec::new(
restored,
output_offset,
output_limit,
)))
} else {
Ok(restored)
}
}
fn wrap_offsets_explanation(
lookup: &str,
occurrence_count: usize,
output_offset: usize,
output_limit: Option<usize>,
preserve_order: bool,
) -> String {
fn indent(plan: &str, spaces: usize) -> String {
let indentation = " ".repeat(spaces);
plan.lines()
.map(|line| format!("{indentation}{line}"))
.collect::<Vec<_>>()
.join("\n")
}
let restored = if preserve_order {
format!(
"TakeRestoreExec: occurrences={occurrence_count}\n CoalescePartitionsExec\n{}",
indent(lookup, 4)
)
} else {
format!(
"TakeRestoreExec: occurrences={occurrence_count}\n{}",
indent(lookup, 2)
)
};
if output_offset > 0 || output_limit.is_some() {
let fetch = output_limit
.map(|limit| limit.to_string())
.unwrap_or_else(|| "None".to_string());
format!(
"GlobalLimitExec: skip={output_offset}, fetch={fetch}\n{}",
indent(&restored, 2)
)
} else {
restored
}
}
async fn create_offsets_plan(
&self,
offsets: &[u64],
options: QueryExecutionOptions,
) -> Result<Arc<dyn ExecutionPlan>> {
create_take_offsets_plan(
self.parent.as_ref(),
&self.request,
offsets,
options,
self.preserve_order,
)
.await
}
/// Convert the `TakeQuery` into a `QueryRequest`.
pub fn into_request(self) -> QueryRequest {
self.request
@@ -2126,63 +1622,6 @@ impl TakeQuery {
}
}
pub(crate) async fn create_take_offsets_plan(
parent: &dyn BaseTable,
request: &QueryRequest,
offsets: &[u64],
options: QueryExecutionOptions,
preserve_order: bool,
) -> Result<Arc<dyn ExecutionPlan>> {
let (request, ordering_column, drop_ordering_column, output_offset, output_limit) =
TakeQuery::prepare_offsets_lookup(parent, request).await?;
let lookup_options = if preserve_order {
options.without_output_batch_length_limit()
} else {
options
};
let lookup = parent
.create_plan(&AnyQuery::Query(request), lookup_options)
.await?;
TakeQuery::wrap_offsets_plan(
lookup,
offsets,
ordering_column,
drop_ordering_column,
output_offset,
output_limit,
preserve_order,
)
}
pub(crate) async fn explain_take_offsets_plan(
parent: &dyn BaseTable,
request: &QueryRequest,
offsets: &[u64],
verbose: bool,
) -> Result<String> {
let (request, _, _, output_offset, output_limit) =
TakeQuery::prepare_offsets_lookup(parent, request).await?;
let lookup = parent
.explain_plan(&AnyQuery::Query(request), verbose)
.await?;
Ok(TakeQuery::wrap_offsets_explanation(
&lookup,
offsets.len(),
output_offset,
output_limit,
false,
))
}
pub(crate) async fn prepare_take_offsets_request(
parent: &dyn BaseTable,
request: &QueryRequest,
) -> Result<QueryRequest> {
let (request, _, _, _, _) = TakeQuery::prepare_offsets_lookup(parent, request).await?;
Ok(request)
}
impl HasQuery for TakeQuery {
fn mut_query(&mut self) -> &mut QueryRequest {
&mut self.request
@@ -2191,10 +1630,6 @@ impl HasQuery for TakeQuery {
impl ExecutableQuery for TakeQuery {
async fn create_plan(&self, options: QueryExecutionOptions) -> Result<Arc<dyn ExecutionPlan>> {
if let Some(offsets) = &self.offsets {
return self.create_offsets_plan(offsets, options).await;
}
let req = AnyQuery::Query(self.request.clone());
self.parent.clone().create_plan(&req, options).await
}
@@ -2203,18 +1638,6 @@ impl ExecutableQuery for TakeQuery {
&self,
options: QueryExecutionOptions,
) -> Result<SendableRecordBatchStream> {
if self.offsets.is_some() {
let plan = self.create_plan(options.clone()).await?;
let inner = execute_plan(plan, Default::default())?;
let inner = MaxBatchLengthStream::new_boxed(inner, options.max_batch_length as usize);
let inner = if let Some(timeout) = options.timeout {
TimeoutStream::new_boxed(inner, timeout)
} else {
inner
};
return Ok(DatasetRecordBatchStream::new(inner).into());
}
let query = AnyQuery::Query(self.request.clone());
Ok(SendableRecordBatchStream::from(
self.parent.clone().query(&query, options).await?,
@@ -2222,51 +1645,11 @@ impl ExecutableQuery for TakeQuery {
}
async fn explain_plan(&self, verbose: bool) -> Result<String> {
if let Some(offsets) = &self.offsets {
let (request, _, _, output_offset, output_limit) =
Self::prepare_offsets_lookup(self.parent.as_ref(), &self.request).await?;
// Ask the backend to explain only the distinct-row lookup. This keeps
// remote explanation non-executing while still showing the client-side
// operators that create_plan and execution place above that lookup.
let lookup = self
.parent
.explain_plan(&AnyQuery::Query(request), verbose)
.await?;
return Ok(Self::wrap_offsets_explanation(
&lookup,
offsets.len(),
output_offset,
output_limit,
self.preserve_order,
));
}
let query = AnyQuery::Query(self.request.clone());
self.parent.explain_plan(&query, verbose).await
}
async fn analyze_plan_with_options(&self, options: QueryExecutionOptions) -> Result<String> {
if self.offsets.is_some() {
if self.parent.analyze_plan_is_remote() {
let (request, _, _, _, _) =
Self::prepare_offsets_lookup(self.parent.as_ref(), &self.request).await?;
// Remote analysis is owned by the service. The current wire
// request represents only the distinct-row lookup, so return
// the service report unchanged instead of fabricating metrics
// for client-side restoration operators.
return self
.parent
.analyze_plan(&AnyQuery::Query(request), options)
.await;
}
let plan = self.create_plan(options).await?;
execute_plan(plan.clone(), Default::default())?
.try_collect::<Vec<_>>()
.await?;
return Ok(format_analyzed_plan(plan));
}
let query = AnyQuery::Query(self.request.clone());
self.parent.analyze_plan(&query, options).await
}
@@ -2287,7 +1670,6 @@ mod tests {
StringArray, cast::AsArray, types::Float32Type,
};
use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
use datafusion_physical_plan::display::DisplayableExecutionPlan;
use futures::{StreamExt, TryStreamExt};
use lance_testing::datagen::{BatchGenerator, IncrementingInt32, RandomVector};
use rand::seq::IndexedRandom;
@@ -3542,218 +2924,6 @@ mod tests {
assert_eq!(results[0].num_columns(), 1);
}
#[tokio::test]
async fn test_take_offsets_preserves_duplicate_multiplicity() {
let tmp_dir = tempdir().unwrap();
let table = make_test_table(&tmp_dir).await;
let results = table
.take_offsets(vec![5, 1, 5, 17])
.select(Select::Columns(vec!["id".to_string()]))
.execute_with_options(QueryExecutionOptions {
max_batch_length: 2,
..Default::default()
})
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
assert_eq!(results.len(), 2);
assert!(results.iter().all(|batch| batch.num_columns() == 1));
let mut ids = results
.iter()
.flat_map(|batch| {
batch
.column_by_name("id")
.unwrap()
.as_primitive::<Int32Type>()
.values()
.to_vec()
})
.collect::<Vec<_>>();
ids.sort_unstable();
assert_eq!(ids, vec![1, 5, 5, 17]);
}
#[tokio::test]
async fn test_take_offsets_plan_is_incremental() {
let tmp_dir = tempdir().unwrap();
let table = make_test_table(&tmp_dir).await;
let plan = table
.take_offsets(vec![5, 1, 17])
.create_plan(QueryExecutionOptions {
max_batch_length: 1,
..Default::default()
})
.await
.unwrap();
assert_eq!(plan.properties().emission_type, EmissionType::Incremental);
let displayed = DisplayableExecutionPlan::new(plan.as_ref())
.indent(false)
.to_string();
assert!(displayed.contains("TakeRestoreExec"));
assert!(!displayed.contains("CoalescePartitionsExec"));
}
#[tokio::test]
async fn test_take_into_request_preserves_duplicate_multiplicity() {
let tmp_dir = tempdir().unwrap();
let table = make_test_table(&tmp_dir).await;
let request = table.take_offsets(vec![5, 5]).into_request();
assert_eq!(request.take_offsets, Some(vec![5, 5]));
let batches = table
.base_table()
.query(&AnyQuery::Query(request), QueryExecutionOptions::default())
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 2);
}
#[test]
fn test_restore_take_batch_only_reorders_when_requested() {
let batch = RecordBatch::try_from_iter([
(
"id",
Arc::new(Int32Array::from(vec![17, 5, 1])) as Arc<dyn Array>,
),
(
"_rowoffset",
Arc::new(UInt64Array::from(vec![17, 5, 1])) as Arc<dyn Array>,
),
])
.unwrap();
let restored =
restore_take_batch(batch.clone(), &[5, 1, 5, 17], "_rowoffset", true, false).unwrap();
assert_eq!(
restored
.column_by_name("id")
.unwrap()
.as_primitive::<Int32Type>()
.values(),
&[17, 5, 5, 1]
);
let ordered = restore_take_batch(batch, &[5, 1, 5, 17], "_rowoffset", true, true).unwrap();
assert_eq!(
ordered
.column_by_name("id")
.unwrap()
.as_primitive::<Int32Type>()
.values(),
&[5, 1, 5, 17]
);
}
#[tokio::test]
async fn test_take_offsets_applies_pagination_after_restoration() {
let tmp_dir = tempdir().unwrap();
let table = make_test_table(&tmp_dir).await;
let limited = table
.take_offsets(vec![0, 1, 0, 2])
.select(Select::Columns(vec!["id".to_string()]))
.limit(3)
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
let limited = concat_batches(&limited[0].schema(), &limited).unwrap();
assert_eq!(limited.num_rows(), 3);
assert!(
limited
.column_by_name("id")
.unwrap()
.as_primitive::<Int32Type>()
.values()
.iter()
.all(|id| [0, 1, 2].contains(id))
);
let offset = table
.take_offsets(vec![5, 1, 5, 17])
.select(Select::Columns(vec!["id".to_string()]))
.offset(1)
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
let offset = concat_batches(&offset[0].schema(), &offset).unwrap();
assert_eq!(offset.num_rows(), 3);
assert!(
offset
.column_by_name("id")
.unwrap()
.as_primitive::<Int32Type>()
.values()
.iter()
.all(|id| [1, 5, 17].contains(id))
);
}
#[tokio::test]
async fn test_take_offsets_create_plan_restores_occurrences() {
let tmp_dir = tempdir().unwrap();
let table = make_test_table(&tmp_dir).await;
let take = table
.take_offsets(vec![5, 1, 5, 17])
.select(Select::Columns(vec!["id".to_string()]));
let plan = take
.create_plan(QueryExecutionOptions::default())
.await
.unwrap();
assert_eq!(plan.schema().fields().len(), 1);
assert_eq!(plan.schema().field(0).name(), "id");
let planned = execute_plan(plan, Default::default())
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
let planned = concat_batches(&planned[0].schema(), &planned).unwrap();
let mut ids = planned
.column_by_name("id")
.unwrap()
.as_primitive::<Int32Type>()
.values()
.to_vec();
ids.sort_unstable();
assert_eq!(ids, vec![1, 5, 5, 17]);
}
#[tokio::test]
async fn test_take_offsets_plan_introspection_shows_restoration() {
let tmp_dir = tempdir().unwrap();
let table = make_test_table(&tmp_dir).await;
let take = table
.take_offsets(vec![0, 1, 0, 2])
.select(Select::Columns(vec!["id".to_string()]))
.limit(3);
let explained = take.explain_plan(false).await.unwrap();
assert!(explained.contains("GlobalLimitExec"));
assert!(explained.contains("TakeRestoreExec"));
assert!(!explained.contains("CoalescePartitionsExec"));
let analyzed = take.analyze_plan().await.unwrap();
assert!(analyzed.contains("GlobalLimitExec"));
assert!(analyzed.contains("TakeRestoreExec"));
assert!(!analyzed.contains("CoalescePartitionsExec"));
}
#[tokio::test]
async fn test_take_row_ids() {
let tmp_dir = tempdir().unwrap();
+1 -202
View File
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::collections::{HashMap, HashSet};
use std::collections::HashMap;
use std::sync::Arc;
use async_trait::async_trait;
@@ -533,24 +533,6 @@ struct RemoteListJobsResponse {
page_token: Option<String>,
}
#[derive(serde::Deserialize)]
struct RemoteListedFunctionVersion {
definition: FunctionVersion,
}
#[derive(serde::Deserialize)]
struct RemoteListFunctionsResponse {
#[serde(default)]
functions: Vec<RemoteListedFunctionVersion>,
#[serde(default)]
page_token: Option<String>,
}
#[derive(serde::Deserialize)]
struct RemoteDropFunctionResponse {
dropped: bool,
}
/// Bound on `list_jobs` page walking; a warning is logged when the listing
/// is truncated at this many pages.
const MAX_LIST_JOBS_PAGES: usize = 100;
@@ -601,57 +583,6 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
response.json().await.err_to_http(request_id)
}
async fn list_functions(&self) -> Result<Vec<FunctionVersion>> {
let mut functions = Vec::new();
let mut page_token: Option<String> = None;
let mut seen_page_tokens = HashSet::new();
loop {
let mut body = serde_json::json!({ "include_definition": true });
if let Some(token) = &page_token {
body["page_token"] = serde_json::Value::String(token.clone());
}
let req = self.client.post("/v1/functions/list").json(&body);
let (request_id, response) = self.client.send(req).await?;
let response = self.client.check_response(&request_id, response).await?;
let status = response.status();
let response: RemoteListFunctionsResponse =
response.json().await.err_to_http(request_id.clone())?;
functions.extend(
response
.functions
.into_iter()
.map(|listed| listed.definition),
);
let Some(next_page_token) = response.page_token.filter(|token| !token.is_empty())
else {
break;
};
if !seen_page_tokens.insert(next_page_token.clone()) {
return Err(Error::Http {
source: "Function listing response repeated a page_token".into(),
request_id,
status_code: Some(status),
});
}
page_token = Some(next_page_token);
}
Ok(functions)
}
async fn drop_function(&self, name: &str, version: &str) -> Result<bool> {
let req = self
.client
.post("/v1/functions/drop")
.json(&serde_json::json!({
"name": name,
"version": version,
}));
let (request_id, response) = self.client.send(req).await?;
let response = self.client.check_response(&request_id, response).await?;
let response: RemoteDropFunctionResponse = response.json().await.err_to_http(request_id)?;
Ok(response.dropped)
}
fn job(&self, job_id: &str) -> Result<crate::job::Job> {
Ok(crate::job::Job::new(Box::new(super::job::RemoteJob::new(
self.client.clone(),
@@ -2758,138 +2689,6 @@ mod tests {
assert_eq!(version.version(), "fv_01K3EXACT");
}
#[tokio::test]
async fn test_list_functions_requests_definitions_and_paginates() {
const VERSION: &str = include_str!(
"../../tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json"
);
let version: serde_json::Value = serde_json::from_str(VERSION).unwrap();
let page = Arc::new(AtomicUsize::new(0));
let conn = Connection::new_with_handler(move |request| {
assert_eq!(request.method(), &reqwest::Method::POST);
assert_eq!(request.url().path(), "/v1/functions/list");
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
assert_eq!(body["include_definition"], true);
match page.fetch_add(1, Ordering::SeqCst) {
0 => {
assert!(body.get("page_token").is_none());
http::Response::builder()
.status(200)
.body(r#"{"functions": [], "page_token": "next"}"#.to_string())
.unwrap()
}
_ => {
assert_eq!(body["page_token"], "next");
http::Response::builder()
.status(200)
.body(
serde_json::json!({
"functions": [{
"name": "embed",
"version": "fv_01K3EXACT",
"definition": version.clone(),
}],
})
.to_string(),
)
.unwrap()
}
}
});
let functions = conn.list_functions().await.unwrap();
assert_eq!(functions.len(), 1);
assert_eq!(functions[0].name(), "embed");
assert_eq!(functions[0].version(), "fv_01K3EXACT");
}
#[tokio::test]
async fn test_list_functions_stops_on_an_empty_page_token() {
let requests = Arc::new(AtomicUsize::new(0));
let seen = requests.clone();
let conn = Connection::new_with_handler(move |request| {
seen.fetch_add(1, Ordering::SeqCst);
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
assert!(body.get("page_token").is_none());
http::Response::builder()
.status(200)
.body(r#"{"functions": [], "page_token": ""}"#)
.unwrap()
});
let functions = conn.list_functions().await.unwrap();
assert!(functions.is_empty());
assert_eq!(requests.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_list_functions_rejects_a_page_token_cycle() {
let page = Arc::new(AtomicUsize::new(0));
let requests = page.clone();
let conn = Connection::new_with_handler(move |request| {
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
let next_page_token = match page.fetch_add(1, Ordering::SeqCst) {
0 => {
assert!(body.get("page_token").is_none());
"one"
}
1 => {
assert_eq!(body["page_token"], "one");
"two"
}
2 => {
assert_eq!(body["page_token"], "two");
"one"
}
page => panic!("unexpected page: {page}"),
};
http::Response::builder()
.status(200)
.body(
serde_json::json!({
"functions": [],
"page_token": next_page_token,
})
.to_string(),
)
.unwrap()
});
let error = conn.list_functions().await.unwrap_err();
assert!(
matches!(
&error,
Error::Http {
status_code: Some(http::StatusCode::OK),
..
}
),
"got {error:?}"
);
assert_eq!(requests.load(Ordering::SeqCst), 3);
}
#[tokio::test]
async fn test_drop_function_sends_exact_version_and_decodes_replay() {
let conn = Connection::new_with_handler(|request| {
assert_eq!(request.method(), &reqwest::Method::POST);
assert_eq!(request.url().path(), "/v1/functions/drop");
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
assert_eq!(
body,
serde_json::json!({"name": "embed", "version": "fv_01K3EXACT"})
);
http::Response::builder()
.status(200)
.body(r#"{"dropped":false}"#)
.unwrap()
});
assert!(!conn.drop_function("embed", "fv_01K3EXACT").await.unwrap());
}
#[tokio::test]
async fn test_conn_job_waits_to_done() {
let polls = Arc::new(AtomicUsize::new(0));
+18 -429
View File
@@ -40,8 +40,8 @@ use crate::table::{
use crate::table::{AnyQuery, Filter, Predicate, PreprocessingOutput, TableStatistics};
use crate::utils::background_cache::BackgroundCache;
use crate::utils::{
MaxBatchLengthStream, TimeoutStream, resolve_arrow_field_path, resolve_arrow_fts_field_path,
supported_btree_data_type, supported_vector_data_type,
resolve_arrow_field_path, resolve_arrow_fts_field_path, supported_btree_data_type,
supported_vector_data_type,
};
use crate::{DistanceType, Error};
use crate::{
@@ -72,7 +72,7 @@ use lance_datafusion::exec::{OneShotExec, execute_plan};
use reqwest::{RequestBuilder, Response};
use serde::{Deserialize, Serialize};
use serde_json::Number;
use std::collections::{HashMap, HashSet};
use std::collections::HashMap;
use std::io::Cursor;
use std::pin::Pin;
use std::sync::{Arc, Mutex};
@@ -524,8 +524,7 @@ impl<S: HttpSend> RemoteTable<S> {
_ => resolve_arrow_field_path(&schema, &column)?,
};
let mut body = serde_json::json!({
"column": canonical_column,
"replace": index.replace,
"column": canonical_column
});
// Add name parameter if provided (for backwards compatibility, only include if Some)
@@ -2023,9 +2022,6 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
fn as_any(&self) -> &dyn std::any::Any {
self
}
fn analyze_plan_is_remote(&self) -> bool {
true
}
fn name(&self) -> &str {
&self.name
}
@@ -2598,13 +2594,6 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
query: &AnyQuery,
options: QueryExecutionOptions,
) -> Result<Arc<dyn ExecutionPlan>> {
if let AnyQuery::Query(request) = query
&& let Some(offsets) = &request.take_offsets
{
return crate::query::create_take_offsets_plan(self, request, offsets, options, false)
.await;
}
let streams = self.execute_query(query, &options).await?;
if streams.len() == 1 {
let stream = streams.into_iter().next().unwrap();
@@ -2623,27 +2612,6 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
query: &AnyQuery,
options: QueryExecutionOptions,
) -> Result<DatasetRecordBatchStream> {
if let AnyQuery::Query(request) = query
&& let Some(offsets) = &request.take_offsets
{
let plan = crate::query::create_take_offsets_plan(
self,
request,
offsets,
options.clone(),
false,
)
.await?;
let inner = execute_plan(plan, Default::default())?;
let inner = MaxBatchLengthStream::new_boxed(inner, options.max_batch_length as usize);
let inner = if let Some(timeout) = options.timeout {
TimeoutStream::new_boxed(inner, timeout)
} else {
inner
};
return Ok(DatasetRecordBatchStream::new(inner));
}
let streams = self.execute_query(query, &options).await?;
if streams.len() == 1 {
@@ -2681,12 +2649,6 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
}
async fn explain_plan(&self, query: &AnyQuery, verbose: bool) -> Result<String> {
if let AnyQuery::Query(request) = query
&& let Some(offsets) = &request.take_offsets
{
return crate::query::explain_take_offsets_plan(self, request, offsets, verbose).await;
}
let base_request = self
.client
.post(&format!("/v1/table/{}/explain_plan/", self.identifier));
@@ -2739,17 +2701,6 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
query: &AnyQuery,
options: QueryExecutionOptions,
) -> Result<String> {
let prepared_query = if let AnyQuery::Query(request) = query
&& request.take_offsets.is_some()
{
Some(AnyQuery::Query(
crate::query::prepare_take_offsets_request(self, request).await?,
))
} else {
None
};
let query = prepared_query.as_ref().unwrap_or(query);
let mut request = self
.client
.post(&format!("/v1/table/{}/analyze_plan/", self.identifier));
@@ -3229,8 +3180,8 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
self.schema().await?.as_ref(),
"schema evolution",
)?;
// The server plans the declaration against its table schema, including
// Blob v2 semantics inherited by a direct field projection.
// The server plans the declaration: expression validation, type
// inference and the persisted binding all happen there.
let entries = columns
.iter()
.map(
@@ -3648,12 +3599,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
#[derive(Serialize, Clone, Debug)]
pub struct MergeInsertRequest {
// Sent as one repeated `on` query parameter per column, which is how the
// namespace spec encodes an array-valued `on`. serde_urlencoded (which
// reqwest's `query()` uses) cannot serialize a sequence nested in a struct,
// so this field is emitted separately by [`Self::on_query_params`].
#[serde(skip_serializing)]
on: Vec<String>,
on: String,
when_matched_update_all: bool,
when_matched_update_all_filt: Option<String>,
when_not_matched_insert_all: bool,
@@ -3669,17 +3615,6 @@ pub struct MergeInsertRequest {
use_lsm: Option<bool>,
}
impl MergeInsertRequest {
/// The `on` columns as repeated query parameters: `?on=a&on=b`.
///
/// A single column serializes to `?on=a`, exactly what clients sent before
/// `on` became a list, so a server that predates composite keys sees no
/// change from a single-column caller.
pub(crate) fn on_query_params(&self) -> Vec<(&str, &str)> {
self.on.iter().map(|col| ("on", col.as_str())).collect()
}
}
fn is_true(b: &bool) -> bool {
*b
}
@@ -3692,15 +3627,12 @@ impl TryFrom<MergeInsertBuilder> for MergeInsertRequest {
return Err(Error::InvalidInput {
message: "MergeInsertBuilder missing required 'on' field".into(),
});
}
// The server rejects a repeated column with a 400; catching it here
// names the offending column and costs no round trip.
let mut seen = HashSet::with_capacity(value.on.len());
if let Some(dup) = value.on.iter().find(|col| !seen.insert(*col)) {
return Err(Error::InvalidInput {
message: format!("MergeInsertBuilder 'on' column '{dup}' is repeated"),
} else if value.on.len() > 1 {
return Err(Error::NotSupported {
message: "MergeInsertBuilder only supports a single 'on' column".into(),
});
}
let on = value.on[0].clone();
let when_matched_update_all_filt = match value.when_matched_update_all_filt {
Some(MergeFilter::Sql(sql)) => Some(sql),
@@ -3724,7 +3656,7 @@ impl TryFrom<MergeInsertBuilder> for MergeInsertRequest {
};
Ok(Self {
on: value.on,
on,
when_matched_update_all: value.when_matched_update_all,
when_matched_update_all_filt,
when_not_matched_insert_all: value.when_not_matched_insert_all,
@@ -3758,7 +3690,7 @@ mod tests {
};
use arrow_schema::{DataType, Field, Schema};
use chrono::{DateTime, Utc};
use futures::{StreamExt, TryFutureExt, TryStreamExt, future::BoxFuture};
use futures::{StreamExt, TryFutureExt, future::BoxFuture};
use lance_index::scalar::inverted::{DocumentGranularity, query::MatchQuery};
use lance_index::scalar::{FullTextSearchQuery, InvertedIndexParams};
use reqwest::Body;
@@ -4568,76 +4500,6 @@ mod tests {
}
}
#[tokio::test]
async fn test_merge_insert_composite_key() {
let batch = RecordBatch::try_new(
Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
)
.unwrap();
let data: Box<dyn RecordBatchReader + Send> = Box::new(RecordBatchIterator::new(
[Ok(batch.clone())],
batch.schema(),
));
let table = Table::new_with_handler("my_table", move |request| {
assert_eq!(request.url().path(), "/v1/table/my_table/merge_insert/");
// One repeated `on` per column, in the order the caller gave them.
let on = request
.url()
.query_pairs()
.filter(|(key, _)| key == "on")
.map(|(_, value)| value.into_owned())
.collect::<Vec<_>>();
assert_eq!(on, vec!["shard_key".to_string(), "id".to_string()]);
let params = request.url().query_pairs().collect::<HashMap<_, _>>();
assert_eq!(params["when_matched_update_all"], "true");
assert_eq!(params["when_not_matched_insert_all"], "true");
http::Response::builder()
.status(200)
.body(r#"{"version": 43, "num_deleted_rows": 0, "num_inserted_rows": 3, "num_updated_rows": 0}"#)
.unwrap()
});
let mut merge = table.merge_insert(&["shard_key", "id"]);
merge.when_matched_update_all(None);
merge.when_not_matched_insert_all();
let result = table.base_table().merge_insert(merge, data).await.unwrap();
assert_eq!(result.num_inserted_rows, 3);
}
#[tokio::test]
async fn test_merge_insert_rejects_repeated_on_column() {
let batch = RecordBatch::try_new(
Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
vec![Arc::new(Int32Array::from(vec![1]))],
)
.unwrap();
let data: Box<dyn RecordBatchReader + Send> = Box::new(RecordBatchIterator::new(
[Ok(batch.clone())],
batch.schema(),
));
let table = Table::new_with_handler::<&str>("my_table", |request| {
panic!("Unexpected request: {}", request.url());
});
let merge = table.merge_insert(&["id", "id"]);
let err = table
.base_table()
.merge_insert(merge, data)
.await
.unwrap_err();
assert!(
matches!(&err, Error::InvalidInput { message } if message.contains("'id' is repeated")),
"unexpected error: {err}"
);
}
#[tokio::test]
async fn test_merge_insert_retries_on_409() {
let batch = RecordBatch::try_new(
@@ -5749,114 +5611,6 @@ mod tests {
assert_eq!(result, "analyzed plan");
}
#[tokio::test]
async fn test_take_offsets_explain_plan_does_not_execute_query() {
let table = Table::new_with_handler("my_table", |request| {
assert_eq!(request.method(), "POST");
assert_eq!(request.url().path(), "/v1/table/my_table/explain_plan/");
http::Response::builder()
.status(200)
.body(r#""RemoteLookupExec""#)
.unwrap()
});
let explained = table
.take_offsets(vec![0, 1, 0, 2])
.select(crate::query::Select::columns(&["id"]))
.limit(3)
.explain_plan(false)
.await
.unwrap();
assert!(explained.contains("GlobalLimitExec"));
assert!(explained.contains("TakeRestoreExec"));
assert!(!explained.contains("CoalescePartitionsExec"));
assert!(explained.contains("RemoteLookupExec"));
}
#[tokio::test]
async fn test_converted_take_request_restores_remote_occurrences() {
let table = Table::new_with_handler("my_table", |request| {
assert_eq!(request.method(), "POST");
assert_eq!(request.url().path(), "/v1/table/my_table/query/");
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
assert_eq!(body["columns"], json!(["id", "_rowoffset"]));
let data = RecordBatch::try_new(
Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new("_rowoffset", DataType::UInt64, false),
])),
vec![
Arc::new(Int32Array::from(vec![5])),
Arc::new(arrow_array::UInt64Array::from(vec![5])),
],
)
.unwrap();
http::Response::builder()
.status(200)
.header(CONTENT_TYPE, ARROW_FILE_CONTENT_TYPE)
.body(write_ipc_file(&data))
.unwrap()
});
let request = table
.take_offsets(vec![5, 5])
.select(crate::query::Select::columns(&["id"]))
.into_request();
let batches = table
.base_table()
.query(&AnyQuery::Query(request), QueryExecutionOptions::default())
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 2);
assert!(
batches
.iter()
.all(|batch| batch.schema().fields().len() == 1)
);
}
#[tokio::test]
async fn test_take_offsets_analyze_plan_delegates_to_remote() {
let table = Table::new_with_handler("my_table", |request| {
assert_eq!(request.method(), "POST");
assert_eq!(request.url().path(), "/v1/table/my_table/analyze_plan/");
assert_eq!(
request
.url()
.query_pairs()
.find(|(key, _)| key == "distributed_metrics"),
Some(("distributed_metrics".into(), "per_worker".into()))
);
http::Response::builder()
.status(200)
.body(r#""Remote analyzed plan: worker metrics""#)
.unwrap()
});
let analyzed = table
.take_offsets(vec![0, 1, 0, 2])
.select(crate::query::Select::columns(&["id"]))
.limit(3)
.analyze_plan_with_options(QueryExecutionOptions {
analyze_plan_distributed_metrics: AnalyzePlanDistributedMetrics::PerWorker,
..Default::default()
})
.await
.unwrap();
assert_eq!(analyzed, "Remote analyzed plan: worker metrics");
}
#[tokio::test]
async fn test_query_structured_fts() {
let table =
@@ -5980,8 +5734,9 @@ mod tests {
))
.execute()
.await;
let Err(err) = result else {
panic!("legacy remote query unexpectedly succeeded")
let err = match result {
Ok(_) => panic!("legacy remote query unexpectedly succeeded"),
Err(err) => err,
};
assert!(
@@ -6306,7 +6061,6 @@ mod tests {
let mut expected_body = expected_body.clone();
expected_body["column"] = "a".into();
expected_body[INDEX_TYPE_KEY] = index_type.into();
expected_body["replace"] = false.into();
assert_eq!(body, expected_body);
@@ -6324,73 +6078,6 @@ mod tests {
}
}
#[tokio::test]
async fn test_create_index_forwards_default_replace_false_on_existing_route() {
let table = Table::new_with_handler("my_table", move |request| {
assert_eq!(request.method(), "POST");
match request.url().path() {
"/v1/table/my_table/describe/" => {
let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
http::Response::builder()
.status(200)
.body(describe_response(&schema))
.unwrap()
}
"/v1/table/my_table/create_index/" => {
let body = request.body().unwrap().as_bytes().unwrap();
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
assert_eq!(body["replace"], json!(false));
http::Response::builder()
.status(200)
.body("{}".to_string())
.unwrap()
}
path => panic!("Unexpected path: {}", path),
}
});
table
.create_index(&["a"], Index::BTree(Default::default()))
.execute()
.await
.unwrap();
}
#[tokio::test]
async fn test_create_index_forwards_explicit_replace_true_on_existing_route() {
let table = Table::new_with_handler("my_table", move |request| {
assert_eq!(request.method(), "POST");
match request.url().path() {
"/v1/table/my_table/describe/" => {
let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
http::Response::builder()
.status(200)
.body(describe_response(&schema))
.unwrap()
}
"/v1/table/my_table/create_index/" => {
let body = request.body().unwrap().as_bytes().unwrap();
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
assert_eq!(body["replace"], json!(true));
http::Response::builder()
.status(200)
.body("{}".to_string())
.unwrap()
}
path => panic!("Unexpected path: {}", path),
}
});
table
.create_index(&["a"], Index::BTree(Default::default()))
.replace(true)
.execute()
.await
.unwrap();
}
#[tokio::test]
async fn test_create_index_returns_job() {
let describe_calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
@@ -6616,46 +6303,38 @@ mod tests {
json!({
"column": "rowId",
"index_type": "BTREE",
"replace": false,
}),
json!({
"column": "`row-id`",
"index_type": "BTREE",
"replace": false,
}),
json!({
"column": "userId",
"index_type": "BTREE",
"replace": false,
}),
json!({
"column": "MetaData.userId",
"index_type": "BTREE",
"replace": false,
}),
json!({
"column": "metadata.user_id",
"index_type": "BTREE",
"replace": false,
}),
json!({
"column": "image.embedding",
"index_type": "IVF_PQ",
"metric_type": "l2",
"replace": false,
}),
{
let mut body = serde_json::to_value(InvertedIndexParams::default()).unwrap();
body["column"] = "payload.text".into();
body["index_type"] = "FTS".into();
body["replace"] = false.into();
body
},
{
let mut body = serde_json::to_value(InvertedIndexParams::default()).unwrap();
body["column"] = "docs.content".into();
body["index_type"] = "FTS".into();
body["replace"] = false.into();
body
},
{
@@ -6663,18 +6342,15 @@ mod tests {
body["column"] = "docs.content".into();
body["index_type"] = "FTS".into();
body["document_granularity"] = "list_element".into();
body["replace"] = false.into();
body
},
json!({
"column": "`meta-data`.`user-id`",
"index_type": "BTREE",
"replace": false,
}),
json!({
"column": "literal.`a.b`",
"index_type": "BTREE",
"replace": false,
}),
]);
let request_idx = Arc::new(AtomicUsize::new(0));
@@ -7712,8 +7388,8 @@ mod tests {
assert_eq!(result.version, if old_server { 0 } else { 43 });
}
/// A declaration is sent as `{name, computed}` for the server to plan; the
/// client never types the expression itself.
/// A declaration is sent as `{name, computed}` entries for the server to
/// plan; the client never types the expression itself.
#[tokio::test]
async fn test_add_computed_columns_sends_the_expression() {
let table = Table::new_with_handler("my_table", |request| match request.url().path() {
@@ -7789,93 +7465,6 @@ mod tests {
assert_eq!(result.version, 8);
}
#[tokio::test]
async fn test_add_function_column_allows_an_existing_binding() {
let binding = crate::function::FunctionBinding::from_json(include_str!(
"../../tests/fixtures/first_class_functions/v1/remote_function_binding.json"
))
.unwrap();
let binding_metadata = crate::table::computed_columns::function_bindings_metadata(
std::slice::from_ref(&binding),
)
.unwrap();
let mut fields = vec![
Field::new("title", DataType::Utf8, true),
Field::new("body", DataType::Utf8, true),
];
fields.extend(binding.outputs().iter().map(|output| {
let data_type = match output.arrow_type.as_str() {
"utf8" => DataType::Utf8,
"int64" => DataType::Int64,
other => panic!("unexpected fixture output type {other}"),
};
Field::new(&output.output_name, data_type, true).with_metadata(
crate::table::computed_columns::function_computed_column_metadata(
binding.binding_id(),
output.output_ordinal,
&["title".into(), "body".into()],
),
)
}));
let schema = Schema::new_with_metadata(
fields,
HashMap::from([(
crate::table::computed_columns::FUNCTION_BINDINGS_META_KEY.to_string(),
binding_metadata,
)]),
);
let table =
Table::new_with_handler("my_table", move |request| match request.url().path() {
"/v1/table/my_table/describe/" => http::Response::builder()
.status(200)
.body(describe_response(&schema))
.unwrap(),
"/v1/table/my_table/add_columns/" => {
let actual: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap())
.unwrap();
assert_eq!(
actual["new_columns"],
serde_json::json!([
{"name":"secondary_text","all_null":true},
{"name":"secondary_token_count","all_null":true}
])
);
http::Response::builder()
.status(200)
.body(r#"{"version":10}"#.to_string())
.unwrap()
}
path => panic!("Unexpected path: {path}"),
});
let application = crate::function::FunctionApplication::from_json(
r#"{
"function":{"name":"text_features","version":"fv_01K3TEXT"},
"inputs":[
{"parameter":"title","kind":"column","value":{"path":"title"}},
{"parameter":"body","kind":"column","value":{"path":"body"}}
],
"output":{"kind":"named_struct","fields":[
{"name":"normalized_text","arrow_type":"utf8","nullable":false},
{"name":"token_count","arrow_type":"int64","nullable":false}
]},
"columns":{
"normalized_text":"secondary_text",
"token_count":"secondary_token_count"
}
}"#,
)
.unwrap();
let result = table
.add_columns()
.function(application)
.execute()
.await
.unwrap();
assert_eq!(result.version, 10);
}
#[tokio::test]
async fn test_add_fixed_size_list_function_column_declares_the_vector_type() {
let table = Table::new_with_handler("my_table", |request| {
+1 -2
View File
@@ -734,7 +734,6 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
WriteOp::MergeInsert { query, timeout } => {
let mut request = client
.post(&format!("/v1/table/{}/merge_insert/", identifier))
.query(&query.on_query_params())
.query(query)
.header(CONTENT_TYPE, ARROW_STREAM_CONTENT_TYPE);
if let Some(timeout) = timeout {
@@ -1490,7 +1489,7 @@ mod tests {
});
let query = MergeInsertRequest {
on: vec!["id".to_string()],
on: "id".to_string(),
when_matched_update_all: false,
when_matched_update_all_filt: None,
when_not_matched_insert_all: false,
+6 -23
View File
@@ -595,14 +595,6 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
query: &AnyQuery,
options: QueryExecutionOptions,
) -> Result<String>;
/// Whether [`BaseTable::analyze_plan`] is provided by a remote service.
///
/// Client-side query wrappers use this to preserve backend metrics and
/// distributed-analysis options instead of replacing them with a local plan.
#[doc(hidden)]
fn analyze_plan_is_remote(&self) -> bool {
false
}
/// Add new records to the table.
async fn add(&self, add: AddDataBuilder) -> Result<AddResult>;
@@ -758,8 +750,8 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
/// Declare computed columns, each defined by a SQL expression.
///
/// Where the declaration is planned depends on the backend: a local table
/// validates and types the expression itself, while a remote one sends the
/// expression for the server to plan.
/// validates and types the expression itself, a remote one sends the text
/// for the server to plan.
async fn add_computed_columns(
&self,
_columns: &[(String, String)],
@@ -1506,9 +1498,7 @@ impl Table {
///
/// * `on` One or more columns to join on. This is how records from the
/// source table and target table are matched. Typically this is some
/// kind of key or id column. Several columns match on the composite
/// key: a source row updates a target row only when it agrees on every
/// one of them.
/// kind of key or id column.
///
/// # Examples
///
@@ -1662,9 +1652,9 @@ impl Table {
/// Offsets are useful for sampling as the set of all valid offsets is easily
/// known in advance to be [0, len(table)).
///
/// No guarantees are made regarding the order in which results are returned.
/// Repeated offsets produce repeated rows, which makes this method suitable for
/// sampling with replacement.
/// No guarantees are made regarding the order in which results are returned. If you
/// desire an output order that matches the order of the given offsets, you will need
/// to add the row offset column to the output and align it yourself.
///
/// Parameters
/// ----------
@@ -5773,13 +5763,6 @@ mod tests {
assert!(index_bytes > 0);
assert_eq!(with_index, data_only + index_bytes);
// Release builds reject unstable overlay datasets unless explicitly opted in.
if !lance_table::feature_flags::can_read_dataset(
lance_table::feature_flags::FLAG_UNSTABLE_DATA_OVERLAY_FILES,
) {
return;
}
// Commit an overlay file supplying new `foo` values for the first three
// rows of fragment 0. There is no high-level API that writes overlays
// yet, so write the overlay's data file and commit the `DataOverlay`
File diff suppressed because it is too large Load Diff
+5 -80
View File
@@ -133,7 +133,7 @@ impl NativeTable {
),
});
}
(resolved.canonical_path, resolved.terminal_field)
(resolved.canonical_path, resolved.field)
} else {
Self::resolve_index_field(dataset.schema(), &opts.columns[0])?
};
@@ -439,8 +439,7 @@ mod tests {
use arrow_array::record_batch;
use arrow_array::{
Array, ArrayRef, BinaryArray, BooleanArray, FixedSizeListArray, Float32Array, Int32Array,
LargeBinaryArray, LargeStringArray, ListArray, RecordBatch, StringArray, StructArray,
UInt32Array,
LargeBinaryArray, LargeStringArray, RecordBatch, StringArray, StructArray,
};
use arrow_data::ArrayDataBuilder;
use arrow_schema::{DataType, Field, Schema};
@@ -459,7 +458,6 @@ mod tests {
use crate::query::{ExecutableQuery, QueryBase};
use crate::table::optimize::{CompactionOptions, OptimizeAction};
use lance_index::scalar::FullTextSearchQuery;
use lance_index::scalar::inverted::query::{FtsQuery, MatchQuery};
fn create_fixed_size_list<T: Array>(
values: T,
@@ -601,80 +599,6 @@ mod tests {
assert!(invalid_granularity.is_err());
}
#[tokio::test]
async fn test_nested_list_fts_uses_deepest_document_coordinates() {
let conn = connect("memory://").execute().await.unwrap();
let mut docs = ListBuilder::new(ListBuilder::new(StringBuilder::new()));
docs.values().values().append_value("alpha");
docs.values().values().append_value("beta");
docs.values().append(true);
docs.values().values().append_value("gamma");
docs.values().values().append_value("alpha delta");
docs.values().append(true);
docs.append(true);
docs.values().append(true);
docs.values().values().append_value("alpha");
docs.values().append(true);
docs.append(true);
let batch = RecordBatch::try_from_iter(vec![
("id", Arc::new(Int32Array::from(vec![0, 1])) as ArrayRef),
("docs", Arc::new(docs.finish()) as ArrayRef),
])
.unwrap();
let table = conn.create_table("nested", batch).execute().await.unwrap();
let job = table
.create_index(
&["docs"],
Index::FTS(
FtsIndexBuilder::default()
.document_granularity(DocumentGranularity::ListElement),
),
)
.execute_async()
.await
.unwrap();
job.wait().await.unwrap();
let query = FullTextSearchQuery::new_query(FtsQuery::Match(
MatchQuery::new("alpha".to_string())
.with_column(Some("docs".to_string()))
.with_document_granularity(DocumentGranularity::ListElement),
));
let batches = table
.query()
.full_text_search(query)
.limit(10)
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
let mut hits = Vec::new();
for batch in batches {
let ids = batch["id"].as_any().downcast_ref::<Int32Array>().unwrap();
let coordinates = batch["_doc_index"]
.as_any()
.downcast_ref::<ListArray>()
.unwrap();
for row in 0..batch.num_rows() {
let coordinate = coordinates.value(row);
let coordinate = coordinate.as_any().downcast_ref::<UInt32Array>().unwrap();
hits.push((ids.value(row), coordinate.values().to_vec()));
}
}
hits.sort_unstable();
assert_eq!(
hits,
vec![(0, vec![0, 0]), (0, vec![1, 1]), (1, vec![1, 0])]
);
}
/// Concurrent waiters, and a wait issued after the job settled, all
/// succeed once the build does.
#[tokio::test]
@@ -725,11 +649,12 @@ mod tests {
.await
.unwrap();
// Rebuilding the same index without explicit replace fails once the build
// Rebuilding the same index without replace fails once the build
// starts, so the failure reaches the job rather than execute_async.
let job = Arc::new(
table
.create_index(&["id"], Index::BTree(BTreeIndexBuilder::default()))
.replace(false)
.execute_async()
.await
.unwrap(),
@@ -768,6 +693,7 @@ mod tests {
let job = table
.create_index(&["id"], Index::BTree(BTreeIndexBuilder::default()))
.replace(false)
.execute_async()
.await
.unwrap();
@@ -1104,7 +1030,6 @@ mod tests {
// Can also specify btree
table
.create_index(&["i"], Index::BTree(BTreeIndexBuilder::default()))
.replace(true)
.execute()
.await
.unwrap();
+1 -8
View File
@@ -110,7 +110,7 @@ fn requires_local_namespace_execution(query: &AnyQuery) -> bool {
// pushing these down would silently ignore the user's setting. For use_lsm that
// is worse than a tuning miss: MemWAL read routing lives only in `create_plan`,
// so a pushed-down query would return stale base-only data with no error.
if query.base().use_lsm.is_some() || query.base().take_offsets.is_some() {
if query.base().use_lsm.is_some() {
return true;
}
matches!(
@@ -154,13 +154,6 @@ pub async fn create_plan(
options: QueryExecutionOptions,
) -> Result<Arc<dyn ExecutionPlan>> {
let query = query.canonicalized()?;
if let AnyQuery::Query(request) = &query
&& let Some(offsets) = &request.take_offsets
{
return crate::query::create_take_offsets_plan(table, request, offsets, options, false)
.await;
}
let query = match query {
AnyQuery::VectorQuery(query) => query,
AnyQuery::Query(query) => VectorQueryRequest::from_plain_query(query),
+7 -515
View File
@@ -29,14 +29,10 @@
//! inputs masked to null first, so a poison value in a row nobody is filling
//! cannot fail the refresh.
use std::collections::HashSet;
use std::sync::Arc;
use arrow_array::{
Array, ArrayRef, BooleanArray, LargeBinaryArray, RecordBatch, RecordBatchOptions, StructArray,
new_null_array,
};
use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
use arrow_array::{ArrayRef, BooleanArray, RecordBatch, RecordBatchOptions};
use arrow_schema::Schema as ArrowSchema;
use datafusion_expr::ColumnarValue;
use futures::{Stream, StreamExt, TryStreamExt};
use lance::Dataset;
@@ -44,7 +40,7 @@ use lance::dataset::WriteDestination;
use lance::dataset::fragment::FileFragment;
use lance::dataset::transaction::Operation;
use lance_core::ROW_ID;
use lance_core::datatypes::{BlobHandling, Schema as LanceSchema};
use lance_core::datatypes::Schema as LanceSchema;
use serde::{Deserialize, Serialize};
use super::computed_columns::{BoundExpression, ComputedColumnKind, computed_column_from_field};
@@ -108,7 +104,6 @@ async fn execute_refresh_column_with_source(
fields: vec![field.clone()],
metadata: Default::default(),
};
let output_is_blob = field.is_blob_v2();
let mut rows_filled = 0u64;
let mut replacements = Vec::new();
@@ -118,8 +113,7 @@ async fn execute_refresh_column_with_source(
continue;
}
rows_filled += gained;
let values =
fill_stream(&dataset, &fragment, bound.clone(), column, output_is_blob).await?;
let values = fill_stream(&dataset, &fragment, bound.clone(), column).await?;
replacements.push(fragment.write_columns(values, &column_schema).await?);
}
@@ -300,15 +294,12 @@ fn evaluation_batch(
mask_out: Option<&BooleanArray>,
) -> lance_core::Result<RecordBatch> {
let mut columns = Vec::with_capacity(bound.roots.len());
let mut fields = Vec::with_capacity(bound.roots.len());
for name in &bound.roots {
let index = batch.schema_ref().index_of(name).map_err(|_| {
let column = batch.column_by_name(name).ok_or_else(|| {
lance_core::Error::invalid_input(format!(
"refreshing a computed column read no {name} column"
))
})?;
let column = batch.column(index);
fields.push(batch.schema_ref().field(index).clone());
// Rows outside the mask must not reach the expression: a value in a
// deleted or already-filled row can be one it would choke on.
columns.push(match mask_out {
@@ -317,7 +308,7 @@ fn evaluation_batch(
});
}
Ok(RecordBatch::try_new_with_options(
Arc::new(ArrowSchema::new(fields)),
bound.read_schema.clone(),
columns,
&RecordBatchOptions::new().with_row_count(Some(batch.num_rows())),
)?)
@@ -338,99 +329,6 @@ fn evaluate(bound: &BoundExpression, batch: &RecordBatch) -> lance_core::Result<
}
}
fn materialized_blob_ids(schema: &LanceSchema, paths: &[String]) -> Result<HashSet<u32>> {
paths
.iter()
.map(|path| {
let field = schema
.resolve(path)
.and_then(|fields| fields.last().copied())
.ok_or_else(|| Error::InvalidInput {
message: format!("computed Blob input '{path}' no longer exists"),
})?;
if !field.is_blob_v2() {
return Err(Error::InvalidInput {
message: format!("computed Blob input '{path}' is no longer Blob v2"),
});
}
u32::try_from(field.id).map_err(|_| Error::InvalidInput {
message: format!(
"computed Blob input '{path}' has invalid field id {}",
field.id
),
})
})
.collect()
}
fn configure_blob_inputs(
scanner: &mut lance::dataset::scanner::Scanner,
schema: &LanceSchema,
bound: &BoundExpression,
extra_blob_id: Option<u32>,
) -> Result<()> {
let mut ids = materialized_blob_ids(schema, &bound.blob_paths)?;
ids.extend(extra_blob_id);
scanner.blob_handling(BlobHandling::SomeBlobsBinary(ids));
Ok(())
}
fn blob_array_from_binary(
array: &ArrayRef,
target_field: &ArrowField,
) -> lance_core::Result<ArrayRef> {
let values = array
.as_any()
.downcast_ref::<LargeBinaryArray>()
.ok_or_else(|| {
lance_core::Error::invalid_input(format!(
"a Blob v2 computed output produced {}, expected LargeBinary",
array.data_type()
))
})?;
let mut builder = lance::blob::BlobArrayBuilder::new(values.len());
for index in 0..values.len() {
if values.is_null(index) {
builder.push_null()?;
} else {
builder.push_bytes(values.value(index))?;
}
}
let minimal = builder.finish()?;
let minimal = minimal
.as_any()
.downcast_ref::<StructArray>()
.ok_or_else(|| lance_core::Error::internal("Blob builder returned a non-struct array"))?;
let DataType::Struct(target_fields) = target_field.data_type() else {
return Err(lance_core::Error::invalid_input(format!(
"Blob v2 output field '{}' has non-struct type {}",
target_field.name(),
target_field.data_type()
)));
};
let columns = target_fields
.iter()
.map(|field| match field.name().as_str() {
"data" | "uri" => minimal
.column_by_name(field.name())
.cloned()
.ok_or_else(|| {
lance_core::Error::internal(format!("Blob builder omitted '{}'", field.name()))
}),
"position" | "size" => Ok(new_null_array(field.data_type(), minimal.len())),
name => Err(lance_core::Error::invalid_input(format!(
"Blob v2 output field '{}' has unsupported logical child '{name}'",
target_field.name()
))),
})
.collect::<lance_core::Result<Vec<_>>>()?;
Ok(Arc::new(StructArray::try_new(
target_fields.clone(),
columns,
minimal.nulls().cloned(),
)?))
}
/// How many rows of one fragment would gain a value.
///
/// Scans only the unfilled live rows -- deleted rows never reach the
@@ -449,7 +347,6 @@ async fn count_fragment_gains(
.with_row_id()
.filter(&format!("{} IS NULL", quote_identifier(column)))?
.project(&bound.roots)?;
configure_blob_inputs(&mut scanner, dataset.schema(), bound, None)?;
let mut gained = 0u64;
let mut batches = scanner.try_into_stream().await?;
@@ -471,7 +368,6 @@ async fn fill_stream(
fragment: &FileFragment,
bound: Arc<BoundExpression>,
column: &str,
output_is_blob: bool,
) -> Result<impl Stream<Item = lance_core::Result<RecordBatch>> + Send + use<>> {
let mut projection: Vec<String> = bound.roots.clone();
projection.push(column.to_string());
@@ -481,20 +377,6 @@ async fn fill_stream(
.with_row_id()
.include_deleted_rows()
.project(&projection)?;
let output_blob_id = output_is_blob
.then(|| {
dataset
.schema()
.field(column)
.and_then(|field| u32::try_from(field.id).ok())
})
.flatten();
configure_blob_inputs(
&mut scanner,
dataset.schema(),
bound.as_ref(),
output_blob_id,
)?;
let projected = Arc::new(ArrowSchema::new(vec![
ArrowSchema::from(dataset.schema())
@@ -530,11 +412,6 @@ async fn fill_stream(
let computed = evaluate(&bound, &evaluation_batch(&batch, &bound, Some(&keep))?)?;
let merged = arrow_select::zip::zip(&fill, &computed, existing)?;
let merged = if output_is_blob {
blob_array_from_binary(&merged, projected.field(0))?
} else {
merged
};
Ok(RecordBatch::try_new(projected.clone(), vec![merged])?)
}))
}
@@ -543,12 +420,8 @@ async fn fill_stream(
mod tests {
use std::sync::Arc;
use arrow_array::{
Array, ArrayRef, Int32Array, LargeBinaryArray, RecordBatch, StructArray, record_batch,
};
use arrow_schema::Field as ArrowField;
use arrow_array::{Int32Array, record_batch};
use futures::TryStreamExt;
use lance_core::ROW_ID;
use crate::connect;
use crate::query::{ExecutableQuery, QueryBase, Select};
@@ -604,25 +477,6 @@ mod tests {
table.add(batch).execute().await.unwrap();
}
#[test]
fn test_blob_output_matches_complete_logical_field() {
let values: ArrayRef = Arc::new(LargeBinaryArray::from(vec![
Some(b"hello".as_slice()),
None,
]));
let field = ArrowField::new(
"image",
lance_core::datatypes::BLOB_V2_LOGICAL_TYPE.clone(),
true,
);
let output = super::blob_array_from_binary(&values, &field).unwrap();
assert_eq!(output.data_type(), field.data_type());
let output = output.as_any().downcast_ref::<StructArray>().unwrap();
assert_eq!(output.column_by_name("position").unwrap().null_count(), 2);
assert_eq!(output.column_by_name("size").unwrap().null_count(), 2);
}
/// The gate's reproducer: `b = coalesce(a, 0)` refreshed before `a`
/// must not bake zeros from `a`'s placeholder null. It is refused, and
/// names the input, until `a` is filled -- after every append too.
@@ -1310,366 +1164,4 @@ mod tests {
let err = table.refresh_column("embedding").await.unwrap_err();
assert!(matches!(err, Error::NotSupported { message } if message.contains("udf")));
}
fn blob_batch(ids: Vec<i32>, payloads: Vec<Option<&[u8]>>) -> RecordBatch {
use arrow_array::Int32Array;
use arrow_schema::{Field, Schema};
let mut builder = lance::blob::BlobArrayBuilder::new(payloads.len());
for payload in payloads {
match payload {
Some(payload) => builder.push_bytes(payload).unwrap(),
None => builder.push_null().unwrap(),
}
}
RecordBatch::try_new(
Arc::new(Schema::new(vec![
Field::new("id", arrow_schema::DataType::Int32, false),
crate::blob("image", true),
])),
vec![Arc::new(Int32Array::from(ids)), builder.finish().unwrap()],
)
.unwrap()
}
async fn create_blob_table(path: &std::path::Path, batch: RecordBatch) -> Table {
let conn = connect(path.to_str().unwrap()).execute().await.unwrap();
conn.create_table("blobs", batch).execute().await.unwrap()
}
#[tokio::test]
async fn test_refresh_inherits_and_publishes_blob_output() {
use arrow_array::UInt64Array;
use lance_arrow::{
BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, BLOB_INLINE_SIZE_THRESHOLD_META_KEY,
};
use lance_core::datatypes::BlobKind;
use crate::table::schema_evolution::FieldMetadataUpdate;
let tmp = tempfile::tempdir().unwrap();
let table = create_blob_table(
tmp.path(),
blob_batch(
vec![1, 2, 3, 4],
vec![Some(b"hello"), Some(b"ab"), Some(b""), None],
),
)
.await;
table
.add_columns()
.computed("image_copy", "image")
.execute()
.await
.unwrap();
table
.update_field_metadata(&[FieldMetadataUpdate::new("image_copy")
.set(BLOB_INLINE_SIZE_THRESHOLD_META_KEY, "1")
.set(BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, "4")])
.await
.unwrap();
let first_refresh = table.refresh_column("image_copy").await.unwrap();
assert_eq!(first_refresh.rows_filled, 3);
assert_eq!(
table.blob_columns().await.unwrap(),
vec!["image".to_string(), "image_copy".to_string()]
);
let batches = table
.query()
.with_row_id()
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
let batch = arrow_select::concat::concat_batches(&batches[0].schema(), &batches).unwrap();
assert!(
batch
.column_by_name("image_copy")
.unwrap()
.as_any()
.is::<arrow_array::StructArray>()
);
let row_ids = batch
.column_by_name(ROW_ID)
.unwrap()
.as_any()
.downcast_ref::<UInt64Array>()
.unwrap()
.values()
.to_vec();
let original = table.fetch_blobs("image", &row_ids).await.unwrap();
let copied = table.fetch_blobs("image_copy", &row_ids).await.unwrap();
assert_eq!(original, copied);
let ids = batch
.column_by_name("id")
.unwrap()
.as_any()
.downcast_ref::<Int32Array>()
.unwrap();
let files = table
.fetch_blob_files("image_copy", &row_ids)
.await
.unwrap();
let mut layouts = ids
.values()
.iter()
.copied()
.zip(files)
.map(|(id, file)| (id, file.and_then(|file| file.kind())))
.collect::<Vec<_>>();
layouts.sort_by_key(|(id, _)| *id);
assert_eq!(
layouts,
vec![
(1, Some(BlobKind::Dedicated)),
(2, Some(BlobKind::Packed)),
(3, Some(BlobKind::Inline)),
(4, None),
]
);
table
.add(blob_batch(vec![5], vec![Some(b"appended")]))
.execute()
.await
.unwrap();
table
.optimize(crate::table::OptimizeAction::Compact {
options: crate::table::CompactionOptions::default(),
remap_options: None,
})
.await
.unwrap();
assert_eq!(
table
.refresh_column("image_copy")
.await
.unwrap()
.rows_filled,
1
);
assert_eq!(
table
.refresh_column("image_copy")
.await
.unwrap()
.rows_filled,
0
);
table.checkout(first_refresh.version).await.unwrap();
assert_eq!(table.count_rows(None).await.unwrap(), 4);
assert_eq!(
table.blob_columns().await.unwrap(),
vec!["image".to_string(), "image_copy".to_string()]
);
table.checkout_latest().await.unwrap();
}
#[tokio::test]
async fn test_refresh_inherits_nested_struct_blob_input() {
use arrow_array::{Int32Array, StructArray, UInt64Array};
use arrow_schema::{DataType, Field, Fields, Schema};
let tmp = tempfile::tempdir().unwrap();
let mut blob_builder = lance::blob::BlobArrayBuilder::new(2);
blob_builder.push_bytes(b"nested").unwrap();
blob_builder.push_null().unwrap();
let blob_field = crate::blob("image", true);
let metadata_fields = Fields::from(vec![blob_field.clone()]);
let metadata = StructArray::new(
metadata_fields.clone(),
vec![blob_builder.finish().unwrap()],
None,
);
let batch = RecordBatch::try_new(
Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new("metadata", DataType::Struct(metadata_fields), true),
])),
vec![Arc::new(Int32Array::from(vec![1, 2])), Arc::new(metadata)],
)
.unwrap();
let table = create_blob_table(tmp.path(), batch).await;
table
.add_columns()
.computed("payload_copy", "metadata.image")
.execute()
.await
.unwrap();
assert_eq!(
table
.refresh_column("payload_copy")
.await
.unwrap()
.rows_filled,
1
);
assert_eq!(
table.blob_columns().await.unwrap(),
vec!["metadata.image".to_string(), "payload_copy".to_string()]
);
let batches = table
.query()
.with_row_id()
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
let row_ids = batches[0]
.column_by_name(ROW_ID)
.unwrap()
.as_any()
.downcast_ref::<UInt64Array>()
.unwrap()
.values();
let payloads = table.fetch_blobs("payload_copy", row_ids).await.unwrap();
assert_eq!(payloads.value(0), b"nested");
assert!(payloads.is_null(1));
}
#[tokio::test]
async fn test_refresh_preserves_list_shape_when_materializing_blob_input() {
use arrow_array::{Int32Array, ListArray};
use arrow_buffer::{OffsetBuffer, ScalarBuffer};
use arrow_schema::{DataType, Field, Schema};
let tmp = tempfile::tempdir().unwrap();
let mut blob_builder = lance::blob::BlobArrayBuilder::new(3);
blob_builder.push_bytes(b"a").unwrap();
blob_builder.push_bytes(b"bb").unwrap();
blob_builder.push_null().unwrap();
let item = Arc::new(crate::blob("item", true));
let images = ListArray::new(
item.clone(),
OffsetBuffer::new(ScalarBuffer::from(vec![0, 2, 3])),
blob_builder.finish().unwrap(),
None,
);
let batch = RecordBatch::try_new(
Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new("images", DataType::List(item), true),
])),
vec![Arc::new(Int32Array::from(vec![1, 2])), Arc::new(images)],
)
.unwrap();
let table = create_blob_table(tmp.path(), batch).await;
table
.add_columns()
.computed("image_payloads", "images")
.execute()
.await
.unwrap();
assert_eq!(
table
.refresh_column("image_payloads")
.await
.unwrap()
.rows_filled,
2
);
let batches = table
.query()
.select(Select::columns(&["image_payloads"]))
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
let output = batches[0]
.column_by_name("image_payloads")
.unwrap()
.as_any()
.downcast_ref::<ListArray>()
.unwrap();
assert_eq!(output.value_offsets(), &[0, 2, 3]);
assert!(output.values().as_any().is::<LargeBinaryArray>());
}
#[tokio::test]
async fn test_refresh_inherits_external_blob_input() {
use arrow_array::{Int32Array, StringArray, UInt64Array};
use arrow_schema::{DataType, Field, Schema};
let tmp = tempfile::tempdir().unwrap();
let payload = b"external-payload";
let path = tmp.path().join("payload.bin");
std::fs::write(&path, payload).unwrap();
let uri = url::Url::from_file_path(path).unwrap().to_string();
let conn = connect(tmp.path().join("db").to_str().unwrap())
.execute()
.await
.unwrap();
let table = conn
.create_empty_table(
"external",
Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
crate::blob("image", true),
])),
)
.execute()
.await
.unwrap();
let batch = RecordBatch::try_new(
Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new("image", DataType::Utf8, true),
])),
vec![
Arc::new(Int32Array::from(vec![1])),
Arc::new(StringArray::from(vec![Some(uri)])),
],
)
.unwrap();
table
.add(batch)
.allow_external_blob_outside_bases(true)
.execute()
.await
.unwrap();
table
.add_columns()
.computed("payload_copy", "image")
.execute()
.await
.unwrap();
assert_eq!(
table
.refresh_column("payload_copy")
.await
.unwrap()
.rows_filled,
1
);
let batches = table
.query()
.with_row_id()
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
let row_ids = batches[0]
.column_by_name(ROW_ID)
.unwrap()
.as_any()
.downcast_ref::<UInt64Array>()
.unwrap()
.values();
let payloads = table.fetch_blobs("payload_copy", row_ids).await.unwrap();
assert_eq!(payloads.value(0), payload);
}
}
+4 -5
View File
@@ -227,7 +227,7 @@ pub(crate) fn resolve_arrow_field_path(schema: &Schema, column: &str) -> Result<
pub(crate) struct ResolvedFtsField {
pub canonical_path: String,
pub terminal_field: Field,
pub field: Field,
pub list_depth: usize,
}
@@ -309,7 +309,7 @@ pub(crate) fn resolve_lance_fts_field_path(
);
Ok(ResolvedFtsField {
canonical_path,
terminal_field: Field::from(terminal),
field: Field::from(field),
list_depth,
})
}
@@ -375,7 +375,7 @@ pub(crate) fn resolve_arrow_fts_field_path(
message: format!("Invalid schema: {}", e),
})?;
let resolved = resolve_lance_fts_field_path(&lance_schema, column)?;
Ok((resolved.canonical_path, resolved.terminal_field))
Ok((resolved.canonical_path, resolved.field))
}
pub fn supported_btree_data_type(dtype: &DataType) -> bool {
@@ -647,9 +647,8 @@ mod tests {
Field::new("docs", text_list(), true),
]);
let (path, field) = resolve_arrow_fts_field_path(&schema, "docs.content").unwrap();
let (path, _) = resolve_arrow_fts_field_path(&schema, "docs.content").unwrap();
assert_eq!(path, "docs.content");
assert_eq!(field.data_type(), &DataType::Utf8);
let lance_schema = lance_core::datatypes::Schema::try_from(&schema).unwrap();
let field_id = lance_schema
@@ -45,11 +45,7 @@ async fn local_function_catalog_operations_return_stable_not_supported() {
.get_function("normalize_score", "fv_exact")
.await
.unwrap_err();
let drop_error = connection
.drop_function("normalize_score", "fv_exact")
.await
.unwrap_err();
for error in [create_error, lookup_error, drop_error] {
for error in [create_error, lookup_error] {
assert!(matches!(
error,
Error::NotSupported { message }
@@ -78,12 +78,6 @@
"type": "utf8"
}
},
{
"arrow_type": "large_utf8",
"json": {
"type": "large_utf8"
}
},
{
"arrow_type": "binary",
"json": {
@@ -177,21 +171,6 @@
]
}
},
{
"arrow_type": "list<large_utf8>",
"json": {
"type": "list",
"fields": [
{
"name": "item",
"nullable": false,
"type": {
"type": "large_utf8"
}
}
]
}
},
{
"arrow_type": "large_list<utf8>",
"json": {
@@ -207,21 +186,6 @@
]
}
},
{
"arrow_type": "large_list<large_utf8>",
"json": {
"type": "large_list",
"fields": [
{
"name": "item",
"nullable": false,
"type": {
"type": "large_utf8"
}
}
]
}
},
{
"arrow_type": "fixed_size_list<float32, 384>",
"json": {
@@ -366,4 +330,4 @@
"timestamp[us]",
"struct<a: int32>"
]
}
}