mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-04 20:48:50 +00:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d8f321c1f4 | |||
| 1443bf703f | |||
| 9e8140ff88 | |||
| 77c6dffc4a | |||
| a87cada90e |
@@ -44,3 +44,27 @@ updates:
|
||||
python-deps:
|
||||
patterns:
|
||||
- "*"
|
||||
|
||||
# The npm ecosystem covers pnpm lockfiles. There are two separate installs:
|
||||
# the bindings themselves and the examples, which have their own lockfile.
|
||||
# As with cargo and pip above, only bump the lockfile — the version ranges
|
||||
# in package.json are our consumers' constraints, not ours.
|
||||
- package-ecosystem: npm
|
||||
directory: /nodejs
|
||||
schedule:
|
||||
interval: weekly
|
||||
versioning-strategy: lockfile-only
|
||||
groups:
|
||||
nodejs-deps:
|
||||
patterns:
|
||||
- "*"
|
||||
|
||||
- package-ecosystem: npm
|
||||
directory: /nodejs/examples
|
||||
schedule:
|
||||
interval: weekly
|
||||
versioning-strategy: lockfile-only
|
||||
groups:
|
||||
nodejs-examples-deps:
|
||||
patterns:
|
||||
- "*"
|
||||
|
||||
@@ -29,12 +29,14 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "18"
|
||||
node-version: "24"
|
||||
- uses: pnpm/action-setup@v6
|
||||
with:
|
||||
version: 11.1.1
|
||||
# These rules are disabled because Github will always ensure there
|
||||
# is a blank line between the title and the body and Github will
|
||||
# word wrap the description field to ensure a reasonable max line
|
||||
# length.
|
||||
- run: npm install @commitlint/config-conventional
|
||||
- run: >
|
||||
echo 'module.exports = {
|
||||
"rules": {
|
||||
@@ -43,7 +45,11 @@ jobs:
|
||||
"body-leading-blank": [0, "always"]
|
||||
}
|
||||
}' > .commitlintrc.js
|
||||
- run: npx commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
|
||||
- run: >
|
||||
pnpm dlx
|
||||
--package @commitlint/cli@21.2.2
|
||||
--package @commitlint/config-conventional@21.2.2
|
||||
commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
|
||||
env:
|
||||
COMMIT_MSG: >
|
||||
${{ github.event.pull_request.title }}
|
||||
@@ -54,7 +60,7 @@ jobs:
|
||||
with:
|
||||
script: |
|
||||
const message = `**ACTION NEEDED**
|
||||
|
||||
|
||||
Lance follows the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) for release automation.
|
||||
|
||||
The PR title and description are used as the merge commit message.\
|
||||
|
||||
@@ -56,7 +56,7 @@ jobs:
|
||||
uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0
|
||||
with:
|
||||
# Restricted to http(s) on purpose. Much of docs/src is generated
|
||||
# API reference (the js/ tree comes from `npm run docs` in nodejs)
|
||||
# API reference (the js/ tree comes from `pnpm run docs` in nodejs)
|
||||
# and the hand-written pages use mkdocstrings cross-references and
|
||||
# nav-relative paths that only resolve in the site mkdocs builds,
|
||||
# not in this checkout, so relative links would be reported as
|
||||
|
||||
@@ -55,9 +55,7 @@ jobs:
|
||||
- name: Set up node
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 20
|
||||
cache: 'npm'
|
||||
cache-dependency-path: docs/package-lock.json
|
||||
node-version: 24
|
||||
- name: Install node dependencies
|
||||
working-directory: nodejs
|
||||
run: |
|
||||
|
||||
@@ -47,9 +47,8 @@ jobs:
|
||||
version: 11.1.1
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
||||
# in October. The library itself still supports Node >= 18
|
||||
# (see test matrix below).
|
||||
# Build on a supported LTS; the matrix job below covers every
|
||||
# Node version the library claims to support.
|
||||
node-version: 24
|
||||
cache: 'pnpm'
|
||||
cache-dependency-path: nodejs/pnpm-lock.yaml
|
||||
@@ -84,7 +83,7 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
node-version: [ "18", "20" ]
|
||||
node-version: [ "22", "24", "26" ]
|
||||
runs-on: "ubuntu-22.04"
|
||||
defaults:
|
||||
run:
|
||||
@@ -101,9 +100,9 @@ jobs:
|
||||
- uses: actions/setup-node@v6
|
||||
name: Setup Node.js 24 for build
|
||||
with:
|
||||
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
||||
# in October. Build/install runs on Node 24; tests run on the
|
||||
# matrix version below using direct jest invocation.
|
||||
# Build and install once on a fixed version so the generated docs
|
||||
# are identical across matrix legs; the tests below then run on each
|
||||
# supported Node version.
|
||||
node-version: 24
|
||||
cache: 'pnpm'
|
||||
cache-dependency-path: nodejs/pnpm-lock.yaml
|
||||
@@ -152,9 +151,9 @@ jobs:
|
||||
S3_TEST: "1"
|
||||
# Newer @smithy/core uses dynamic ESM imports.
|
||||
NODE_OPTIONS: "--experimental-vm-modules"
|
||||
# Invoke jest directly because pnpm 11 itself requires Node 22+
|
||||
# while the matrix tests on older Node versions.
|
||||
run: npx jest --verbose
|
||||
# Invoke the installed jest binary directly; the pnpm shim is set up
|
||||
# against the build-phase Node, not the version selected above.
|
||||
run: node_modules/.bin/jest --verbose
|
||||
- name: Test examples
|
||||
working-directory: ./
|
||||
env:
|
||||
@@ -164,7 +163,7 @@ jobs:
|
||||
run: |
|
||||
python ci/mock_openai.py &
|
||||
cd nodejs/examples
|
||||
npx jest --testEnvironment jest-environment-node-single-context --verbose
|
||||
node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose
|
||||
macos:
|
||||
timeout-minutes: 30
|
||||
# macos-15 ships a newer linker; the older macos-14 linker fails to insert
|
||||
@@ -185,8 +184,7 @@ jobs:
|
||||
version: 11.1.1
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
||||
# in October.
|
||||
# pnpm 11 requires Node >= 22.13.
|
||||
node-version: 24
|
||||
cache: 'pnpm'
|
||||
cache-dependency-path: nodejs/pnpm-lock.yaml
|
||||
|
||||
@@ -168,8 +168,7 @@ jobs:
|
||||
- name: Setup node
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
||||
# in October.
|
||||
# pnpm 11 requires Node >= 22.13.
|
||||
node-version: 24
|
||||
cache: pnpm
|
||||
cache-dependency-path: nodejs/pnpm-lock.yaml
|
||||
@@ -251,7 +250,7 @@ jobs:
|
||||
run: |
|
||||
set -e
|
||||
${{ matrix.settings.pre_build }}
|
||||
npx napi build --platform --release \
|
||||
node_modules/.bin/napi build --platform --release \
|
||||
--features ${{ matrix.settings.features }} \
|
||||
--target ${{ matrix.settings.target }} \
|
||||
--dts ../lancedb/native.d.ts \
|
||||
@@ -271,7 +270,7 @@ jobs:
|
||||
- name: Build
|
||||
run: |
|
||||
${{ matrix.settings.pre_build }}
|
||||
npx napi build --platform --release \
|
||||
node_modules/.bin/napi build --platform --release \
|
||||
--features ${{ matrix.settings.features }} \
|
||||
--target ${{ matrix.settings.target }} \
|
||||
--dts ../lancedb/native.d.ts \
|
||||
@@ -339,7 +338,7 @@ jobs:
|
||||
- target: aarch64-unknown-linux-gnu
|
||||
host: ubuntu-2404-8x-arm64
|
||||
node:
|
||||
- '20'
|
||||
- '22'
|
||||
runs-on: ${{ matrix.settings.host }}
|
||||
defaults:
|
||||
run:
|
||||
@@ -385,9 +384,9 @@ jobs:
|
||||
- name: Move built files
|
||||
run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
|
||||
- name: Test bindings
|
||||
# Invoke jest directly because pnpm 11 itself requires Node 22+
|
||||
# while the matrix tests on older Node versions.
|
||||
run: npx jest --verbose
|
||||
# Invoke the installed jest binary directly; the pnpm shim is set up
|
||||
# against the install-phase Node, not the version selected above.
|
||||
run: node_modules/.bin/jest --verbose
|
||||
publish:
|
||||
name: Publish
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
name: Update package-lock.json
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: main
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
- uses: ./.github/workflows/update_package_lock
|
||||
with:
|
||||
github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
|
||||
@@ -1,22 +0,0 @@
|
||||
name: Update NodeJs package-lock.json
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: main
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
- uses: ./.github/workflows/update_package_lock_nodejs
|
||||
with:
|
||||
github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
|
||||
@@ -20,7 +20,10 @@ repos:
|
||||
hooks:
|
||||
- id: local-biome-check
|
||||
name: biome check
|
||||
entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
|
||||
# Use the biome from nodejs/package.json rather than a separately
|
||||
# pinned one: the two drifted apart and disagreed on formatting, so
|
||||
# this hook rejected code that `pnpm lint` accepted.
|
||||
entry: nodejs/node_modules/.bin/biome check --config-path nodejs/biome.json nodejs/
|
||||
language: system
|
||||
types: [text]
|
||||
files: "nodejs/.*"
|
||||
|
||||
@@ -38,7 +38,7 @@ Before committing changes, run formatting for every language you touched. At min
|
||||
* Rust changes: run `cargo fmt --all`.
|
||||
* Python changes: run `ruff format .` and `ruff check .` from the repository root,
|
||||
and run targeted tests through `cd python && uv run ...`.
|
||||
* TypeScript changes: run the relevant `npm`/`pnpm` lint, format, build, and docs commands in `nodejs`.
|
||||
* TypeScript changes: run the relevant `pnpm` lint, format, build, and docs commands in `nodejs`.
|
||||
|
||||
Before creating a PR, the exact value passed to `gh pr create --title` must follow
|
||||
Conventional Commits, such as `fix: support nested field paths in native index creation`
|
||||
@@ -101,12 +101,12 @@ Python bindings changes:
|
||||
TypeScript bindings changes:
|
||||
|
||||
1. Add napi-rs method binding on `Table` in `nodejs/src/table.rs`.
|
||||
2. Run `npm run build` to generate TypeScript definitions.
|
||||
2. Run `pnpm build` to generate TypeScript definitions.
|
||||
3. Add typescript method on abstract class `Table` in `nodejs/src/table.ts`.
|
||||
4. Add concrete method on `LocalTable` class in `nodejs/src/native_table.ts`.
|
||||
* Note: despite the name, this class is also used for remote tables.
|
||||
5. Add test in `nodejs/__test__/table.test.ts`.
|
||||
6. Run `npm run docs` to generate TypeScript documentation.
|
||||
6. Run `pnpm run docs` to generate TypeScript documentation.
|
||||
|
||||
## Python API reference
|
||||
|
||||
|
||||
Generated
+42
-42
@@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.5",
|
||||
@@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -4888,8 +4888,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4911,7 +4911,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4925,7 +4925,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -4934,8 +4934,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -4945,8 +4945,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4983,8 +4983,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5013,8 +5013,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5031,8 +5031,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5041,8 +5041,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5075,8 +5075,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5107,8 +5107,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5172,8 +5172,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index-core"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5195,8 +5195,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5236,8 +5236,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5251,8 +5251,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5264,8 +5264,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -5318,8 +5318,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5333,8 +5333,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5374,8 +5374,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5388,8 +5388,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"frostem",
|
||||
"icu_segmenter",
|
||||
|
||||
+14
-14
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=12.0.0-beta.4", default-features = false, "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=12.0.0-beta.4", default-features = false, "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=12.0.0-beta.4", default-features = false, "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lancedb = { path = "rust/lancedb", default-features = false }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
|
||||
@@ -5,5 +5,5 @@ licenses:
|
||||
cd python && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
|
||||
cd python && uv sync --all-extras && uv tool run pip-licenses --python .venv/bin/python --format=markdown --with-urls --output-file=PYTHON_THIRD_PARTY_LICENSES.md
|
||||
cd nodejs && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
|
||||
cd nodejs && npx license-checker --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
|
||||
cd nodejs && pnpm dlx license-checker@25 --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
|
||||
cd java && ./mvnw license:aggregate-add-third-party -q
|
||||
|
||||
@@ -12,16 +12,12 @@ done
|
||||
# This updates the lockfile without building
|
||||
cargo metadata --quiet > /dev/null
|
||||
|
||||
pushd nodejs || exit 1
|
||||
npm install --package-lock-only --silent
|
||||
popd
|
||||
|
||||
if git diff --quiet --exit-code; then
|
||||
echo "No lockfile changes to commit; skipping amend."
|
||||
elif $AMEND; then
|
||||
git add Cargo.lock nodejs/package-lock.json
|
||||
git add Cargo.lock
|
||||
git commit --amend --no-edit
|
||||
else
|
||||
git add Cargo.lock nodejs/package-lock.json
|
||||
git add Cargo.lock
|
||||
git commit -m "Update lockfiles"
|
||||
fi
|
||||
|
||||
+11
-8
@@ -47,22 +47,24 @@ pytest -vv python/tests/docs
|
||||
|
||||
### Checking typescript examples
|
||||
|
||||
The `@lancedb/lancedb` package must be built before running the tests:
|
||||
The examples depend on `@lancedb/lancedb` at `file:../dist`, so the package must be
|
||||
built before running the tests. This uses pnpm; see the
|
||||
[Typescript contributing guide](../nodejs/CONTRIBUTING.md) for the toolchain setup.
|
||||
|
||||
```shell
|
||||
pushd nodejs
|
||||
npm ci
|
||||
npm run build
|
||||
pnpm install
|
||||
pnpm build
|
||||
popd
|
||||
```
|
||||
|
||||
Then you can run the examples by going to the `nodejs/examples` directory and
|
||||
running the tests like a normal npm package:
|
||||
Then you can run the examples by going to the `nodejs/examples` directory, which is a
|
||||
separate pnpm package with its own lockfile:
|
||||
|
||||
```shell
|
||||
pushd nodejs/examples
|
||||
npm ci
|
||||
npm test
|
||||
pnpm install
|
||||
pnpm test
|
||||
popd
|
||||
```
|
||||
|
||||
@@ -84,6 +86,7 @@ The new files should be checked into the repository.
|
||||
|
||||
```shell
|
||||
pushd nodejs
|
||||
npm run docs
|
||||
# `pnpm docs` would invoke pnpm's built-in `docs` command, not the script.
|
||||
pnpm run docs
|
||||
popd
|
||||
```
|
||||
|
||||
Generated
-135
@@ -1,135 +0,0 @@
|
||||
{
|
||||
"name": "lancedb-docs-test",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "lancedb-docs-test",
|
||||
"version": "1.0.0",
|
||||
"license": "Apache 2",
|
||||
"dependencies": {
|
||||
"apache-arrow": "file:../node/node_modules/apache-arrow",
|
||||
"vectordb": "file:../node"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.11.8",
|
||||
"typescript": "^5.3.3"
|
||||
}
|
||||
},
|
||||
"../node": {
|
||||
"name": "vectordb",
|
||||
"version": "0.21.2-beta.0",
|
||||
"cpu": [
|
||||
"x64",
|
||||
"arm64"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"os": [
|
||||
"darwin",
|
||||
"linux",
|
||||
"win32"
|
||||
],
|
||||
"dependencies": {
|
||||
"@neon-rs/load": "^0.0.74",
|
||||
"axios": "^1.4.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@neon-rs/cli": "^0.0.160",
|
||||
"@types/chai": "^4.3.4",
|
||||
"@types/chai-as-promised": "^7.1.5",
|
||||
"@types/mocha": "^10.0.1",
|
||||
"@types/node": "^18.16.2",
|
||||
"@types/sinon": "^10.0.15",
|
||||
"@types/temp": "^0.9.1",
|
||||
"@types/uuid": "^9.0.3",
|
||||
"@typescript-eslint/eslint-plugin": "^5.59.1",
|
||||
"apache-arrow-old": "npm:apache-arrow@13.0.0",
|
||||
"cargo-cp-artifact": "^0.1",
|
||||
"chai": "^4.3.7",
|
||||
"chai-as-promised": "^7.1.1",
|
||||
"eslint": "^8.39.0",
|
||||
"eslint-config-standard-with-typescript": "^34.0.1",
|
||||
"eslint-plugin-import": "^2.26.0",
|
||||
"eslint-plugin-n": "^15.7.0",
|
||||
"eslint-plugin-promise": "^6.1.1",
|
||||
"mocha": "^10.2.0",
|
||||
"openai": "^4.24.1",
|
||||
"sinon": "^15.1.0",
|
||||
"temp": "^0.9.4",
|
||||
"ts-node": "^10.9.1",
|
||||
"ts-node-dev": "^2.0.0",
|
||||
"typedoc": "^0.24.7",
|
||||
"typedoc-plugin-markdown": "^3.15.3",
|
||||
"typescript": "^5.1.0",
|
||||
"uuid": "^9.0.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@lancedb/vectordb-darwin-arm64": "0.21.2-beta.0",
|
||||
"@lancedb/vectordb-darwin-x64": "0.21.2-beta.0",
|
||||
"@lancedb/vectordb-linux-arm64-gnu": "0.21.2-beta.0",
|
||||
"@lancedb/vectordb-linux-x64-gnu": "0.21.2-beta.0",
|
||||
"@lancedb/vectordb-win32-x64-msvc": "0.21.2-beta.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@apache-arrow/ts": "^14.0.2",
|
||||
"apache-arrow": "^14.0.2"
|
||||
}
|
||||
},
|
||||
"../node/node_modules/apache-arrow": {
|
||||
"version": "14.0.2",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@types/command-line-args": "5.2.0",
|
||||
"@types/command-line-usage": "5.0.2",
|
||||
"@types/node": "20.3.0",
|
||||
"@types/pad-left": "2.1.1",
|
||||
"command-line-args": "5.2.1",
|
||||
"command-line-usage": "7.0.1",
|
||||
"flatbuffers": "23.5.26",
|
||||
"json-bignum": "^0.0.3",
|
||||
"pad-left": "^2.1.0",
|
||||
"tslib": "^2.5.3"
|
||||
},
|
||||
"bin": {
|
||||
"arrow2csv": "bin/arrow2csv.js"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "20.11.8",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.8.tgz",
|
||||
"integrity": "sha512-i7omyekpPTNdv4Jb/Rgqg0RU8YqLcNsI12quKSDkRXNfx7Wxdm6HhK1awT3xTgEkgxPn3bvnSpiEAc7a7Lpyow==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"undici-types": "~5.26.4"
|
||||
}
|
||||
},
|
||||
"node_modules/apache-arrow": {
|
||||
"resolved": "../node/node_modules/apache-arrow",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/typescript": {
|
||||
"version": "5.3.3",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz",
|
||||
"integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==",
|
||||
"dev": true,
|
||||
"bin": {
|
||||
"tsc": "bin/tsc",
|
||||
"tsserver": "bin/tsserver"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.17"
|
||||
}
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "5.26.5",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
|
||||
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/vectordb": {
|
||||
"resolved": "../node",
|
||||
"link": true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"name": "lancedb-docs-test",
|
||||
"version": "1.0.0",
|
||||
"description": "auto-generated tests from doc",
|
||||
"author": "dev@lancedb.com",
|
||||
"license": "Apache 2",
|
||||
"dependencies": {
|
||||
"apache-arrow": "file:../node/node_modules/apache-arrow",
|
||||
"vectordb": "file:../node"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc -b && cd ../node && npm run build-release",
|
||||
"example": "npm run build && node",
|
||||
"test": "npm run build && ls dist/*.js | xargs -n 1 node"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.11.8",
|
||||
"typescript": "^5.3.3"
|
||||
}
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
{
|
||||
"include": [
|
||||
"src/*.ts",
|
||||
],
|
||||
"compilerOptions": {
|
||||
"target": "es2022",
|
||||
"module": "nodenext",
|
||||
"declaration": true,
|
||||
"outDir": "./dist",
|
||||
"strict": true,
|
||||
"allowJs": true,
|
||||
"resolveJsonModule": true,
|
||||
},
|
||||
"exclude": [
|
||||
"./dist/*",
|
||||
]
|
||||
}
|
||||
+1
-1
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>12.0.0-beta.2</lance-core.version>
|
||||
<lance-core.version>12.0.0-beta.4</lance-core.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||
|
||||
@@ -5,8 +5,8 @@ import packageJson = require("../package.json");
|
||||
|
||||
describe("package metadata", () => {
|
||||
it("requires Node.js type declarations compatible with the runtime", () => {
|
||||
expect(packageJson.engines.node).toBe(">= 18");
|
||||
expect(packageJson.peerDependencies["@types/node"]).toBe(">=18");
|
||||
expect(packageJson.engines.node).toBe(">= 22");
|
||||
expect(packageJson.peerDependencies["@types/node"]).toBe(">=22");
|
||||
expect(packageJson.peerDependenciesMeta["@types/node"]).toEqual({
|
||||
optional: true,
|
||||
});
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
import * as http from "http";
|
||||
import { RequestListener } from "http";
|
||||
import packageJson = require("../package.json");
|
||||
import {
|
||||
ClientConfig,
|
||||
Connection,
|
||||
@@ -70,7 +71,13 @@ async function withMockDatabase(
|
||||
try {
|
||||
await callback(db);
|
||||
} finally {
|
||||
server.close();
|
||||
// `close()` alone leaves the port bound until keep-alive sockets drain, so
|
||||
// a single failing test would cascade into EADDRINUSE for every test after
|
||||
// it. Destroy the connections and wait for the port to actually be free.
|
||||
await new Promise<void>((resolve) => {
|
||||
server.closeAllConnections();
|
||||
server.close(() => resolve());
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -131,7 +138,7 @@ describe("remote connection", () => {
|
||||
(req, res) => {
|
||||
expect(req.headers["x-api-key"]).toEqual("fake");
|
||||
expect(req.headers["user-agent"]).toEqual(
|
||||
`LanceDB-Node-Client/${process.env.npm_package_version}`,
|
||||
`LanceDB-Node-Client/${packageJson.version}`,
|
||||
);
|
||||
|
||||
const body = JSON.stringify({ tables: [] });
|
||||
|
||||
@@ -8,7 +8,8 @@
|
||||
"//1": "--experimental-vm-modules is needed to run jest with sentence-transformers",
|
||||
"//2": "--testEnvironment is needed to run jest with sentence-transformers",
|
||||
"//3": "See: https://github.com/huggingface/transformers.js/issues/57",
|
||||
"test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
|
||||
"//4": "jest is invoked by its JS entry, not node_modules/.bin/jest: under pnpm that path is a shell shim, which `node` cannot execute",
|
||||
"test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testEnvironment jest-environment-node-single-context --verbose",
|
||||
"lint": "biome check *.ts && biome format *.ts",
|
||||
"lint-ci": "biome ci .",
|
||||
"lint-fix": "biome check --write *.ts && pnpm format",
|
||||
|
||||
Generated
-11106
File diff suppressed because it is too large
Load Diff
+2
-2
@@ -67,7 +67,7 @@
|
||||
"timeout": "3m"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 18"
|
||||
"node": ">= 22"
|
||||
},
|
||||
"packageManager": "pnpm@11.1.1",
|
||||
"cpu": ["x64", "arm64"],
|
||||
@@ -101,7 +101,7 @@
|
||||
"openai": "4.29.2"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@types/node": ">=18",
|
||||
"@types/node": ">=22",
|
||||
"apache-arrow": ">=15.0.0 <=18.1.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
|
||||
@@ -54,6 +54,18 @@ _UInt32 = conint(strict=True, ge=0, le=2**32 - 1)
|
||||
_UInt64 = conint(strict=True, ge=0, le=2**64 - 1)
|
||||
|
||||
|
||||
def _validate_gpu_wire_marker(value: Any) -> bool:
|
||||
if value is not True:
|
||||
raise ValueError("runtime.gpu must be true")
|
||||
return True
|
||||
|
||||
|
||||
def _normalize_gpu_marker(value: bool) -> Optional[bool]:
|
||||
if not isinstance(value, bool):
|
||||
raise ValueError("gpu must be a boolean")
|
||||
return True if value else None
|
||||
|
||||
|
||||
class _FrozenDict(dict):
|
||||
def _immutable(self, *args, **kwargs):
|
||||
raise TypeError("remote canonical values are immutable")
|
||||
@@ -239,6 +251,23 @@ class PythonRuntimeSpec(_RemoteValue):
|
||||
python_version: Optional[str] = None
|
||||
environment: Optional[PythonEnvironmentSpec] = None
|
||||
env: Optional[Mapping[str, str]] = None
|
||||
gpu: Optional[bool] = None
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def _discard_unknown_runtime_payload(cls, value):
|
||||
if isinstance(value, Mapping):
|
||||
kind = value.get("kind")
|
||||
if isinstance(kind, str) and kind not in {"python", "python_v2"}:
|
||||
return {"kind": kind}
|
||||
return value
|
||||
|
||||
@field_validator("gpu", mode="before")
|
||||
@classmethod
|
||||
def _validate_gpu_marker(cls, value):
|
||||
if value is None:
|
||||
return None
|
||||
return _validate_gpu_wire_marker(value)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_runtime_kind(self):
|
||||
@@ -247,18 +276,28 @@ class PythonRuntimeSpec(_RemoteValue):
|
||||
raise ValueError("python runtime requires python_version")
|
||||
if self.environment is None:
|
||||
raise ValueError("python runtime requires environment")
|
||||
if self.gpu is not None:
|
||||
raise ValueError("python runtime with gpu requires kind='python_v2'")
|
||||
elif self.kind == "python_v2":
|
||||
if self.python_version is None:
|
||||
raise ValueError("python_v2 runtime requires python_version")
|
||||
if self.environment is None:
|
||||
raise ValueError("python_v2 runtime requires environment")
|
||||
if self.gpu is None:
|
||||
raise ValueError("python_v2 runtime requires gpu")
|
||||
else:
|
||||
object.__setattr__(self, "python_version", None)
|
||||
object.__setattr__(self, "environment", None)
|
||||
object.__setattr__(self, "env", None)
|
||||
object.__setattr__(self, "gpu", None)
|
||||
return self
|
||||
|
||||
|
||||
class FunctionVersion(_RemoteValue):
|
||||
"""An exact immutable Function version returned by Enterprise.
|
||||
|
||||
Scheduling resources, priority, concurrency, and retry policy belong to
|
||||
the submitting Job and are not part of this identity.
|
||||
The GPU execution requirement is part of this identity. CPU and memory sizing,
|
||||
priority, concurrency, and retry policy belong to the execution platform.
|
||||
"""
|
||||
|
||||
name: str
|
||||
@@ -910,6 +949,7 @@ class UdfDefinition:
|
||||
pip: tuple[str, ...],
|
||||
env: Mapping[str, str],
|
||||
python_version: Optional[str],
|
||||
gpu: bool = False,
|
||||
conda: tuple[str, ...] = (),
|
||||
conda_channels: tuple[str, ...] = (),
|
||||
):
|
||||
@@ -938,12 +978,14 @@ class UdfDefinition:
|
||||
signature = _infer_signature(function, input_schema, output_schema)
|
||||
source = _package_source(function)
|
||||
digest = f"sha256:{hashlib.sha256(source).hexdigest()}"
|
||||
gpu_marker = _normalize_gpu_marker(gpu)
|
||||
runtime = PythonRuntimeSpec(
|
||||
kind="python",
|
||||
kind="python_v2" if gpu_marker is not None else "python",
|
||||
python_version=python_version
|
||||
or f"{sys.version_info.major}.{sys.version_info.minor}",
|
||||
environment=environment_spec,
|
||||
env=environment,
|
||||
gpu=gpu_marker,
|
||||
)
|
||||
self._function = function
|
||||
self._request = FunctionRegistrationRequest(
|
||||
@@ -989,6 +1031,7 @@ def udf(
|
||||
pip: tuple[str, ...] | list[str] = (),
|
||||
env: Optional[Mapping[str, str]] = None,
|
||||
python_version: Optional[str] = None,
|
||||
gpu: bool = False,
|
||||
conda: tuple[str, ...] | list[str] = (),
|
||||
conda_channels: tuple[str, ...] | list[str] = (),
|
||||
) -> Callable[[Callable[..., Any]], UdfDefinition]: ...
|
||||
@@ -1003,6 +1046,7 @@ def udf(
|
||||
pip: tuple[str, ...] | list[str] = (),
|
||||
env: Optional[Mapping[str, str]] = None,
|
||||
python_version: Optional[str] = None,
|
||||
gpu: bool = False,
|
||||
conda: tuple[str, ...] | list[str] = (),
|
||||
conda_channels: tuple[str, ...] | list[str] = (),
|
||||
):
|
||||
@@ -1035,6 +1079,10 @@ def udf(
|
||||
Environment variables included in the Function definition.
|
||||
python_version : str, optional
|
||||
Remote Python major/minor version. Defaults to the client version.
|
||||
gpu : bool, default False
|
||||
Whether every remote execution requires a GPU. The execution platform
|
||||
selects one compatible GPU for each worker. The requirement is part of
|
||||
the immutable Function version.
|
||||
|
||||
The packaged artifact is a snapshot: the function source plus exactly
|
||||
the module-level names it references (modules as imports, importable
|
||||
@@ -1059,6 +1107,11 @@ def udf(
|
||||
... return value * 2
|
||||
>>> score(1.5)
|
||||
3.0
|
||||
>>> @udf(pip=["cupy-cuda12x"], gpu=True)
|
||||
... def gpu_score(value: int) -> int:
|
||||
... return value * 2
|
||||
>>> gpu_score.registration_request.runtime.gpu
|
||||
True
|
||||
"""
|
||||
|
||||
def decorate(target: Callable[..., Any]) -> UdfDefinition:
|
||||
@@ -1070,6 +1123,7 @@ def udf(
|
||||
pip=tuple(pip),
|
||||
env={} if env is None else env,
|
||||
python_version=python_version,
|
||||
gpu=gpu,
|
||||
conda=tuple(conda),
|
||||
conda_channels=tuple(conda_channels),
|
||||
)
|
||||
|
||||
@@ -19,7 +19,7 @@ import pyarrow as pa
|
||||
import pytest
|
||||
|
||||
import lancedb
|
||||
from lancedb.functions import UdfDefinition, udf
|
||||
from lancedb.functions import PythonRuntimeSpec, UdfDefinition, udf
|
||||
|
||||
THRESHOLD = 20
|
||||
_CACHE = None
|
||||
@@ -89,6 +89,58 @@ def test_udf_conda_environment():
|
||||
udf(name="channels", conda_channels=["conda-forge"])(lambda value: value)
|
||||
|
||||
|
||||
def test_udf_gpu_marker_uses_gpu_runtime():
|
||||
@udf(pip=["cupy-cuda12x"], gpu=True)
|
||||
def double_on_gpu(value: int) -> int:
|
||||
return value * 2
|
||||
|
||||
request = json.loads(double_on_gpu.registration_request.to_canonical_json())
|
||||
assert request["runtime"]["kind"] == "python_v2"
|
||||
assert request["runtime"]["gpu"] is True
|
||||
|
||||
@udf(pip=["pyarrow"])
|
||||
def cpu_function(value: int) -> int:
|
||||
return value
|
||||
|
||||
cpu_runtime = json.loads(cpu_function.registration_request.to_canonical_json())[
|
||||
"runtime"
|
||||
]
|
||||
assert cpu_runtime["kind"] == "python"
|
||||
assert "gpu" not in cpu_runtime
|
||||
|
||||
def identity(value: int) -> int:
|
||||
return value
|
||||
|
||||
for invalid in [None, 0, 1, -1, 1.5, "", "true", "1", "H100"]:
|
||||
with pytest.raises(ValueError, match="gpu must be a boolean"):
|
||||
udf(name="invalid_gpu", gpu=invalid)(identity)
|
||||
|
||||
base_runtime = {
|
||||
"kind": "python_v2",
|
||||
"python_version": "3.12",
|
||||
"environment": {"kind": "pip"},
|
||||
}
|
||||
runtime = PythonRuntimeSpec.model_validate({**base_runtime, "gpu": True})
|
||||
assert runtime.gpu is True
|
||||
for invalid in [False, 1, 0, "", "true", "1", "H100"]:
|
||||
with pytest.raises(ValueError, match="runtime.gpu must be true"):
|
||||
PythonRuntimeSpec.model_validate({**base_runtime, "gpu": invalid})
|
||||
|
||||
|
||||
def test_unknown_runtime_discards_payload_before_known_field_validation():
|
||||
for payload in [
|
||||
{"kind": "python_v3", "gpu": {"model": "H100"}},
|
||||
{"kind": "python_v3", "resources": []},
|
||||
{
|
||||
"kind": "python_v3",
|
||||
"environment": {"kind": []},
|
||||
"python_version": 3.15,
|
||||
},
|
||||
]:
|
||||
runtime = PythonRuntimeSpec.model_validate(payload)
|
||||
assert runtime.to_canonical_json() == '{"kind":"python_v3"}'
|
||||
|
||||
|
||||
def test_udf_packages_attribute_access_and_body_imports():
|
||||
@udf
|
||||
def word_norm(body: str) -> float:
|
||||
|
||||
+150
-27
@@ -207,6 +207,33 @@ pub enum PythonRuntimeSpec {
|
||||
environment: PythonEnvironmentSpec,
|
||||
env: BTreeMap<String, String>,
|
||||
},
|
||||
/// The GPU-enabled Sophon-managed Python runtime.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use std::collections::BTreeMap;
|
||||
/// use lancedb::function::{PythonEnvironmentSpec, PythonRuntimeSpec};
|
||||
///
|
||||
/// let runtime = PythonRuntimeSpec::PythonV2 {
|
||||
/// python_version: "3.12".to_string(),
|
||||
/// environment: PythonEnvironmentSpec {
|
||||
/// kind: "pip".to_string(),
|
||||
/// packages: vec!["cupy-cuda12x".to_string()],
|
||||
/// channels: Vec::new(),
|
||||
/// path: None,
|
||||
/// modules: Vec::new(),
|
||||
/// image: None,
|
||||
/// },
|
||||
/// env: BTreeMap::new(),
|
||||
/// };
|
||||
/// assert!(runtime.requires_gpu());
|
||||
/// ```
|
||||
PythonV2 {
|
||||
python_version: String,
|
||||
environment: PythonEnvironmentSpec,
|
||||
env: BTreeMap<String, String>,
|
||||
},
|
||||
/// A runtime kind introduced by a newer server.
|
||||
///
|
||||
/// Unknown payload fields are intentionally not retained because the
|
||||
@@ -219,22 +246,27 @@ impl PythonRuntimeSpec {
|
||||
pub fn kind(&self) -> &str {
|
||||
match self {
|
||||
Self::Python { .. } => "python",
|
||||
Self::PythonV2 { .. } => "python_v2",
|
||||
Self::Unrecognized { kind } => kind,
|
||||
}
|
||||
}
|
||||
|
||||
/// The Python version for the V1 runtime, or `None` for an unknown kind.
|
||||
/// The Python version for a known Python runtime, or `None` for an unknown kind.
|
||||
pub fn python_version(&self) -> Option<&str> {
|
||||
match self {
|
||||
Self::Python { python_version, .. } => Some(python_version),
|
||||
Self::Python { python_version, .. } | Self::PythonV2 { python_version, .. } => {
|
||||
Some(python_version)
|
||||
}
|
||||
Self::Unrecognized { .. } => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The Python environment for the V1 runtime, or `None` for an unknown kind.
|
||||
/// The Python environment for a known Python runtime, or `None` for an unknown kind.
|
||||
pub fn environment(&self) -> Option<&PythonEnvironmentSpec> {
|
||||
match self {
|
||||
Self::Python { environment, .. } => Some(environment),
|
||||
Self::Python { environment, .. } | Self::PythonV2 { environment, .. } => {
|
||||
Some(environment)
|
||||
}
|
||||
Self::Unrecognized { .. } => None,
|
||||
}
|
||||
}
|
||||
@@ -242,38 +274,73 @@ impl PythonRuntimeSpec {
|
||||
/// Environment variables, or `None` for an unknown kind.
|
||||
pub fn env(&self) -> Option<&BTreeMap<String, String>> {
|
||||
match self {
|
||||
Self::Python { env, .. } => Some(env),
|
||||
Self::Python { env, .. } | Self::PythonV2 { env, .. } => Some(env),
|
||||
Self::Unrecognized { .. } => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the runtime requires a GPU selected by the execution platform.
|
||||
pub fn requires_gpu(&self) -> bool {
|
||||
matches!(self, Self::PythonV2 { .. })
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct PythonRuntimeWire {
|
||||
kind: String,
|
||||
#[serde(default)]
|
||||
python_version: Option<String>,
|
||||
#[serde(default)]
|
||||
environment: Option<PythonEnvironmentSpec>,
|
||||
struct PythonRuntimeV1Wire {
|
||||
python_version: String,
|
||||
environment: PythonEnvironmentSpec,
|
||||
#[serde(default)]
|
||||
env: BTreeMap<String, String>,
|
||||
#[serde(default)]
|
||||
gpu: Option<Value>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct PythonRuntimeV2Wire {
|
||||
python_version: String,
|
||||
environment: PythonEnvironmentSpec,
|
||||
#[serde(default)]
|
||||
env: BTreeMap<String, String>,
|
||||
gpu: bool,
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for PythonRuntimeSpec {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
|
||||
let wire = PythonRuntimeWire::deserialize(deserializer)?;
|
||||
if wire.kind == "python" {
|
||||
Ok(Self::Python {
|
||||
python_version: wire
|
||||
.python_version
|
||||
.ok_or_else(|| de::Error::missing_field("python_version"))?,
|
||||
environment: wire
|
||||
.environment
|
||||
.ok_or_else(|| de::Error::missing_field("environment"))?,
|
||||
env: wire.env,
|
||||
})
|
||||
} else {
|
||||
Ok(Self::Unrecognized { kind: wire.kind })
|
||||
let value = Value::deserialize(deserializer)?;
|
||||
let kind = value
|
||||
.get("kind")
|
||||
.ok_or_else(|| de::Error::missing_field("kind"))?
|
||||
.as_str()
|
||||
.ok_or_else(|| de::Error::custom("runtime.kind must be a string"))?
|
||||
.to_string();
|
||||
match kind.as_str() {
|
||||
"python" => {
|
||||
let wire: PythonRuntimeV1Wire =
|
||||
serde_json::from_value(value).map_err(de::Error::custom)?;
|
||||
if wire.gpu.is_some() {
|
||||
return Err(de::Error::custom(
|
||||
"python runtime with gpu requires kind='python_v2'",
|
||||
));
|
||||
}
|
||||
Ok(Self::Python {
|
||||
python_version: wire.python_version,
|
||||
environment: wire.environment,
|
||||
env: wire.env,
|
||||
})
|
||||
}
|
||||
"python_v2" => {
|
||||
let wire: PythonRuntimeV2Wire =
|
||||
serde_json::from_value(value).map_err(de::Error::custom)?;
|
||||
if !wire.gpu {
|
||||
return Err(de::Error::custom("runtime.gpu must be true"));
|
||||
}
|
||||
Ok(Self::PythonV2 {
|
||||
python_version: wire.python_version,
|
||||
environment: wire.environment,
|
||||
env: wire.env,
|
||||
})
|
||||
}
|
||||
_ => Ok(Self::Unrecognized { kind }),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -287,6 +354,8 @@ impl Serialize for PythonRuntimeSpec {
|
||||
environment: &'a PythonEnvironmentSpec,
|
||||
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
|
||||
env: &'a BTreeMap<String, String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
gpu: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -304,6 +373,19 @@ impl Serialize for PythonRuntimeSpec {
|
||||
python_version,
|
||||
environment,
|
||||
env,
|
||||
gpu: None,
|
||||
}
|
||||
.serialize(serializer),
|
||||
Self::PythonV2 {
|
||||
python_version,
|
||||
environment,
|
||||
env,
|
||||
} => PythonRuntimeRef {
|
||||
kind: "python_v2",
|
||||
python_version,
|
||||
environment,
|
||||
env,
|
||||
gpu: Some(true),
|
||||
}
|
||||
.serialize(serializer),
|
||||
Self::Unrecognized { kind } => UnrecognizedRuntimeRef { kind }.serialize(serializer),
|
||||
@@ -313,8 +395,8 @@ impl Serialize for PythonRuntimeSpec {
|
||||
|
||||
/// Immutable Function version returned by the Enterprise catalog.
|
||||
///
|
||||
/// Scheduling resources, priority, concurrency, and retry policy belong to
|
||||
/// the submitting Job and are not part of this identity.
|
||||
/// The GPU execution requirement is part of this identity. CPU and memory sizing,
|
||||
/// priority, concurrency, and retry policy belong to the execution platform.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct FunctionVersion {
|
||||
name: String,
|
||||
@@ -589,7 +671,7 @@ impl_json!(RefreshColumnResult);
|
||||
|
||||
#[cfg(test)]
|
||||
mod conda_environment_tests {
|
||||
use super::PythonEnvironmentSpec;
|
||||
use super::{PythonEnvironmentSpec, PythonRuntimeSpec};
|
||||
|
||||
#[test]
|
||||
fn conda_channels_round_trip_and_pip_stays_bare() {
|
||||
@@ -608,4 +690,45 @@ mod conda_environment_tests {
|
||||
serde_json::from_str(r#"{"kind":"pip","packages":["numpy"]}"#).unwrap();
|
||||
assert!(!serde_json::to_string(&pip).unwrap().contains("channels"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gpu_python_runtime_marker_round_trips_and_validates() {
|
||||
let runtime: PythonRuntimeSpec = serde_json::from_str(
|
||||
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":true}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(runtime.kind(), "python_v2");
|
||||
assert!(runtime.requires_gpu());
|
||||
assert_eq!(
|
||||
super::canonical_json(&runtime).unwrap(),
|
||||
r#"{"environment":{"kind":"pip"},"gpu":true,"kind":"python_v2","python_version":"3.12"}"#
|
||||
);
|
||||
|
||||
for invalid in [
|
||||
r#"{"kind":"python","python_version":"3.12","environment":{"kind":"pip"},"gpu":true}"#,
|
||||
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"}}"#,
|
||||
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":1}"#,
|
||||
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":false}"#,
|
||||
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":"true"}"#,
|
||||
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":"H100"}"#,
|
||||
] {
|
||||
assert!(serde_json::from_str::<PythonRuntimeSpec>(invalid).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_runtime_discards_payload_before_known_field_validation() {
|
||||
for encoded in [
|
||||
r#"{"kind":"python_v3","gpu":{"model":"H100"}}"#,
|
||||
r#"{"kind":"python_v3","resources":[]}"#,
|
||||
r#"{"kind":"python_v3","python_version":3.15,"environment":{"kind":[]}}"#,
|
||||
] {
|
||||
let runtime: PythonRuntimeSpec = serde_json::from_str(encoded).unwrap();
|
||||
assert_eq!(runtime.kind(), "python_v3");
|
||||
assert_eq!(
|
||||
super::canonical_json(&runtime).unwrap(),
|
||||
r#"{"kind":"python_v3"}"#
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5734,9 +5734,8 @@ mod tests {
|
||||
))
|
||||
.execute()
|
||||
.await;
|
||||
let err = match result {
|
||||
Ok(_) => panic!("legacy remote query unexpectedly succeeded"),
|
||||
Err(err) => err,
|
||||
let Err(err) = result else {
|
||||
panic!("legacy remote query unexpectedly succeeded")
|
||||
};
|
||||
|
||||
assert!(
|
||||
|
||||
Reference in New Issue
Block a user