From 8a4effa06a5d08deebf8336e936e45743a0fc538 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 27 Oct 2026 14:32:09 +0800 Subject: [PATCH] fix: gate experimental local functions behind an opt-in feature (#4688) Published SDK builds currently expose local SQL computed columns and materialized-view execution that are not ready for release. Add the opt-in `experimental-functions` Cargo feature to the Rust core and forward it through the Python and Node bindings; published packages leave it disabled. Local declarations and refreshes now return `NotSupported` before mutating tables or spawning jobs, including prepared materialized-view creation. Remote Cloud/Enterprise client operations, Python UDF packaging, ordinary eager `add_columns`, and access to existing data remain available. Stored metadata and write protections stay active, and shared server commit helpers remain usable. CI covers both feature configurations. Existing local execution tests run with the feature enabled; default-build regressions cover early rejection, unchanged table versions, existing metadata protections, and normal eager transforms. API and source-build documentation explain the opt-in. Validation: Rust library tests pass with the feature disabled (1,869 tests) and enabled (1,947 tests), plus three default-build integration regressions. Python and TypeScript bindings were rebuilt and tested in both configurations, including mocked remote requests; Rust formatting, Python package lint, TypeScript lint/build, and generated docs were checked. Repository-wide Ruff still reports 48 pre-existing errors in unchanged `docs/web-tests` files. --- .github/workflows/nodejs.yml | 8 +- .github/workflows/python.yml | 7 +- .github/workflows/rust.yml | 2 + docs/src/js/_media/CONTRIBUTING.md | 13 ++ docs/src/js/classes/Connection.md | 4 + docs/src/js/classes/MaterializedView.md | 1 + docs/src/js/classes/Table.md | 7 + docs/src/python/python.md | 10 +- .../add-computed-column.mdx | 9 +- .../materialized-views.mdx | 9 +- .../feature-engineering/refresh-column.mdx | 9 +- nodejs/CONTRIBUTING.md | 13 ++ nodejs/Cargo.toml | 1 + .../__test__/experimental_functions.test.ts | 44 ++++ nodejs/__test__/materialized_view.test.ts | 65 +++--- nodejs/__test__/table.test.ts | 8 +- nodejs/lancedb/connection.ts | 4 + nodejs/lancedb/materialized_view.ts | 1 + nodejs/lancedb/table.ts | 7 + nodejs/src/lib.rs | 6 + python/CONTRIBUTING.md | 12 ++ python/Cargo.toml | 1 + python/python/lancedb/_lancedb.pyi | 2 + python/python/lancedb/db.py | 16 +- python/python/lancedb/materialized_view.py | 7 +- python/python/lancedb/table.py | 38 +++- python/python/tests/conftest.py | 15 ++ .../tests/test_experimental_functions.py | 58 +++++ .../python/tests/test_materialized_views.py | 24 +++ python/python/tests/test_namespace.py | 24 ++- python/python/tests/test_table.py | 7 + python/src/lib.rs | 4 + rust/lancedb/Cargo.toml | 3 + rust/lancedb/src/lib.rs | 18 ++ rust/lancedb/src/materialized_view.rs | 29 +++ rust/lancedb/src/materialized_view/refresh.rs | 5 + rust/lancedb/src/table/add_columns.rs | 11 + rust/lancedb/src/table/computed_columns.rs | 120 +++++++++++ rust/lancedb/src/table/refresh.rs | 162 ++++++++++++++ rust/lancedb/src/table/schema_evolution.rs | 39 ++-- rust/lancedb/tests/experimental_functions.rs | 198 ++++++++++++++++++ 41 files changed, 955 insertions(+), 66 deletions(-) create mode 100644 nodejs/__test__/experimental_functions.test.ts create mode 100644 python/python/tests/test_experimental_functions.py create mode 100644 rust/lancedb/tests/experimental_functions.rs diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 2e290166e..e83c8d29f 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -79,11 +79,15 @@ jobs: # that dir is missing, so create an empty one for lint-only runs. run: mkdir -p ../dist && pnpm install --frozen-lockfile && pnpm lint-ci linux: - name: Linux (NodeJS ${{ matrix.node-version }}) + name: Linux (NodeJS ${{ matrix.node-version }}, ${{ matrix.features }}) timeout-minutes: 30 strategy: matrix: node-version: [ "22", "24", "26" ] + features: [ "default" ] + include: + - node-version: "24" + features: "experimental-functions" runs-on: "ubuntu-22.04" defaults: run: @@ -121,7 +125,7 @@ jobs: pnpm install --frozen-lockfile # No `--` separator: pnpm forwards it literally, which would # make napi-rs treat `--profile ci` as a cargo passthrough arg. - pnpm build:debug --profile ci + pnpm build:debug --profile ci --features ${{ matrix.features }} pnpm tsc - name: Setup examples working-directory: nodejs/examples diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 3608c24ca..a18ae5ad9 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -162,7 +162,10 @@ jobs: - name: Delete wheels run: rm -rf target/wheels platform: - name: "Mac" + name: "Mac: ${{ matrix.features }}" + strategy: + matrix: + features: ["default", "experimental-functions"] timeout-minutes: 30 runs-on: macos-14 defaults: @@ -188,7 +191,7 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} - uses: ./.github/workflows/build_mac_wheel with: - args: --profile ci + args: --profile ci --features ${{ matrix.features }} - uses: ./.github/workflows/run_tests # Make sure wheels are not included in the Rust cache - name: Delete wheels diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ffd2162c6..96b0f2bfb 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -261,6 +261,8 @@ jobs: sudo apt update sudo apt install -y protobuf-compiler libssl-dev - uses: rui314/setup-mold@v1 + - name: Test the default local Functions gate + run: cargo test --profile ci -p lancedb --features remote --test experimental_functions --locked - name: Run remote tests run: CARGO_ARGS="--profile ci" make -C ./lancedb remote-tests diff --git a/docs/src/js/_media/CONTRIBUTING.md b/docs/src/js/_media/CONTRIBUTING.md index 2faa8cd37..2125cc8aa 100644 --- a/docs/src/js/_media/CONTRIBUTING.md +++ b/docs/src/js/_media/CONTRIBUTING.md @@ -50,6 +50,19 @@ pnpm install pnpm build ``` +Local SQL computed columns and materialized-view creation/refresh are experimental +and disabled in published packages. To opt in when building from this directory: + +```shell +pnpm build:debug --features experimental-functions +pnpm tsc +pnpm test --runInBand table.test.ts materialized_view.test.ts +``` + +Rebuild without `--features experimental-functions` to test the default package. +The tests detect the compiled feature; remote operations, metadata protections, +and ordinary eager `addColumns` transforms remain available in both builds. + Lint: ```shell diff --git a/docs/src/js/classes/Connection.md b/docs/src/js/classes/Connection.md index c110eb38c..ffe7106b7 100644 --- a/docs/src/js/classes/Connection.md +++ b/docs/src/js/classes/Connection.md @@ -180,6 +180,10 @@ abstract createMaterializedView( Define a materialized view named `name` over the table `source`. +Local creation and refresh require a source build with the +`experimental-functions` Cargo feature, disabled in published packages. +Remote Cloud/Enterprise operations do not require this feature. + The view is populated before creation returns. Set `withNoData` to create only its definition and empty backing table. The view is a normal table: it can be queried, indexed and searched, and it appears in `tableNames`. diff --git a/docs/src/js/classes/MaterializedView.md b/docs/src/js/classes/MaterializedView.md index 0e2413328..bb02f5dc2 100644 --- a/docs/src/js/classes/MaterializedView.md +++ b/docs/src/js/classes/MaterializedView.md @@ -64,6 +64,7 @@ refresh(options?): Promise ``` Recompute the view from its source. +Local refresh requires a source build with `experimental-functions`. Every refresh rebuilds the complete view. For local views, `sourceVersion` refreshes to that source version instead of the latest; diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index 7cf87c405..80bba02a0 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -69,6 +69,10 @@ abstract addColumns(newColumnTransforms): Promise Add new columns with defined values. +Local computed-column declaration and refresh require a source build with +the `experimental-functions` Cargo feature. Published packages disable it; +remote Cloud/Enterprise operations remain available. + The `{ computed }` form stores the expression rather than evaluating it now: the column is committed with no values, and rows get them from [Table#refreshColumn](Table.md#refreshcolumn). Declaring one therefore costs the same on a @@ -1035,6 +1039,7 @@ abstract refreshColumn(column): Promise ``` Fill the rows of a computed column that hold no value yet. +Local refresh requires a source build with `experimental-functions`. Rows appended since the last refresh are filled by the next one, and rows whose inputs changed since they were computed are recomputed; @@ -1063,6 +1068,8 @@ abstract refreshColumnAsync(column): Promise Like [Table#refreshColumn](Table.md#refreshcolumn), but returns a handle to the refresh job instead of blocking until it completes. +Local refresh requires a source build with `experimental-functions`. +Remote Cloud/Enterprise jobs do not require this feature. The job may already be complete when returned; callers must not assume the column is filled until [Job.wait](Job.md#wait) resolves. Invalid input -- diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 2fefe91f3..5f87302ad 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -429,6 +429,13 @@ scoped to each invocation and support `read`, `readinto`, `seek`, `tell`, ## Materialized Views (Synchronous) +In the 0.41 SDK release, local materialized-view creation and refresh require a +source build with the `experimental-functions` Cargo feature. Published Python +packages leave it disabled. Opening, reading, listing and dropping existing views +remain available, as do remote Cloud and Enterprise operations. The same feature +requirement applies to local SQL computed-column declaration and refresh; ordinary +eager `add_columns` transforms remain available. + On Cloud and Enterprise, creation accepts `auto_refresh_enabled` and `auto_refresh_interval_seconds`: @@ -443,7 +450,8 @@ view = db.create_materialized_view( Omit `auto_refresh_interval_seconds` to let the server determine when to refresh. These options also apply to asynchronous creation and creation jobs. Embedded -connections support manual refresh and reject automatic scheduling requests. +connections built with `experimental-functions` support manual refresh and reject +automatic scheduling requests. ::: lancedb.materialized_view.MaterializedView diff --git a/docs/web/feature-engineering/add-computed-column.mdx b/docs/web/feature-engineering/add-computed-column.mdx index c287d42f5..707a929f5 100644 --- a/docs/web/feature-engineering/add-computed-column.mdx +++ b/docs/web/feature-engineering/add-computed-column.mdx @@ -8,13 +8,20 @@ availability: oss: available enterprise: available summary: >- - Both offerings support computed columns. Enterprise requires the deployment to enable them; the embedded library evaluates them in your process. + Local computed columns require a source build with the experimental-functions Cargo feature. Enterprise requires the deployment to enable them. --- A computed column is declared with an expression and starts with no values. Declaring it is cheap on any table size, because nothing is evaluated until [a refresh](/feature-engineering/refresh-column) asks for it. + +In the 0.41 SDK release, local computed-column declaration and refresh require a +source build with the `experimental-functions` Cargo feature. Published Python +and TypeScript packages leave it disabled. Remote Cloud and Enterprise operations +do not require this feature. Ordinary eager `add_columns` transforms remain available. + + ## Add a computed column {#declare} ```python diff --git a/docs/web/feature-engineering/materialized-views.mdx b/docs/web/feature-engineering/materialized-views.mdx index 1552c3638..30d2d1bc6 100644 --- a/docs/web/feature-engineering/materialized-views.mdx +++ b/docs/web/feature-engineering/materialized-views.mdx @@ -8,7 +8,7 @@ availability: oss: available enterprise: available summary: >- - Both offerings support materialized views. OSS refreshes run in your process; Enterprise runs the work on remote compute. See [Enterprise refreshes](#availability) for server compatibility requirements. + Local creation and refresh require a source build with the experimental-functions Cargo feature. Enterprise runs the work on remote compute. See [Enterprise refreshes](#availability) for server compatibility requirements. --- import { @@ -20,6 +20,13 @@ A materialized view stores the result of a query as a table. Use it to keep a filtered or transformed dataset that you can query, index and search repeatedly. Refresh the view after its source data changes. + +In the 0.41 SDK release, local materialized-view creation and refresh require a +source build with the `experimental-functions` Cargo feature. Published Python +and TypeScript packages leave it disabled. Opening, reading, listing and dropping +existing views remain available, as do remote Cloud and Enterprise operations. + + In SQL this is [`REFRESH MATERIALIZED VIEW`](/sql/refresh#refresh-materialized-view). diff --git a/docs/web/feature-engineering/refresh-column.mdx b/docs/web/feature-engineering/refresh-column.mdx index ce36daa32..1aa511234 100644 --- a/docs/web/feature-engineering/refresh-column.mdx +++ b/docs/web/feature-engineering/refresh-column.mdx @@ -8,13 +8,20 @@ availability: oss: available enterprise: available summary: >- - Both offerings support column refresh. The embedded library runs it in your process; Enterprise runs it as a server job that must be enabled for your Enterprise cluster. + Local column refresh requires a source build with the experimental-functions Cargo feature. Enterprise runs it as a server job that must be enabled for your Enterprise cluster. --- Refresh a [computed column](/feature-engineering/add-computed-column) to fill new rows and recompute values whose inputs changed. The example below continues with the `doubled` column from the computed-column guide. + +In the 0.41 SDK release, local declaration and refresh require a source build +with the `experimental-functions` Cargo feature. Published Python and TypeScript +packages leave it disabled. Remote Cloud and Enterprise operations do not require +this feature; existing computed-column data remains readable in default builds. + + ```python Python icon="python" table.refresh_column("doubled") diff --git a/nodejs/CONTRIBUTING.md b/nodejs/CONTRIBUTING.md index 2faa8cd37..2125cc8aa 100644 --- a/nodejs/CONTRIBUTING.md +++ b/nodejs/CONTRIBUTING.md @@ -50,6 +50,19 @@ pnpm install pnpm build ``` +Local SQL computed columns and materialized-view creation/refresh are experimental +and disabled in published packages. To opt in when building from this directory: + +```shell +pnpm build:debug --features experimental-functions +pnpm tsc +pnpm test --runInBand table.test.ts materialized_view.test.ts +``` + +Rebuild without `--features experimental-functions` to test the default package. +The tests detect the compiled feature; remote operations, metadata protections, +and ordinary eager `addColumns` transforms remain available in both builds. + Lint: ```shell diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index e8abcf60a..4c86916eb 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -48,3 +48,4 @@ napi-build = "2.3.1" default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/goosefs", "lancedb/metrics-otel"] fp16kernels = ["lancedb/fp16kernels"] remote = ["lancedb/remote"] +experimental-functions = ["lancedb/experimental-functions"] diff --git a/nodejs/__test__/experimental_functions.test.ts b/nodejs/__test__/experimental_functions.test.ts new file mode 100644 index 000000000..fdadb092f --- /dev/null +++ b/nodejs/__test__/experimental_functions.test.ts @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { connect } from "../lancedb"; +import { experimentalFunctionsEnabled } from "../lancedb/native"; + +const defaultBuildTest = experimentalFunctionsEnabled() ? it.skip : it; + +defaultBuildTest( + "rejects local Functions before mutating the database", + async () => { + const directory = await mkdtemp(join(tmpdir(), "lancedb-experimental-")); + try { + const db = await connect(directory); + const table = await db.createTable("source", [{ x: 1 }, { x: 2 }]); + const version = await table.version(); + const operations = [ + () => + table.addColumns({ + computed: [{ name: "doubled", valueSql: "x * 2" }], + }), + () => table.refreshColumn("x"), + () => table.refreshColumnAsync("x"), + () => db.createMaterializedView("blocked", "source"), + () => + db.createMaterializedView("blocked", "source", { withNoData: true }), + ]; + for (const operation of operations) { + await expect(operation()).rejects.toThrow("experimental-functions"); + } + expect(await table.version()).toBe(version); + expect(await db.tableNames()).toEqual(["source"]); + await table.addColumns([{ name: "eager", valueSql: "x * 2" }]); + expect((await table.query().toArray()).map((row) => row.eager)).toEqual([ + 2, 4, + ]); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }, +); diff --git a/nodejs/__test__/materialized_view.test.ts b/nodejs/__test__/materialized_view.test.ts index 3852af717..2324e1fcf 100644 --- a/nodejs/__test__/materialized_view.test.ts +++ b/nodejs/__test__/materialized_view.test.ts @@ -8,8 +8,10 @@ import { DEFINITION_META_KEY, definitionFromMetadata, } from "../lancedb/materialized_view"; +import { experimentalFunctionsEnabled } from "../lancedb/native"; describe("materialized views", () => { + const experimentalTest = experimentalFunctionsEnabled() ? it : it.skip; let tmpDir: tmp.DirResult; let db: Connection; @@ -69,7 +71,7 @@ describe("materialized views", () => { ); }); - it("creates, refreshes and queries a view", async () => { + experimentalTest("creates, refreshes and queries a view", async () => { const view = await db.createMaterializedView("adults", "people", { select: ["name", ["shout", "upper(name)"]], where: "age >= 18", @@ -81,7 +83,7 @@ describe("materialized views", () => { expect(rows.map((r) => r.shout).sort()).toEqual(["ADA", "GRACE"]); }); - it("round-trips the definition", async () => { + experimentalTest("round-trips the definition", async () => { await db.createMaterializedView("adults", "people", { where: "age >= 18", }); @@ -92,7 +94,7 @@ describe("materialized views", () => { ); }); - it("fully refreshes after an append", async () => { + experimentalTest("fully refreshes after an append", async () => { const view = await db.createMaterializedView("copy", "people", { withNoData: true, }); @@ -108,7 +110,7 @@ describe("materialized views", () => { expect((await view.refresh()).mode).toBe("rebuild"); }); - it("lists views and rejects non-views", async () => { + experimentalTest("lists views and rejects non-views", async () => { await db.createMaterializedView("adults", "people", { where: "age >= 18", }); @@ -124,17 +126,20 @@ describe("materialized views", () => { expect(await collect(db.listMaterializedViews())).toEqual([]); }); - it("returns a job when dropping a view asynchronously", async () => { - await db.createMaterializedView("adults", "people"); + experimentalTest( + "returns a job when dropping a view asynchronously", + async () => { + await db.createMaterializedView("adults", "people"); - const job = await db.dropMaterializedViewAsync("adults"); - expect(job.id).toBeNull(); - expect(job.jobType).toBeNull(); - await job.wait(); - expect(await collect(db.listMaterializedViews())).toEqual([]); - }); + const job = await db.dropMaterializedViewAsync("adults"); + expect(job.id).toBeNull(); + expect(job.jobType).toBeNull(); + await job.wait(); + expect(await collect(db.listMaterializedViews())).toEqual([]); + }, + ); - it("rejects an invalid expression at create time", async () => { + experimentalTest("rejects an invalid expression at create time", async () => { await expect( db.createMaterializedView("bad", "people", { select: [["x", "missing + 1"]], @@ -149,16 +154,21 @@ describe("materialized views", () => { ).rejects.toThrow("non-negative integer"); } expect(await collect(db.listMaterializedViews())).toEqual([]); - - const view = await db.createMaterializedView("copy", "people"); - for (const sourceVersion of [-1, 1.5, Infinity, NaN]) { - await expect(view.refresh({ sourceVersion })).rejects.toThrow( - "non-negative integer", - ); - } }); - it("quotes bare select names", async () => { + experimentalTest( + "rejects invalid source versions before refreshing", + async () => { + const view = await db.createMaterializedView("copy", "people"); + for (const sourceVersion of [-1, 1.5, Infinity, NaN]) { + await expect(view.refresh({ sourceVersion })).rejects.toThrow( + "non-negative integer", + ); + } + }, + ); + + experimentalTest("quotes bare select names", async () => { await db.createTable("odd_names", [{ "order item": "widget" }]); const view = await db.createMaterializedView("quoted", "odd_names", { select: ["order item"], @@ -168,11 +178,14 @@ describe("materialized views", () => { expect(Number(result.rowsWritten)).toBe(1); }); - it("does not require stable row ids on the source", async () => { - await db.createTable("plain", [{ x: 1 }]); - const view = await db.createMaterializedView("v", "plain"); - expect(await view.table().countRows()).toBe(1); - }); + experimentalTest( + "does not require stable row ids on the source", + async () => { + await db.createTable("plain", [{ x: 1 }]); + const view = await db.createMaterializedView("v", "plain"); + expect(await view.table().countRows()).toBe(1); + }, + ); }); async function collect(items: AsyncIterable): Promise { diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index d848676cd..c9e1efcb6 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -47,6 +47,7 @@ import { register, } from "../lancedb/embedding"; import { Index } from "../lancedb/indices"; +import { experimentalFunctionsEnabled } from "../lancedb/native"; import { BooleanQuery, Occur, @@ -5477,13 +5478,14 @@ describe("LSM convergence and stats", () => { }); describe("computed columns", () => { + const experimentalTest = experimentalFunctionsEnabled() ? it : it.skip; let tmpDir: tmp.DirResult; beforeEach(() => { tmpDir = tmp.dirSync({ unsafeCleanup: true }); }); afterEach(() => tmpDir.removeCallback()); - it("declares a column and fills it on refresh", async () => { + experimentalTest("declares a column and fills it on refresh", async () => { const db = await connect(tmpDir.name); const table = await db.createTable("computed", [{ x: 1 }, { x: 2 }]); @@ -5508,7 +5510,7 @@ describe("computed columns", () => { ); }); - it("returns a job handle from refreshColumnAsync", async () => { + experimentalTest("returns a job handle from refreshColumnAsync", async () => { const db = await connect(tmpDir.name); const table = await db.createTable("computed_job", [{ x: 1 }, { x: 2 }]); @@ -5530,7 +5532,7 @@ describe("computed columns", () => { ); }); - it("fills rows added since the last refresh", async () => { + experimentalTest("fills rows added since the last refresh", async () => { const db = await connect(tmpDir.name); const table = await db.createTable("computed_append", [{ x: 1 }]); diff --git a/nodejs/lancedb/connection.ts b/nodejs/lancedb/connection.ts index 5f067eeef..80fe2c722 100644 --- a/nodejs/lancedb/connection.ts +++ b/nodejs/lancedb/connection.ts @@ -270,6 +270,10 @@ export abstract class Connection { /** * Define a materialized view named `name` over the table `source`. * + * Local creation and refresh require a source build with the + * `experimental-functions` Cargo feature, disabled in published packages. + * Remote Cloud/Enterprise operations do not require this feature. + * * The view is populated before creation returns. Set `withNoData` to create * only its definition and empty backing table. The view is a normal table: * it can be queried, indexed and searched, and it appears in `tableNames`. diff --git a/nodejs/lancedb/materialized_view.ts b/nodejs/lancedb/materialized_view.ts index 7cf1690e0..cc209b14d 100644 --- a/nodejs/lancedb/materialized_view.ts +++ b/nodejs/lancedb/materialized_view.ts @@ -186,6 +186,7 @@ export class MaterializedView { /** * Recompute the view from its source. + * Local refresh requires a source build with `experimental-functions`. * * Every refresh rebuilds the complete view. For local views, * `sourceVersion` refreshes to that source version instead of the latest; diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 4ae1aefb8..4738b6140 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -594,6 +594,10 @@ export abstract class Table { /** * Add new columns with defined values. * + * Local computed-column declaration and refresh require a source build with + * the `experimental-functions` Cargo feature. Published packages disable it; + * remote Cloud/Enterprise operations remain available. + * * The `{ computed }` form stores the expression rather than evaluating it * now: the column is committed with no values, and rows get them from * {@link Table#refreshColumn}. Declaring one therefore costs the same on a @@ -632,6 +636,7 @@ export abstract class Table { /** * Fill the rows of a computed column that hold no value yet. + * Local refresh requires a source build with `experimental-functions`. * * Rows appended since the last refresh are filled by the next one, and * rows whose inputs changed since they were computed are recomputed; @@ -646,6 +651,8 @@ export abstract class Table { /** * Like {@link Table#refreshColumn}, but returns a handle to the refresh * job instead of blocking until it completes. + * Local refresh requires a source build with `experimental-functions`. + * Remote Cloud/Enterprise jobs do not require this feature. * * The job may already be complete when returned; callers must not assume * the column is filled until {@link Job.wait} resolves. Invalid input -- diff --git a/nodejs/src/lib.rs b/nodejs/src/lib.rs index f47bc113e..a92b8db98 100644 --- a/nodejs/src/lib.rs +++ b/nodejs/src/lib.rs @@ -30,6 +30,12 @@ mod session; mod table; mod util; +/// Whether this native binding was built with local experimental Functions. +#[napi] +pub fn experimental_functions_enabled() -> bool { + cfg!(feature = "experimental-functions") +} + #[napi(object)] #[derive(Debug)] pub struct ConnectionOptions { diff --git a/python/CONTRIBUTING.md b/python/CONTRIBUTING.md index 122677364..f86dea706 100644 --- a/python/CONTRIBUTING.md +++ b/python/CONTRIBUTING.md @@ -48,6 +48,18 @@ Build the package make develop ``` +Local SQL computed columns and materialized-view creation/refresh are experimental +and disabled in published packages. To opt in when building from this directory: + +```shell +uv run --extra tests --extra dev maturin develop --features experimental-functions --extras tests,dev +uv run --no-sync pytest python/tests -m experimental_functions +``` + +Rebuild without `--features experimental-functions` to test the default package. +The tests detect the compiled feature; remote operations, metadata protections, +and ordinary eager `add_columns` transforms remain available in both builds. + Format: ```shell diff --git a/python/Cargo.toml b/python/Cargo.toml index b1581b366..f84ab80a3 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -52,3 +52,4 @@ pyo3-build-config = { version = "0.28", features = ["abi3-py310"] } default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/cos", "lancedb/goosefs", "lancedb/metrics-otel"] fp16kernels = ["lancedb/fp16kernels"] remote = ["lancedb/remote"] +experimental-functions = ["lancedb/experimental-functions"] diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 1f3cfa9ed..333fca92e 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -26,6 +26,8 @@ from lance_namespace import ( from .remote import ClientConfig from .types import BaseTokenizerType +_experimental_functions_enabled: bool + IvfHnswPq: type[HnswPq] = HnswPq IvfHnswSq: type[HnswSq] = HnswSq IvfHnswFlat: type[HnswFlat] = HnswFlat diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index b6dfe8fb8..f38608188 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -626,6 +626,10 @@ class DBConnection(EnforceOverrides): ) -> MaterializedView: """Define a materialized view named ``name`` over the table ``source``. + Local creation and refresh require a source build with the + ``experimental-functions`` Cargo feature. Published packages leave it + disabled; remote Cloud/Enterprise operations remain available. + The view is populated before creation returns. Pass ``with_no_data=True`` to create only its definition and empty backing table. The view is a normal table: it can be queried, indexed and @@ -1760,6 +1764,10 @@ class LanceDBConnection(DBConnection): auto_refresh_interval_seconds: Optional[float] = None, ) -> MaterializedView: """Define a materialized view named ``name`` over the table ``source``. + + Local creation and refresh require a source build with the + ``experimental-functions`` Cargo feature. Published packages leave it + disabled; remote Cloud/Enterprise operations remain available. See [DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view]. @@ -1769,13 +1777,13 @@ class LanceDBConnection(DBConnection): >>> db = lancedb.connect("./.lancedb") >>> data = [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}] >>> table = db.create_table("people", data) - >>> view = db.create_materialized_view( + >>> view = db.create_materialized_view( # doctest: +SKIP ... "adults", ... "people", ... select=["name", ("shout", "upper(name)")], ... where="age >= 18", ... ) - >>> view.table.count_rows() + >>> view.table.count_rows() # doctest: +SKIP 1 """ LOOP.run( @@ -2846,6 +2854,10 @@ class AsyncConnection(object): auto_refresh_interval_seconds: Optional[float] = None, ) -> AsyncMaterializedView: """Define a materialized view named ``name`` over the table ``source``. + + Local creation and refresh require a source build with the + ``experimental-functions`` Cargo feature. Published packages leave it + disabled; remote Cloud/Enterprise operations remain available. See [DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view]. """ diff --git a/python/python/lancedb/materialized_view.py b/python/python/lancedb/materialized_view.py index 6342641ff..8efe31f05 100644 --- a/python/python/lancedb/materialized_view.py +++ b/python/python/lancedb/materialized_view.py @@ -2,7 +2,12 @@ # SPDX-FileCopyrightText: Copyright The LanceDB Authors """Materialized views: tables defined by a query over a source table and -maintained by refresh. See ``DBConnection.create_materialized_view``.""" +maintained by refresh. See ``DBConnection.create_materialized_view``. + +Local creation and refresh require the ``experimental-functions`` Cargo feature +in a source build. Published packages disable it. Opening, reading, listing and +dropping existing views and remote Cloud/Enterprise operations remain available. +""" from __future__ import annotations diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 048c46a6a..6df109d6c 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -2988,6 +2988,10 @@ class Table(ABC): Function columns are supported only on LanceDB Cloud and Enterprise. computed: Dict[str, str], optional + Local declaration and refresh require a source build with the + ``experimental-functions`` Cargo feature, which published packages + do not enable. Remote Cloud/Enterprise operations remain available. + A mapping from output column names to SQL expressions derives each output field from its expression. A direct projection of a Blob v2 field inherits Blob v2 semantics; other expressions derive their @@ -3029,11 +3033,11 @@ class Table(ABC): >>> import lancedb >>> db = lancedb.connect("./.lancedb") >>> table = db.create_table("computed_demo", [{"x": 1}, {"x": 2}]) - >>> table.add_columns(computed={"doubled": "x * 2"}) + >>> table.add_columns(computed={"doubled": "x * 2"}) # doctest: +SKIP AddColumnsResult(version=2) - >>> table.refresh_column("doubled") + >>> table.refresh_column("doubled") # doctest: +SKIP RefreshColumnResult(rows_filled=2, version=4) - >>> table.to_arrow().sort_by("x").to_pandas() + >>> table.to_arrow().sort_by("x").to_pandas() # doctest: +SKIP x doubled 0 1 2 1 2 4 @@ -3044,6 +3048,9 @@ class Table(ABC): """ Fill the rows of a computed column that hold no value yet. + Local refresh requires the ``experimental-functions`` Cargo feature + in a source build. Published packages leave it disabled. + Declared with ``add_columns(computed=...)``, a column starts empty and gets its values here. Rows appended since the last refresh are filled by the next one, and rows whose inputs changed since they were computed @@ -3070,6 +3077,9 @@ class Table(ABC): Like :meth:`refresh_column`, but returns a handle to the refresh job instead of blocking until it completes. + Local refresh requires a source build with ``experimental-functions``. + Remote Cloud/Enterprise jobs do not require this feature. + The job may already be complete when returned; callers must not assume the column is filled until :meth:`Job.wait` returns. Invalid input -- an unknown column, or one that is not computed -- raises here rather @@ -3087,13 +3097,13 @@ class Table(ABC): >>> import lancedb >>> db = lancedb.connect("./.lancedb") >>> table = db.create_table("computed_job_demo", [{"x": 1}, {"x": 2}]) - >>> table.add_columns(computed={"doubled": "x * 2"}) + >>> table.add_columns(computed={"doubled": "x * 2"}) # doctest: +SKIP AddColumnsResult(version=2) - >>> job = table.refresh_column_async("doubled") - >>> result = job.wait() - >>> result.rows_assigned + >>> job = table.refresh_column_async("doubled") # doctest: +SKIP + >>> result = job.wait() # doctest: +SKIP + >>> result.rows_assigned # doctest: +SKIP 2 - >>> job.status() + >>> job.status() # doctest: +SKIP 'finished' """ @@ -7195,6 +7205,10 @@ class AsyncTable: Function columns are supported only on LanceDB Cloud and Enterprise. computed: Dict[str, str], optional + Local declaration and refresh require a source build with the + ``experimental-functions`` Cargo feature, which published packages + do not enable. Remote Cloud/Enterprise operations remain available. + A mapping from output column names to SQL expressions derives each output field from its expression. A direct projection of a Blob v2 field inherits Blob v2 semantics; other expressions derive their @@ -7314,6 +7328,9 @@ class AsyncTable: """ Fill the rows of a computed column that hold no value yet. + Local refresh requires the ``experimental-functions`` Cargo feature + in a source build. Published packages leave it disabled. + Declared with ``add_columns(computed=...)``, a column starts empty and gets its values here. Rows appended since the last refresh are filled by the next one, and rows whose inputs changed since they were computed @@ -7384,6 +7401,9 @@ class AsyncTable: Like :meth:`refresh_column`, but returns a handle to the refresh job instead of blocking until it completes. + Local refresh requires a source build with ``experimental-functions``. + Remote Cloud/Enterprise jobs do not require this feature. + The job may already be complete when returned; callers must not assume the column is filled until :meth:`AsyncJob.wait` resolves. Invalid input -- an unknown column, or one that is not computed -- raises here @@ -7409,7 +7429,7 @@ class AsyncTable: ... result = await job.wait() ... assert result.rows_assigned == 1 ... return await job.status() - >>> asyncio.run(refresh_in_background()) + >>> asyncio.run(refresh_in_background()) # doctest: +SKIP 'finished' """ return _typed_job( diff --git a/python/python/tests/conftest.py b/python/python/tests/conftest.py index 8567a9830..86c4f00bb 100644 --- a/python/python/tests/conftest.py +++ b/python/python/tests/conftest.py @@ -9,6 +9,21 @@ import pytest import pytest_asyncio +def pytest_configure(config): + config.addinivalue_line( + "markers", "experimental_functions: requires the experimental-functions build" + ) + + +def pytest_collection_modifyitems(items): + if lancedb._lancedb._experimental_functions_enabled: + return + skip = pytest.mark.skip(reason="requires the experimental-functions Cargo feature") + for item in items: + if item.get_closest_marker("experimental_functions"): + item.add_marker(skip) + + def pandas_string_type(): """Return the PyArrow string type that pandas uses for string columns. diff --git a/python/python/tests/test_experimental_functions.py b/python/python/tests/test_experimental_functions.py new file mode 100644 index 000000000..81de84606 --- /dev/null +++ b/python/python/tests/test_experimental_functions.py @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import lancedb +import pytest + +pytestmark = pytest.mark.skipif( + lancedb._lancedb._experimental_functions_enabled, + reason="tests the default build without experimental-functions", +) + + +@pytest.mark.parametrize("namespace", [False, True]) +def test_default_build_rejects_local_functions(tmp_path, namespace): + db = ( + lancedb.connect_namespace("dir", {"root": str(tmp_path)}) + if namespace + else lancedb.connect(tmp_path) + ) + table = db.create_table("source", [{"x": 1}, {"x": 2}]) + version = table.version + operations = [ + lambda: table.add_columns(computed={"doubled": "x * 2"}), + lambda: table.refresh_column("x"), + lambda: table.refresh_column_async("x"), + lambda: db.create_materialized_view("blocked", "source"), + lambda: db.create_materialized_view("blocked", "source", with_no_data=True), + lambda: db.create_materialized_view_async("blocked", "source"), + ] + for operation in operations: + with pytest.raises(NotImplementedError, match="experimental-functions"): + operation() + assert table.version == version + assert db.table_names() == ["source"] + table.add_columns({"eager": "x * 2"}) + assert table.to_arrow()["eager"].to_pylist() == [2, 4] + + +@pytest.mark.asyncio +async def test_async_default_build_rejects_local_functions(tmp_path): + db = await lancedb.connect_async(tmp_path) + table = await db.create_table("source", [{"x": 1}, {"x": 2}]) + version = await table.version() + operations = [ + lambda: table.add_columns(computed={"doubled": "x * 2"}), + lambda: table.refresh_column("x"), + lambda: table.refresh_column_async("x"), + lambda: db.create_materialized_view("blocked", "source"), + lambda: db.create_materialized_view("blocked", "source", with_no_data=True), + lambda: db.create_materialized_view_async("blocked", "source"), + ] + for operation in operations: + with pytest.raises(NotImplementedError, match="experimental-functions"): + await operation() + assert await table.version() == version + assert await db.table_names() == ["source"] + await table.add_columns({"eager": "x * 2"}) + assert (await table.query().to_arrow())["eager"].to_pylist() == [2, 4] diff --git a/python/python/tests/test_materialized_views.py b/python/python/tests/test_materialized_views.py index e2be9c0d4..337fcad78 100644 --- a/python/python/tests/test_materialized_views.py +++ b/python/python/tests/test_materialized_views.py @@ -173,6 +173,7 @@ def test_sync_remote_create_uses_public_async_connection(): assert calls[-1] == ("drop", "adults", ["analytics"]) +@pytest.mark.experimental_functions def test_create_refresh_and_query(tmp_path): db = make_db(tmp_path) view = db.create_materialized_view( @@ -188,6 +189,7 @@ def test_create_refresh_and_query(tmp_path): assert sorted(row["shout"] for row in rows) == ["ADA", "GRACE"] +@pytest.mark.experimental_functions def test_create_and_refresh_jobs(tmp_path): db = make_db(tmp_path) create_job = db.create_materialized_view_async( @@ -212,6 +214,7 @@ def test_create_and_refresh_jobs(tmp_path): assert "adults" not in list(db.list_materialized_views()) +@pytest.mark.experimental_functions def test_definition_round_trips(tmp_path): db = make_db(tmp_path) db.create_materialized_view("adults", "people", where="age >= 18") @@ -222,6 +225,7 @@ def test_definition_round_trips(tmp_path): ) +@pytest.mark.experimental_functions def test_full_refresh_after_append(tmp_path): db = make_db(tmp_path) view = db.create_materialized_view("copy", "people", with_no_data=True) @@ -236,6 +240,7 @@ def test_full_refresh_after_append(tmp_path): assert view.refresh().mode == "rebuild" +@pytest.mark.experimental_functions def test_full_refresh_after_update(tmp_path): db = make_db(tmp_path) view = db.create_materialized_view("copy", "people", with_no_data=True) @@ -249,6 +254,7 @@ def test_full_refresh_after_update(tmp_path): assert sorted(row["age"] for row in rows) == [8, 36, 85] +@pytest.mark.experimental_functions def test_legacy_storage_source_update_rebuilds(tmp_path): db = lancedb.connect( tmp_path, @@ -265,6 +271,7 @@ def test_legacy_storage_source_update_rebuilds(tmp_path): assert sorted(row["age"] for row in rows) == [8, 36] +@pytest.mark.experimental_functions def test_direct_writes_to_a_view_are_refused(tmp_path): db = make_db(tmp_path) view = db.create_materialized_view("copy", "people") @@ -290,6 +297,7 @@ def test_direct_writes_to_a_view_are_refused(tmp_path): assert view.refresh().rows_written == 4 +@pytest.mark.experimental_functions def test_list_and_not_a_view(tmp_path): db = make_db(tmp_path) db.create_materialized_view("adults", "people", where="age >= 18") @@ -304,6 +312,7 @@ def test_list_and_not_a_view(tmp_path): assert list(db.list_materialized_views()) == [] +@pytest.mark.experimental_functions def test_invalid_expression_fails_at_create(tmp_path): db = make_db(tmp_path) with pytest.raises(Exception, match="missing"): @@ -311,6 +320,7 @@ def test_invalid_expression_fails_at_create(tmp_path): assert "bad" not in db.list_tables().tables +@pytest.mark.experimental_functions @pytest.mark.asyncio async def test_async_create_refresh_and_open(tmp_path): db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS) @@ -332,6 +342,7 @@ async def test_async_create_refresh_and_open(tmp_path): assert [item async for item in db.list_materialized_views()] == ["shouts"] +@pytest.mark.experimental_functions @pytest.mark.asyncio async def test_async_create_and_refresh_jobs(tmp_path): db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS) @@ -358,6 +369,7 @@ async def test_async_create_and_refresh_jobs(tmp_path): assert "adults" not in [item async for item in db.list_materialized_views()] +@pytest.mark.experimental_functions @pytest.mark.asyncio async def test_async_refresh_rebuilds(tmp_path): db = await lancedb.connect_async(tmp_path) @@ -372,6 +384,7 @@ async def test_async_refresh_rebuilds(tmp_path): assert result.rows_written == 2 +@pytest.mark.experimental_functions def test_source_does_not_require_stable_row_ids(tmp_path): db = lancedb.connect(tmp_path) db.create_table("plain", [{"x": 1}]) @@ -379,6 +392,7 @@ def test_source_does_not_require_stable_row_ids(tmp_path): assert view.table.count_rows() == 1 +@pytest.mark.experimental_functions def test_bare_select_names_are_quoted(tmp_path): db = lancedb.connect(tmp_path, storage_options=STABLE_ROW_IDS) db.create_table("odd_names", [{"order item": "widget", "select": 2}]) @@ -396,6 +410,7 @@ def test_bare_select_names_are_quoted(tmp_path): assert rows[0]["select"] == 2 +@pytest.mark.experimental_functions def test_scalar_select_is_one_column(tmp_path): db = make_db(tmp_path) view = db.create_materialized_view("just_name", "people", select="name") @@ -405,6 +420,7 @@ def test_scalar_select_is_one_column(tmp_path): assert sorted(row["name"] for row in rows) == ["ada", "grace", "kid"] +@pytest.mark.experimental_functions @pytest.mark.asyncio async def test_async_scalar_select_is_one_column(tmp_path): db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS) @@ -415,6 +431,7 @@ async def test_async_scalar_select_is_one_column(tmp_path): assert set(rows[0]) == {"name"} +@pytest.mark.experimental_functions def test_limit_above_i64_max_is_refused(tmp_path): db = make_db(tmp_path) with pytest.raises(ValueError, match="exceeds the maximum"): @@ -434,6 +451,7 @@ def _namespace_db(tmp_path): ) +@pytest.mark.experimental_functions def test_namespace_connection_materialized_views(tmp_path): db = _namespace_db(tmp_path) db.create_table( @@ -464,6 +482,7 @@ def test_namespace_connection_materialized_views(tmp_path): assert list(db.list_materialized_views()) == [] +@pytest.mark.experimental_functions @pytest.mark.asyncio async def test_async_namespace_connection_materialized_views(tmp_path): db = lancedb.connect_namespace_async( @@ -557,6 +576,7 @@ def test_stored_queries_and_legacy_layouts_are_read(): read(newer) +@pytest.mark.experimental_functions def test_local_materialized_view_listing_pages_and_resume(tmp_path): db = make_db(tmp_path) db.create_materialized_view("a_view", "people", with_no_data=True) @@ -646,6 +666,7 @@ async def test_async_remote_create_keeps_interval_unspecified(): assert native.calls[0][2]["auto_refresh_interval_seconds"] is None +@pytest.mark.experimental_functions @pytest.mark.parametrize( "options", [ @@ -673,12 +694,14 @@ def test_remote_create_rejects_invalid_interval_without_request(seconds): assert requests == [] +@pytest.mark.experimental_functions def test_namespace_forwards_automatic_refresh_request(tmp_path): db = lancedb.connect_namespace("dir", {"root": str(tmp_path)}) with pytest.raises(NotImplementedError, match="Cloud or Enterprise"): db.create_materialized_view("mv", "source", auto_refresh_enabled=True) +@pytest.mark.experimental_functions @pytest.mark.asyncio async def test_async_namespace_forwards_automatic_refresh_request(tmp_path): db = lancedb.connect_namespace_async("dir", {"root": str(tmp_path)}) @@ -686,6 +709,7 @@ async def test_async_namespace_forwards_automatic_refresh_request(tmp_path): await db.create_materialized_view("mv", "source", auto_refresh_enabled=True) +@pytest.mark.experimental_functions def test_embedded_explicitly_disabled_refresh_is_supported(tmp_path): db = make_db(tmp_path) view = db.create_materialized_view("mv", "people", auto_refresh_enabled=False) diff --git a/python/python/tests/test_namespace.py b/python/python/tests/test_namespace.py index b55da4d57..c70ba3d8e 100644 --- a/python/python/tests/test_namespace.py +++ b/python/python/tests/test_namespace.py @@ -16,6 +16,7 @@ import pyarrow as pa import lancedb from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError from lancedb.listing import AsyncListing, Listing +from lancedb.materialized_view import DEFINITION_FORMAT, DEFINITION_META_KEY from lancedb.namespace import _MAX_QUERY_K, _query_to_namespace_request from lancedb.query import Query from lancedb.table import AsyncTable, LanceTable @@ -359,7 +360,17 @@ class TestNamespaceConnection: def test_list_materialized_views(self): db = lancedb.connect_namespace("dir", {"root": self.temp_dir}) db.create_table("source", [{"id": 1}]) - db.create_materialized_view("view", "source", with_no_data=True) + # Model a stored view from an enabled build: listing must also work + # when local creation is disabled in the current build. + schema = pa.schema( + [("id", pa.int64())], + metadata={ + DEFINITION_META_KEY: json.dumps( + {"format": DEFINITION_FORMAT, "query": "SELECT id FROM source"} + ).encode() + }, + ) + db.create_table("view", schema=schema) names = db.list_materialized_views(page_limit=1) assert isinstance(names, Listing) @@ -872,7 +883,16 @@ class TestAsyncNamespaceConnection: async def test_list_materialized_views(self): db = lancedb.connect_namespace_async("dir", {"root": self.temp_dir}) await db.create_table("source", [{"id": 1}]) - await db.create_materialized_view("view", "source", with_no_data=True) + # Model a stored view independently of the local creation feature. + schema = pa.schema( + [("id", pa.int64())], + metadata={ + DEFINITION_META_KEY: json.dumps( + {"format": DEFINITION_FORMAT, "query": "SELECT id FROM source"} + ).encode() + }, + ) + await db.create_table("view", schema=schema) names = db.list_materialized_views(page_limit=1) assert isinstance(names, AsyncListing) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 458f419ae..21fce93d3 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -5489,6 +5489,7 @@ async def test_async_search_runs_embedding_on_dedicated_executor( ) +@pytest.mark.experimental_functions def test_computed_column_declare_and_refresh(tmp_path): db = lancedb.connect(tmp_path) table = db.create_table("computed", [{"x": 1}, {"x": 2}]) @@ -5512,6 +5513,7 @@ def test_computed_column_rejects_transforms_and_computed_together(tmp_path): table.add_columns({"a": "x + 1"}, computed={"b": "x * 2"}) +@pytest.mark.experimental_functions def test_computed_column_blob_projection_inherits_semantics(tmp_path): schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) db = lancedb.connect(tmp_path) @@ -5535,6 +5537,7 @@ def test_computed_column_blob_projection_inherits_semantics(tmp_path): assert copied.to_pylist() == [b"hello", b"", None] +@pytest.mark.experimental_functions @pytest.mark.asyncio async def test_computed_column_async(tmp_path): db = await lancedb.connect_async(tmp_path) @@ -5553,6 +5556,7 @@ def test_function_errors_are_remote_only(tmp_path): table.function_errors() +@pytest.mark.experimental_functions def test_refresh_column_async_returns_job(tmp_path): db = lancedb.connect(tmp_path) table = db.create_table("computed_job", [{"x": 1}, {"x": 2}]) @@ -5581,6 +5585,7 @@ def test_refresh_column_async_returns_job(tmp_path): table.refresh_column_async("x") +@pytest.mark.experimental_functions @pytest.mark.asyncio async def test_refresh_column_async_job_async_table(tmp_path): db = await lancedb.connect_async(tmp_path) @@ -5597,6 +5602,7 @@ async def test_refresh_column_async_job_async_table(tmp_path): assert (await table.to_arrow())["tripled"].to_pylist() == [9] +@pytest.mark.experimental_functions @pytest.mark.parametrize("interval", [None, 300]) def test_computed_auto_refresh_options(tmp_path, interval): db = lancedb.connect(tmp_path) @@ -5635,6 +5641,7 @@ def test_auto_refresh_options_require_computed_columns(tmp_path): table.add_columns({"doubled": "x * 2"}, auto_refresh_enabled=True) +@pytest.mark.experimental_functions @pytest.mark.asyncio async def test_async_computed_auto_refresh_uses_server_interval(tmp_path): db = await lancedb.connect_async(tmp_path) diff --git a/python/src/lib.rs b/python/src/lib.rs index 3814f2caf..71662add1 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -132,6 +132,10 @@ pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(expr_lit, m)?)?; m.add_function(wrap_pyfunction!(expr_func, m)?)?; m.add("__version__", env!("CARGO_PKG_VERSION"))?; + m.add( + "_experimental_functions_enabled", + cfg!(feature = "experimental-functions"), + )?; // Give the shared runtime a coordinated, bounded shutdown at normal // process exit -- see `shutdown_runtime` and `runtime::shutdown` for why. py.import("atexit")? diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 9015bcdd7..62029bb92 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -137,6 +137,9 @@ pprof = { version = "0.14", features = ["flamegraph"] } [features] default = [] +# Local SQL computed columns and materialized-view creation/refresh. +# Remote clients and shared server primitives do not require this feature. +experimental-functions = [] # Native spatial indexes and queries. Remote RTree requests do not require this feature. geo = ["lance/geo"] aws = [ diff --git a/rust/lancedb/src/lib.rs b/rust/lancedb/src/lib.rs index 68521ff2c..8b1ad97c3 100644 --- a/rust/lancedb/src/lib.rs +++ b/rust/lancedb/src/lib.rs @@ -31,6 +31,11 @@ //! - `gcs` - Enable Google Cloud Storage object store support. //! - `oss` - Enable Alibaba Cloud OSS object store support. //! - `remote` - Enable remote client to connect to LanceDB cloud. +//! - `experimental-functions` - Opt in to local SQL computed-column declarations +//! and refresh, and local materialized-view creation and refresh. Disabled by +//! default, including in published Python and TypeScript packages. Remote +//! Cloud/Enterprise operations, reading existing data, metadata protections, +//! and ordinary eager `add_columns` transforms do not require this feature. //! - `huggingface` - Enable HuggingFace Hub integration for loading datasets from the Hub. //! - `fp16kernels` - Enable FP16 kernels for faster vector search on CPU. //! - `metrics` - Publish LanceDB's internal metrics through the @@ -229,6 +234,19 @@ pub use materialized_view::{ pub use metrics; pub use table::{FtsToken, Table}; +/// Gate local execution without hiding client types or metadata protections. +pub(crate) fn require_experimental_functions() -> Result<()> { + if !cfg!(feature = "experimental-functions") { + return Err(Error::NotSupported { + message: "local computed columns and materialized-view creation/refresh require \ + building LanceDB with the 'experimental-functions' Cargo feature; \ + official Python and TypeScript packages do not enable it" + .into(), + }); + } + Ok(()) +} + /// Tokenize a full-text search query using an explicit FTS tokenizer configuration. /// /// This does not require a table or FTS index. The tokenizer options are the diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs index 7ee76a160..b7124d930 100644 --- a/rust/lancedb/src/materialized_view.rs +++ b/rust/lancedb/src/materialized_view.rs @@ -9,6 +9,11 @@ //! query this version cannot maintain reads back as unrefreshable, not as a //! plain table. Queries, indexes and search work on the view unchanged; //! writes to its rows or stored columns are refused. +//! +//! Local creation and refresh require the `experimental-functions` Cargo +//! feature. Opening, reading, listing and dropping existing views, remote +//! client operations, and shared server commit primitives remain available +//! without it. mod grouped; pub use grouped::IVF_PARTITION; @@ -1245,6 +1250,7 @@ impl PreparedDeclaration { namespace_path: &[String], name: &str, ) -> Result { + crate::require_experimental_functions()?; ensure_no_source_cycle(&self.database, namespace_path, name, &self.definition).await?; let empty: Vec> = vec![]; @@ -1733,6 +1739,7 @@ impl CreateMaterializedViewBuilder { }) .await; } + crate::require_experimental_functions()?; Ok(Job::spawned(tokio::spawn(async move { self.execute_native().await.map(|_| ()) }))) @@ -1756,6 +1763,7 @@ impl CreateMaterializedViewBuilder { } async fn execute_native(self) -> Result { + crate::require_experimental_functions()?; if self.auto_refresh_enabled == Some(true) || self.auto_refresh_interval_seconds.is_some() { return Err(Error::NotSupported { message: "automatic materialized-view refresh requires Cloud or Enterprise" @@ -1879,6 +1887,7 @@ impl RefreshMaterializedViewBuilder { .refresh_materialized_view_async(self.source_version) .await; } + crate::require_experimental_functions()?; Ok(Job::spawned(tokio::spawn(async move { refresh::execute_refresh(&self.view.table, self.source_version).await }))) @@ -2061,6 +2070,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn refresh_always_rebuilds_without_source_row_ids() { let conn = connect("memory://").execute().await.unwrap(); let source = conn @@ -2124,6 +2137,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn a_view_cannot_lead_back_to_itself() { let conn = connect("memory://").execute().await.unwrap(); let rows = || record_batch!(("x", Int32, [1, 2])).unwrap(); @@ -2159,6 +2176,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn direct_writes_to_a_view_are_refused() { use arrow_schema::Schema; use datafusion::prelude::SessionContext; @@ -2295,6 +2316,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn indexes_and_optimize_still_apply_to_a_view() { use crate::index::Index; use crate::table::{CompactionOptions, OptimizeAction}; @@ -2339,6 +2364,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn computed_columns_of_a_view_are_still_filled_and_dropped() { use crate::table::computed_columns::{ COMPUTED_COLUMN_META_KEY, EXPRESSION_META_KEY, INPUTS_META_KEY, KIND_META_KEY, SQL_KIND, diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs index 6b4bb6d8a..1186ac576 100644 --- a/rust/lancedb/src/materialized_view/refresh.rs +++ b/rust/lancedb/src/materialized_view/refresh.rs @@ -72,6 +72,7 @@ pub(crate) async fn execute_refresh( view: &Table, pinned: Option, ) -> Result { + crate::require_experimental_functions()?; let native = view.as_native().ok_or_else(|| Error::NotSupported { message: "materialized views are supported only on local tables".into(), })?; @@ -576,6 +577,10 @@ mod tests { use crate::connect; #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn raced_first_full_refresh_lands_one_result() { let directory = tempfile::tempdir().unwrap(); let connection = connect(directory.path().to_str().unwrap()) diff --git a/rust/lancedb/src/table/add_columns.rs b/rust/lancedb/src/table/add_columns.rs index 4f9cbb772..0f13718fd 100644 --- a/rust/lancedb/src/table/add_columns.rs +++ b/rust/lancedb/src/table/add_columns.rs @@ -69,6 +69,9 @@ impl AddColumnsBuilder { /// renamed, retyped or dropped while a declaration reads it, since the /// expression names it. /// + /// Local declaration and refresh require the `experimental-functions` + /// Cargo feature. Remote operations do not require this feature. + /// /// On LanceDB Cloud and Enterprise the expression is planned by the /// server, and the refresh runs as a server job -- see /// [`Table::refresh_column_async`](super::Table::refresh_column_async). @@ -252,6 +255,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn computed_options_commit_with_declaration() { let table = table_with_two_columns("options").await; let version = table.version().await.unwrap(); @@ -313,6 +320,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn omitted_refresh_interval_stays_unset() { let table = table_with_two_columns("server_interval").await; let version = table.version().await.unwrap(); diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index fe4c17b1f..942c43d4e 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -2631,6 +2631,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_declare_infers_type_and_inputs() { let table = table_with_ints("declare_infers").await; let initial = table.version().await.unwrap(); @@ -2698,6 +2702,10 @@ mod tests { /// The binding reaches the schema only if `AllNulls` carries per-field /// metadata through the commit. The whole representation rests on it. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_all_nulls_preserves_field_metadata() { let table = table_with_ints("metadata_survives").await; add_computed(&table, &[("doubled".into(), "x * 2".into())]) @@ -2722,6 +2730,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_declared_column_is_all_null() { let table = table_with_ints("declare_is_null").await; add_computed(&table, &[("doubled".into(), "x * 2".into())]) @@ -2746,6 +2758,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_unknown_column_fails_at_declare_time() { let table = table_with_ints("unknown_input").await; let err = add_computed(&table, &[("bad".into(), "missing + 1".into())]) @@ -2758,6 +2774,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_unparsable_expression_fails_at_declare_time() { let table = table_with_ints("bad_syntax").await; let err = add_computed(&table, &[("bad".into(), "x *".into())]) @@ -2778,6 +2798,10 @@ mod tests { /// resolution is missing. When a registry-aware planner exists this /// becomes a supported declaration rather than a new API. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_unregistered_function_is_rejected_for_now() { let table = table_with_ints("udf_not_yet").await; let err = add_computed(&table, &[("vec".into(), "embed(x)".into())]) @@ -2795,6 +2819,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_existing_column_name_is_rejected() { let table = table_with_ints("name_taken").await; let err = add_computed(&table, &[("x".into(), "x * 2".into())]) @@ -2805,6 +2833,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_constant_expression_needs_no_inputs() { let table = table_with_ints("constant").await; add_computed(&table, &[("answer".into(), "42".into())]) @@ -2817,6 +2849,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_multiple_columns_in_one_commit() { let table = table_with_ints("multi").await; let initial = table.version().await.unwrap(); @@ -2839,6 +2875,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_duplicate_declaration_in_one_call_is_rejected() { let table = table_with_ints("dupe").await; let err = add_computed( @@ -2857,6 +2897,10 @@ mod tests { /// A batch may build on itself: one commit, and the later entry's inputs /// name the earlier one. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_declaration_may_read_one_declared_before_it() { let table = table_with_ints("chain").await; let before = table.version().await.unwrap(); @@ -2909,6 +2953,10 @@ mod tests { /// Built-in functions type the column the same way an operator does. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_builtin_function_inference() { let conn = connect("memory://").execute().await.unwrap(); let batch = record_batch!(("name", Utf8, ["ada", "grace"]), ("n", Int32, [-1, 2])).unwrap(); @@ -2959,6 +3007,10 @@ mod tests { /// an ordinary column. Reported as absent it would be refreshable by /// nothing and redeclarable over, silently. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_unrecognized_kind_is_reported_rather_than_hidden() { let table = table_with_ints("foreign_kind").await; super::add_foreign_kind(&table, "embedding", "udf").await; @@ -3002,6 +3054,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_inputs_are_deduplicated_and_sorted() { let conn = connect("memory://").execute().await.unwrap(); let batch = record_batch!(("b", Int32, [1, 2]), ("a", Int32, [3, 4])).unwrap(); @@ -3018,6 +3074,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_dropping_an_input_is_refused() { let table = table_with_ints("drop_input").await; add_computed(&table, &[("doubled".into(), "x * 2".into())]) @@ -3032,6 +3092,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_renaming_an_input_is_refused() { let table = table_with_ints("rename_input").await; add_computed(&table, &[("doubled".into(), "x * 2".into())]) @@ -3050,6 +3114,10 @@ mod tests { /// Nothing resolves against nullability, so it is not a rebinding. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_altering_an_input_nullability_is_allowed() { let table = table_with_ints("nullable_input").await; add_computed(&table, &[("doubled".into(), "x * 2".into())]) @@ -3066,6 +3134,10 @@ mod tests { /// the counting and writing passes, so the declared value is incoherent. /// Refused at declare time. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_volatile_expression_is_refused() { let table = table_with_ints("volatile_expr").await; let err = add_computed(&table, &[("maybe".into(), "random() < 0.5".into())]) @@ -3082,6 +3154,10 @@ mod tests { /// constant, but the stored SQL still names `x`, so the recorded inputs /// must too -- otherwise dropping `x` is allowed and refresh breaks. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_inputs_survive_expression_optimization() { let table = table_with_ints("optimized_inputs").await; add_computed(&table, &[("flag".into(), "true OR x > 0".into())]) @@ -3099,6 +3175,10 @@ mod tests { /// The gate's reproducer: casting a computed column rewrites the field /// without its metadata, silently destroying the declaration. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_retyping_the_computed_column_is_refused() { use arrow_schema::DataType as ArrowDataType; @@ -3123,6 +3203,10 @@ mod tests { /// A declaration cannot be edited, fabricated or erased through field /// metadata: it is validated as a whole at declare time. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_declaration_metadata_is_immutable() { use crate::table::FieldMetadataUpdate; @@ -3170,6 +3254,10 @@ mod tests { /// The gate's reproducer: only refresh materializes a declared column; /// a direct write would store an arbitrary durable value. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_computed_column_cannot_be_written_directly() { let table = table_with_ints("direct_write").await; add_computed(&table, &[("doubled".into(), "x * 2".into())]) @@ -3211,6 +3299,10 @@ mod tests { /// The gate's reproducer: the reciprocal of the declare-under-spec check. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_installing_an_lsm_spec_over_computed_columns_is_refused() { use crate::table::RTEWriteSpec; @@ -3279,6 +3371,10 @@ mod tests { /// The gate's reproducer: SQL INSERT is a write path too. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_sql_insert_cannot_write_a_computed_column() { use datafusion::prelude::SessionContext; @@ -3378,6 +3474,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_sql_insert_omitting_computed_is_allowed() { use datafusion::prelude::SessionContext; @@ -3404,6 +3504,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_nested_computed_field_cannot_be_renamed() { let table = table_with_ints("computed_struct_rename").await; add_computed(&table, &[("payload".into(), "named_struct('a', x)".into())]) @@ -3422,6 +3526,10 @@ mod tests { /// Stale handles must not commit the computed/RTE state in either order. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_stale_handles_cannot_mix_computed_and_lsm() { use crate::table::RTEWriteSpec; @@ -3472,6 +3580,10 @@ mod tests { /// unset, retained SSTable rows survive without a live spec. The catch-up /// flag is the durable marker; declaration refuses on it. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_unset_with_retained_lsm_rows_cannot_admit_a_declaration() { use crate::table::RTEWriteSpec; use arrow_array::{Int64Array, RecordBatchIterator}; @@ -3523,6 +3635,10 @@ mod tests { /// A declaration does not read itself, so it travels with its binding. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_dropping_the_computed_column_is_allowed() { let table = table_with_ints("drop_computed").await; add_computed(&table, &[("doubled".into(), "x * 2".into())]) @@ -4663,6 +4779,10 @@ mod tests { /// downstream binding's immutable input schema must not record them, or /// the first upstream refresh strands the table. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_computed_input_survives_upstream_refresh() { use arrow_array::{Int32Array, Int64Array, RecordBatch}; diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index 8f7f576ff..c5eadeeaf 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -77,6 +77,7 @@ pub(crate) async fn execute_refresh_column( table: &NativeTable, column: &str, ) -> Result { + crate::require_experimental_functions()?; Ok(execute_refresh_column_with_source(table, column) .await? .result) @@ -300,6 +301,7 @@ pub(crate) async fn execute_refresh_column_async( table: &NativeTable, column: &str, ) -> Result> { + crate::require_experimental_functions()?; // Validate before spawning so bad input is reported by this call rather // than only by the job. table.dataset.ensure_mutable()?; @@ -731,6 +733,10 @@ mod tests { /// must not bake zeros from `a`'s placeholder null. It is refused, and /// names the input, until `a` is filled -- after every append too. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_dependent_refresh_refuses_an_unfilled_input() { let table = table_with("dependent_refresh_order", vec![1, 2, 3]).await; table @@ -766,6 +772,10 @@ mod tests { /// declaration metadata and the dependency check: the recorded inputs /// are matched by name, never re-parsed as SQL. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_dependent_refresh_handles_awkward_column_names() { use arrow_array::{Int32Array, StructArray}; use arrow_schema::{DataType, Field, Fields}; @@ -820,6 +830,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_fills_a_declared_column() { let table = table_with("refresh_fills", vec![1, 2, 3]).await; let declared = declare_doubled(&table).await.unwrap(); @@ -848,6 +862,10 @@ mod tests { /// Values written after the last refresh must be reachable by another one. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_fills_rows_appended_since_the_last_refresh() { let table = table_with("refresh_appended", vec![1, 2]).await; declare_doubled(&table).await.unwrap(); @@ -868,6 +886,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_with_nothing_to_fill() { let table = table_with("refresh_noop", vec![1, 2, 3]).await; declare_doubled(&table).await.unwrap(); @@ -884,6 +906,10 @@ mod tests { /// A branch inherits the freshness main recorded: its first refresh /// fills nothing, and only its own appends after that. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_branch_inherits_freshness() { let dir = tempfile::tempdir().unwrap(); let conn = connect(dir.path().to_str().unwrap()) @@ -919,6 +945,10 @@ mod tests { /// A shallow clone inherits the freshness its source recorded. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_shallow_clone_inherits_freshness() { let dir = tempfile::tempdir().unwrap(); let conn = connect(dir.path().to_str().unwrap()) @@ -953,6 +983,10 @@ mod tests { /// settles at once instead of re-selecting the same rows forever: the /// second refresh finds the fragment signed and moves nothing. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_converges_on_a_null_result() { let table = table_with("refresh_null_result", vec![1, 2, 3]).await; let declared = table @@ -976,6 +1010,10 @@ mod tests { /// A filled row whose input moved is recomputed: the update rewrites /// the row into a fragment the stamp never signed, and only that one. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_recomputes_a_row_whose_input_moved() { let table = table_with("refresh_mutation", vec![1, 2]).await; declare_doubled(&table).await.unwrap(); @@ -1009,6 +1047,10 @@ mod tests { /// Each stamp is a sidecar under `_computed/`; pruning old versions /// removes the sidecars only they referenced, and keeps the current one. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_pruning_drops_the_sidecars_of_pruned_versions() { let dir = tempfile::tempdir().unwrap(); let conn = connect(dir.path().to_str().unwrap()) @@ -1050,6 +1092,10 @@ mod tests { /// Absolute timestamp pruning applies the same computed-column sidecar /// cleanup as duration-based pruning. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_absolute_pruning_drops_the_sidecars_of_pruned_versions() { let dir = tempfile::tempdir().unwrap(); let conn = connect(dir.path().to_str().unwrap()) @@ -1087,6 +1133,10 @@ mod tests { /// A deleted row is never computed and the rows that stay keep their /// values: a delete recomputes nothing and stamps nothing. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_delete_recomputes_nothing() { let table = table_with("refresh_delete", vec![1, 2, 3]).await; declare_doubled(&table).await.unwrap(); @@ -1106,6 +1156,10 @@ mod tests { /// signed ones is fresh: the refresh recomputes nothing and only records /// the new fragment. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_compaction_of_signed_fragments_recomputes_nothing() { let table = table_with("refresh_compact_signed", vec![1, 2]).await; declare_doubled(&table).await.unwrap(); @@ -1136,6 +1190,10 @@ mod tests { /// refresh keeps the null-fill contract and enrolls what it read from; /// from then on a moved input is recomputed like any other. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_an_unsigned_column_is_enrolled_by_its_first_refresh() { let table = table_with("refresh_legacy", vec![1, 2]).await; declare_doubled(&table).await.unwrap(); @@ -1188,6 +1246,10 @@ mod tests { /// column as null behind a covering data file. Those rows are still /// unfilled and a later refresh has to reach them. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_update_before_the_first_refresh() { let table = table_with("refresh_update_first", vec![1]).await; declare_doubled(&table).await.unwrap(); @@ -1203,6 +1265,10 @@ mod tests { /// for any of its rows: every live row is recomputed, the moved one /// included. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_compaction_of_an_unsigned_fragment_recomputes_it() { let table = table_with("refresh_mixed", vec![1, 2]).await; declare_doubled(&table).await.unwrap(); @@ -1236,6 +1302,10 @@ mod tests { /// computed column. Compaction cannot certify it, so the product is /// recomputed and the supplied value replaced. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_raw_append_values_are_not_trusted_after_compaction() { use arrow_array::RecordBatchIterator; use lance::Dataset; @@ -1288,6 +1358,10 @@ mod tests { /// An appended fragment holds no values, so compacting it into a signed /// one leaves the product fresh: only the appended rows are filled. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_compaction_with_an_appended_fragment_fills_only_its_rows() { let table = table_with("refresh_preserves", vec![1, 2]).await; declare_doubled(&table).await.unwrap(); @@ -1311,6 +1385,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_leaves_deleted_rows_alone() { let table = table_with("refresh_deleted", vec![1, 2, 3, 4]).await; declare_doubled(&table).await.unwrap(); @@ -1325,6 +1403,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_a_constant_expression() { let table = table_with("refresh_constant", vec![1, 2, 3]).await; table @@ -1341,6 +1423,10 @@ mod tests { /// A name needing quotes reaches the evaluator intact: it is carried as a /// projection alias, never spliced into SQL text. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_a_column_whose_name_needs_quoting() { let table = table_with("refresh_quoted", vec![1, 2, 3]).await; table @@ -1362,6 +1448,10 @@ mod tests { /// the probe buffers only until the first gained value and the rest flows /// through write_columns a batch at a time. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_streams_a_multi_batch_fragment() { let values: Vec = (0..20_000).collect(); let table = table_with("refresh_multi_batch", values.clone()).await; @@ -1381,6 +1471,10 @@ mod tests { /// The gate's reproducer: the commit must reuse the configured session, /// or registrations and caches vanish from the handle after a refresh. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_preserves_the_configured_session() { let session = Arc::new(lance::session::Session::default()); let conn = crate::connect("memory://") @@ -1404,6 +1498,10 @@ mod tests { /// The async form's job settles with the fill visible, like /// create_index's execute_async. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_async_job_waits_for_the_fill() { let table = table_with("refresh_async", vec![1, 2, 3]).await; declare_doubled(&table).await.unwrap(); @@ -1426,6 +1524,10 @@ mod tests { /// Bad input is reported by the call, not by the job. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_async_rejects_bad_input_before_spawning() { let table = table_with("refresh_async_bad", vec![1, 2, 3]).await; @@ -1437,6 +1539,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_async_job_reports_success_to_every_waiter() { let table = table_with("refresh_async_waiters", vec![1, 2]).await; declare_doubled(&table).await.unwrap(); @@ -1449,6 +1555,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_rejects_a_plain_column() { let table = table_with("refresh_plain", vec![1, 2, 3]).await; let err = table.refresh_column("x").await.unwrap_err(); @@ -1456,6 +1566,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_rejects_an_unknown_column() { let table = table_with("refresh_missing", vec![1, 2, 3]).await; let err = table.refresh_column("nope").await.unwrap_err(); @@ -1465,6 +1579,10 @@ mod tests { /// The gate's reproducer: a poison value in a deleted row must not /// abort filling the live rows, since nobody can read it. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_deleted_rows_value_is_never_evaluated() { let table = table_with("refresh_deleted_poison", vec![1, 0]).await; table @@ -1484,6 +1602,10 @@ mod tests { /// input did not move is not: the untouched fragment is never read, so /// its poison input is never reached. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_only_a_moved_rows_value_is_re_evaluated() { let table = table_with("refresh_filled_poison", vec![1, 0]).await; table @@ -1515,6 +1637,10 @@ mod tests { /// The gate's reproducer: the old internal projection alias is an /// ordinary column name; a computed column may use it. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_a_column_named_like_the_old_alias() { let table = table_with("refresh_alias_name", vec![1, 2]).await; table @@ -1537,6 +1663,10 @@ mod tests { /// this pins behaviorally; the memory bound is structural -- the fill /// stream retains no batches at all. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_fills_a_late_gain_fragment() { let values: Vec = (0..20_000).collect(); let table = table_with("refresh_late_gain", values).await; @@ -1562,6 +1692,10 @@ mod tests { /// The gate's reproducer: a nested input declares, refreshes, and guards /// its root against invalidating schema changes. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_nested_input_declares_and_refreshes() { use arrow_array::{Int32Array, StructArray}; use arrow_schema::{DataType, Field, Fields}; @@ -1636,6 +1770,10 @@ mod tests { /// schema_evolution tests); refresh's own check covers a dataset another /// writer left in that state. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_refuses_a_foreign_lsm_state() { use crate::table::RTEWriteSpec; @@ -1669,6 +1807,10 @@ mod tests { /// After catch-up activation and unset, no spec remains but the catch-up /// flag still marks retained SSTable rows; refresh refuses on the flag. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_refuses_retained_catchup_state() { use crate::table::RTEWriteSpec; @@ -1719,6 +1861,10 @@ mod tests { /// A declaration of a kind this version cannot evaluate is refused by /// name, rather than mistaken for a plain column or fed to the SQL path. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_rejects_a_kind_it_cannot_evaluate() { let table = table_with("refresh_foreign", vec![1, 2, 3]).await; super::super::computed_columns::add_foreign_kind(&table, "embedding", "udf").await; @@ -1754,6 +1900,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_inherits_and_publishes_blob_output() { use arrow_array::UInt64Array; use lance_arrow::{ @@ -1887,6 +2037,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_inherits_nested_struct_blob_input() { use arrow_array::{Int32Array, StructArray, UInt64Array}; use arrow_schema::{DataType, Field, Fields, Schema}; @@ -1952,6 +2106,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_preserves_list_shape_when_materializing_blob_input() { use arrow_array::{Int32Array, ListArray}; use arrow_buffer::{OffsetBuffer, ScalarBuffer}; @@ -2013,6 +2171,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_inherits_external_blob_input() { use arrow_array::{Int32Array, StringArray, UInt64Array}; use arrow_schema::{DataType, Field, Schema}; diff --git a/rust/lancedb/src/table/schema_evolution.rs b/rust/lancedb/src/table/schema_evolution.rs index f80e89ee7..d78fc5102 100644 --- a/rust/lancedb/src/table/schema_evolution.rs +++ b/rust/lancedb/src/table/schema_evolution.rs @@ -133,6 +133,8 @@ pub(crate) async fn execute_declare( ) -> Result { use lance::dataset::mem_wal::DatasetMemWalExt; + crate::require_experimental_functions()?; + // An RTE write spec keeps visible rows in tiers refresh cannot reach; // checked against latest committed state, not this handle's snapshot. table.checkout_latest().await?; @@ -491,12 +493,14 @@ mod tests { .execute() .await .unwrap(); - table - .add_columns() - .computed("derived", "spare * 2") - .execute() - .await - .unwrap(); + if cfg!(feature = "experimental-functions") { + table + .add_columns() + .computed("derived", "spare * 2") + .execute() + .await + .unwrap(); + } table .update_field_metadata(&[metadata_update("eager")]) .await @@ -510,7 +514,10 @@ mod tests { let schema = table.schema().await.unwrap(); ensure_supported_function_metadata(&schema).unwrap(); assert_eq!(function_bindings(&schema).unwrap().len(), 1); - assert!(schema.field_with_name("derived").is_ok()); + assert_eq!( + schema.field_with_name("derived").is_ok(), + cfg!(feature = "experimental-functions") + ); assert!(schema.field_with_name("moved").is_err()); } @@ -578,14 +585,16 @@ mod tests { .await .unwrap_err(), ); - bound( - table - .add_columns() - .computed(column, "1") - .execute() - .await - .unwrap_err(), - ); + if cfg!(feature = "experimental-functions") { + bound( + table + .add_columns() + .computed(column, "1") + .execute() + .await + .unwrap_err(), + ); + } } bound( table diff --git a/rust/lancedb/tests/experimental_functions.rs b/rust/lancedb/tests/experimental_functions.rs new file mode 100644 index 000000000..f7cd022e3 --- /dev/null +++ b/rust/lancedb/tests/experimental_functions.rs @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +#![cfg(not(feature = "experimental-functions"))] +#![recursion_limit = "256"] + +use std::collections::HashMap; + +use arrow_array::record_batch; +use futures::TryStreamExt; +use lancedb::materialized_view::{ + DEFINITION_META_KEY, definition_metadata_from_sql, prepare_declaration, +}; +use lancedb::table::NewColumnTransform; +use lancedb::table::computed_columns::{ + COMPUTED_COLUMN_META_KEY, EXPRESSION_META_KEY, INPUTS_META_KEY, KIND_META_KEY, computed_columns, +}; +use lancedb::{Error, MaterializedView, Result, connect}; + +fn assert_disabled(result: Result) { + match result { + Err(Error::NotSupported { message }) => { + assert!(message.contains("experimental-functions"), "{message}"); + } + _ => panic!("expected the experimental-functions feature error"), + } +} + +#[tokio::test] +async fn computed_operations_fail_before_mutation_or_job_creation() { + let db = connect("memory://").execute().await.unwrap(); + let table = db + .create_table("source", record_batch!(("x", Int32, [1, 2])).unwrap()) + .execute() + .await + .unwrap(); + let version = table.version().await.unwrap(); + assert_disabled( + table + .add_columns() + .computed("doubled", "x * 2") + .execute() + .await, + ); + assert_disabled(table.refresh_column("x").await); + assert_disabled(table.refresh_column_async("x").await); + assert_eq!(table.version().await.unwrap(), version); + assert_eq!(table.schema().await.unwrap().fields().len(), 1); + + // The eager SQL transform remains an ordinary schema-evolution operation. + table + .add_columns() + .transform(NewColumnTransform::SqlExpressions(vec![( + "eager".into(), + "x * 2".into(), + )])) + .execute() + .await + .unwrap(); + assert_eq!( + table + .count_rows(Some("eager = x * 2".into())) + .await + .unwrap(), + 2 + ); +} + +#[tokio::test] +async fn existing_computed_metadata_remains_readable_and_protected() { + let db = connect("memory://").execute().await.unwrap(); + let table = db + .create_table( + "source", + record_batch!(("x", Int32, [1, 2]), ("doubled", Int32, [2, 4])).unwrap(), + ) + .execute() + .await + .unwrap(); + // Model a table previously written by an enabled build, without enabling + // the feature in this test binary through Cargo feature unification. + let wrapper = table.dataset().unwrap(); + let mut dataset = wrapper.get().await.unwrap().as_ref().clone(); + let field_id = dataset.schema().field("doubled").unwrap().id as u32; + dataset + .replace_field_metadata(vec![( + field_id, + HashMap::from([ + (COMPUTED_COLUMN_META_KEY.into(), "true".into()), + (KIND_META_KEY.into(), "sql".into()), + (EXPRESSION_META_KEY.into(), "x * 2".into()), + (INPUTS_META_KEY.into(), "[\"x\"]".into()), + ]), + )]) + .await + .unwrap(); + wrapper.update(dataset); + let version = table.version().await.unwrap(); + assert_eq!(computed_columns(&table.schema().await.unwrap()).len(), 1); + assert_eq!( + table + .count_rows(Some("doubled = x * 2".into())) + .await + .unwrap(), + 2 + ); + assert_disabled(table.refresh_column("doubled").await); + assert_disabled(table.refresh_column_async("doubled").await); + assert!( + table + .update() + .column("doubled", "42") + .execute() + .await + .is_err() + ); + assert!(table.drop_columns(&["x"]).await.is_err()); + assert_eq!(table.version().await.unwrap(), version); + table.drop_columns(&["doubled"]).await.unwrap(); +} + +#[tokio::test] +async fn views_cannot_be_created_or_refreshed_but_existing_data_is_accessible() { + let db = connect("memory://").execute().await.unwrap(); + let source = db + .create_table("source", record_batch!(("x", Int32, [1, 2])).unwrap()) + .execute() + .await + .unwrap(); + for with_no_data in [false, true] { + assert_disabled( + db.create_materialized_view("blocked", "source") + .with_no_data(with_no_data) + .execute() + .await, + ); + assert_disabled( + db.create_materialized_view("blocked", "source") + .with_no_data(with_no_data) + .execute_async() + .await, + ); + } + let prepared = prepare_declaration(&source, None, None, None) + .await + .unwrap(); + assert_disabled(prepared.clone().create("blocked").await); + assert_disabled(prepared.create_in(&[], "blocked").await); + assert!(matches!( + db.open_table("blocked").execute().await, + Err(Error::TableNotFound { .. }) + )); + + let table = db + .create_table("existing", record_batch!(("x", Int32, [1, 2])).unwrap()) + .execute() + .await + .unwrap(); + let wrapper = table.dataset().unwrap(); + let mut dataset = wrapper.get().await.unwrap().as_ref().clone(); + dataset + .update_schema_metadata(vec![( + DEFINITION_META_KEY.into(), + Some(definition_metadata_from_sql("SELECT x FROM source")), + )]) + .await + .unwrap(); + wrapper.update(dataset); + let version = table.version().await.unwrap(); + let view = MaterializedView::from_table(table.clone()).await.unwrap(); + assert_disabled(view.refresh().execute().await); + assert_disabled(view.refresh().execute_async().await); + assert_eq!(table.version().await.unwrap(), version); + assert_eq!(view.table().count_rows(None).await.unwrap(), 2); + assert_eq!( + db.open_materialized_view("existing") + .await + .unwrap() + .definition_sql(), + "SELECT x FROM source" + ); + assert_eq!( + db.list_materialized_views(Default::default()) + .try_collect::>() + .await + .unwrap(), + vec!["existing"] + ); + assert!( + table + .add(record_batch!(("x", Int32, [3])).unwrap()) + .execute() + .await + .is_err() + ); + assert!(table.delete("true").await.is_err()); + db.drop_materialized_view("existing", &[]).await.unwrap(); +}