diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 2e290166e..e83c8d29f 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -79,11 +79,15 @@ jobs: # that dir is missing, so create an empty one for lint-only runs. run: mkdir -p ../dist && pnpm install --frozen-lockfile && pnpm lint-ci linux: - name: Linux (NodeJS ${{ matrix.node-version }}) + name: Linux (NodeJS ${{ matrix.node-version }}, ${{ matrix.features }}) timeout-minutes: 30 strategy: matrix: node-version: [ "22", "24", "26" ] + features: [ "default" ] + include: + - node-version: "24" + features: "experimental-functions" runs-on: "ubuntu-22.04" defaults: run: @@ -121,7 +125,7 @@ jobs: pnpm install --frozen-lockfile # No `--` separator: pnpm forwards it literally, which would # make napi-rs treat `--profile ci` as a cargo passthrough arg. - pnpm build:debug --profile ci + pnpm build:debug --profile ci --features ${{ matrix.features }} pnpm tsc - name: Setup examples working-directory: nodejs/examples diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 3608c24ca..a18ae5ad9 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -162,7 +162,10 @@ jobs: - name: Delete wheels run: rm -rf target/wheels platform: - name: "Mac" + name: "Mac: ${{ matrix.features }}" + strategy: + matrix: + features: ["default", "experimental-functions"] timeout-minutes: 30 runs-on: macos-14 defaults: @@ -188,7 +191,7 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} - uses: ./.github/workflows/build_mac_wheel with: - args: --profile ci + args: --profile ci --features ${{ matrix.features }} - uses: ./.github/workflows/run_tests # Make sure wheels are not included in the Rust cache - name: Delete wheels diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ffd2162c6..96b0f2bfb 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -261,6 +261,8 @@ jobs: sudo apt update sudo apt install -y protobuf-compiler libssl-dev - uses: rui314/setup-mold@v1 + - name: Test the default local Functions gate + run: cargo test --profile ci -p lancedb --features remote --test experimental_functions --locked - name: Run remote tests run: CARGO_ARGS="--profile ci" make -C ./lancedb remote-tests diff --git a/docs/src/js/_media/CONTRIBUTING.md b/docs/src/js/_media/CONTRIBUTING.md index 2faa8cd37..2125cc8aa 100644 --- a/docs/src/js/_media/CONTRIBUTING.md +++ b/docs/src/js/_media/CONTRIBUTING.md @@ -50,6 +50,19 @@ pnpm install pnpm build ``` +Local SQL computed columns and materialized-view creation/refresh are experimental +and disabled in published packages. To opt in when building from this directory: + +```shell +pnpm build:debug --features experimental-functions +pnpm tsc +pnpm test --runInBand table.test.ts materialized_view.test.ts +``` + +Rebuild without `--features experimental-functions` to test the default package. +The tests detect the compiled feature; remote operations, metadata protections, +and ordinary eager `addColumns` transforms remain available in both builds. + Lint: ```shell diff --git a/docs/src/js/classes/Connection.md b/docs/src/js/classes/Connection.md index c110eb38c..ffe7106b7 100644 --- a/docs/src/js/classes/Connection.md +++ b/docs/src/js/classes/Connection.md @@ -180,6 +180,10 @@ abstract createMaterializedView( Define a materialized view named `name` over the table `source`. +Local creation and refresh require a source build with the +`experimental-functions` Cargo feature, disabled in published packages. +Remote Cloud/Enterprise operations do not require this feature. + The view is populated before creation returns. Set `withNoData` to create only its definition and empty backing table. The view is a normal table: it can be queried, indexed and searched, and it appears in `tableNames`. diff --git a/docs/src/js/classes/MaterializedView.md b/docs/src/js/classes/MaterializedView.md index 0e2413328..bb02f5dc2 100644 --- a/docs/src/js/classes/MaterializedView.md +++ b/docs/src/js/classes/MaterializedView.md @@ -64,6 +64,7 @@ refresh(options?): Promise ``` Recompute the view from its source. +Local refresh requires a source build with `experimental-functions`. Every refresh rebuilds the complete view. For local views, `sourceVersion` refreshes to that source version instead of the latest; diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index 7cf87c405..80bba02a0 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -69,6 +69,10 @@ abstract addColumns(newColumnTransforms): Promise Add new columns with defined values. +Local computed-column declaration and refresh require a source build with +the `experimental-functions` Cargo feature. Published packages disable it; +remote Cloud/Enterprise operations remain available. + The `{ computed }` form stores the expression rather than evaluating it now: the column is committed with no values, and rows get them from [Table#refreshColumn](Table.md#refreshcolumn). Declaring one therefore costs the same on a @@ -1035,6 +1039,7 @@ abstract refreshColumn(column): Promise ``` Fill the rows of a computed column that hold no value yet. +Local refresh requires a source build with `experimental-functions`. Rows appended since the last refresh are filled by the next one, and rows whose inputs changed since they were computed are recomputed; @@ -1063,6 +1068,8 @@ abstract refreshColumnAsync(column): Promise Like [Table#refreshColumn](Table.md#refreshcolumn), but returns a handle to the refresh job instead of blocking until it completes. +Local refresh requires a source build with `experimental-functions`. +Remote Cloud/Enterprise jobs do not require this feature. The job may already be complete when returned; callers must not assume the column is filled until [Job.wait](Job.md#wait) resolves. Invalid input -- diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 2fefe91f3..5f87302ad 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -429,6 +429,13 @@ scoped to each invocation and support `read`, `readinto`, `seek`, `tell`, ## Materialized Views (Synchronous) +In the 0.41 SDK release, local materialized-view creation and refresh require a +source build with the `experimental-functions` Cargo feature. Published Python +packages leave it disabled. Opening, reading, listing and dropping existing views +remain available, as do remote Cloud and Enterprise operations. The same feature +requirement applies to local SQL computed-column declaration and refresh; ordinary +eager `add_columns` transforms remain available. + On Cloud and Enterprise, creation accepts `auto_refresh_enabled` and `auto_refresh_interval_seconds`: @@ -443,7 +450,8 @@ view = db.create_materialized_view( Omit `auto_refresh_interval_seconds` to let the server determine when to refresh. These options also apply to asynchronous creation and creation jobs. Embedded -connections support manual refresh and reject automatic scheduling requests. +connections built with `experimental-functions` support manual refresh and reject +automatic scheduling requests. ::: lancedb.materialized_view.MaterializedView diff --git a/docs/web/feature-engineering/add-computed-column.mdx b/docs/web/feature-engineering/add-computed-column.mdx index c287d42f5..707a929f5 100644 --- a/docs/web/feature-engineering/add-computed-column.mdx +++ b/docs/web/feature-engineering/add-computed-column.mdx @@ -8,13 +8,20 @@ availability: oss: available enterprise: available summary: >- - Both offerings support computed columns. Enterprise requires the deployment to enable them; the embedded library evaluates them in your process. + Local computed columns require a source build with the experimental-functions Cargo feature. Enterprise requires the deployment to enable them. --- A computed column is declared with an expression and starts with no values. Declaring it is cheap on any table size, because nothing is evaluated until [a refresh](/feature-engineering/refresh-column) asks for it. + +In the 0.41 SDK release, local computed-column declaration and refresh require a +source build with the `experimental-functions` Cargo feature. Published Python +and TypeScript packages leave it disabled. Remote Cloud and Enterprise operations +do not require this feature. Ordinary eager `add_columns` transforms remain available. + + ## Add a computed column {#declare} ```python diff --git a/docs/web/feature-engineering/materialized-views.mdx b/docs/web/feature-engineering/materialized-views.mdx index 1552c3638..30d2d1bc6 100644 --- a/docs/web/feature-engineering/materialized-views.mdx +++ b/docs/web/feature-engineering/materialized-views.mdx @@ -8,7 +8,7 @@ availability: oss: available enterprise: available summary: >- - Both offerings support materialized views. OSS refreshes run in your process; Enterprise runs the work on remote compute. See [Enterprise refreshes](#availability) for server compatibility requirements. + Local creation and refresh require a source build with the experimental-functions Cargo feature. Enterprise runs the work on remote compute. See [Enterprise refreshes](#availability) for server compatibility requirements. --- import { @@ -20,6 +20,13 @@ A materialized view stores the result of a query as a table. Use it to keep a filtered or transformed dataset that you can query, index and search repeatedly. Refresh the view after its source data changes. + +In the 0.41 SDK release, local materialized-view creation and refresh require a +source build with the `experimental-functions` Cargo feature. Published Python +and TypeScript packages leave it disabled. Opening, reading, listing and dropping +existing views remain available, as do remote Cloud and Enterprise operations. + + In SQL this is [`REFRESH MATERIALIZED VIEW`](/sql/refresh#refresh-materialized-view). diff --git a/docs/web/feature-engineering/refresh-column.mdx b/docs/web/feature-engineering/refresh-column.mdx index ce36daa32..1aa511234 100644 --- a/docs/web/feature-engineering/refresh-column.mdx +++ b/docs/web/feature-engineering/refresh-column.mdx @@ -8,13 +8,20 @@ availability: oss: available enterprise: available summary: >- - Both offerings support column refresh. The embedded library runs it in your process; Enterprise runs it as a server job that must be enabled for your Enterprise cluster. + Local column refresh requires a source build with the experimental-functions Cargo feature. Enterprise runs it as a server job that must be enabled for your Enterprise cluster. --- Refresh a [computed column](/feature-engineering/add-computed-column) to fill new rows and recompute values whose inputs changed. The example below continues with the `doubled` column from the computed-column guide. + +In the 0.41 SDK release, local declaration and refresh require a source build +with the `experimental-functions` Cargo feature. Published Python and TypeScript +packages leave it disabled. Remote Cloud and Enterprise operations do not require +this feature; existing computed-column data remains readable in default builds. + + ```python Python icon="python" table.refresh_column("doubled") diff --git a/nodejs/CONTRIBUTING.md b/nodejs/CONTRIBUTING.md index 2faa8cd37..2125cc8aa 100644 --- a/nodejs/CONTRIBUTING.md +++ b/nodejs/CONTRIBUTING.md @@ -50,6 +50,19 @@ pnpm install pnpm build ``` +Local SQL computed columns and materialized-view creation/refresh are experimental +and disabled in published packages. To opt in when building from this directory: + +```shell +pnpm build:debug --features experimental-functions +pnpm tsc +pnpm test --runInBand table.test.ts materialized_view.test.ts +``` + +Rebuild without `--features experimental-functions` to test the default package. +The tests detect the compiled feature; remote operations, metadata protections, +and ordinary eager `addColumns` transforms remain available in both builds. + Lint: ```shell diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index e8abcf60a..4c86916eb 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -48,3 +48,4 @@ napi-build = "2.3.1" default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/goosefs", "lancedb/metrics-otel"] fp16kernels = ["lancedb/fp16kernels"] remote = ["lancedb/remote"] +experimental-functions = ["lancedb/experimental-functions"] diff --git a/nodejs/__test__/experimental_functions.test.ts b/nodejs/__test__/experimental_functions.test.ts new file mode 100644 index 000000000..fdadb092f --- /dev/null +++ b/nodejs/__test__/experimental_functions.test.ts @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { connect } from "../lancedb"; +import { experimentalFunctionsEnabled } from "../lancedb/native"; + +const defaultBuildTest = experimentalFunctionsEnabled() ? it.skip : it; + +defaultBuildTest( + "rejects local Functions before mutating the database", + async () => { + const directory = await mkdtemp(join(tmpdir(), "lancedb-experimental-")); + try { + const db = await connect(directory); + const table = await db.createTable("source", [{ x: 1 }, { x: 2 }]); + const version = await table.version(); + const operations = [ + () => + table.addColumns({ + computed: [{ name: "doubled", valueSql: "x * 2" }], + }), + () => table.refreshColumn("x"), + () => table.refreshColumnAsync("x"), + () => db.createMaterializedView("blocked", "source"), + () => + db.createMaterializedView("blocked", "source", { withNoData: true }), + ]; + for (const operation of operations) { + await expect(operation()).rejects.toThrow("experimental-functions"); + } + expect(await table.version()).toBe(version); + expect(await db.tableNames()).toEqual(["source"]); + await table.addColumns([{ name: "eager", valueSql: "x * 2" }]); + expect((await table.query().toArray()).map((row) => row.eager)).toEqual([ + 2, 4, + ]); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }, +); diff --git a/nodejs/__test__/materialized_view.test.ts b/nodejs/__test__/materialized_view.test.ts index 3852af717..2324e1fcf 100644 --- a/nodejs/__test__/materialized_view.test.ts +++ b/nodejs/__test__/materialized_view.test.ts @@ -8,8 +8,10 @@ import { DEFINITION_META_KEY, definitionFromMetadata, } from "../lancedb/materialized_view"; +import { experimentalFunctionsEnabled } from "../lancedb/native"; describe("materialized views", () => { + const experimentalTest = experimentalFunctionsEnabled() ? it : it.skip; let tmpDir: tmp.DirResult; let db: Connection; @@ -69,7 +71,7 @@ describe("materialized views", () => { ); }); - it("creates, refreshes and queries a view", async () => { + experimentalTest("creates, refreshes and queries a view", async () => { const view = await db.createMaterializedView("adults", "people", { select: ["name", ["shout", "upper(name)"]], where: "age >= 18", @@ -81,7 +83,7 @@ describe("materialized views", () => { expect(rows.map((r) => r.shout).sort()).toEqual(["ADA", "GRACE"]); }); - it("round-trips the definition", async () => { + experimentalTest("round-trips the definition", async () => { await db.createMaterializedView("adults", "people", { where: "age >= 18", }); @@ -92,7 +94,7 @@ describe("materialized views", () => { ); }); - it("fully refreshes after an append", async () => { + experimentalTest("fully refreshes after an append", async () => { const view = await db.createMaterializedView("copy", "people", { withNoData: true, }); @@ -108,7 +110,7 @@ describe("materialized views", () => { expect((await view.refresh()).mode).toBe("rebuild"); }); - it("lists views and rejects non-views", async () => { + experimentalTest("lists views and rejects non-views", async () => { await db.createMaterializedView("adults", "people", { where: "age >= 18", }); @@ -124,17 +126,20 @@ describe("materialized views", () => { expect(await collect(db.listMaterializedViews())).toEqual([]); }); - it("returns a job when dropping a view asynchronously", async () => { - await db.createMaterializedView("adults", "people"); + experimentalTest( + "returns a job when dropping a view asynchronously", + async () => { + await db.createMaterializedView("adults", "people"); - const job = await db.dropMaterializedViewAsync("adults"); - expect(job.id).toBeNull(); - expect(job.jobType).toBeNull(); - await job.wait(); - expect(await collect(db.listMaterializedViews())).toEqual([]); - }); + const job = await db.dropMaterializedViewAsync("adults"); + expect(job.id).toBeNull(); + expect(job.jobType).toBeNull(); + await job.wait(); + expect(await collect(db.listMaterializedViews())).toEqual([]); + }, + ); - it("rejects an invalid expression at create time", async () => { + experimentalTest("rejects an invalid expression at create time", async () => { await expect( db.createMaterializedView("bad", "people", { select: [["x", "missing + 1"]], @@ -149,16 +154,21 @@ describe("materialized views", () => { ).rejects.toThrow("non-negative integer"); } expect(await collect(db.listMaterializedViews())).toEqual([]); - - const view = await db.createMaterializedView("copy", "people"); - for (const sourceVersion of [-1, 1.5, Infinity, NaN]) { - await expect(view.refresh({ sourceVersion })).rejects.toThrow( - "non-negative integer", - ); - } }); - it("quotes bare select names", async () => { + experimentalTest( + "rejects invalid source versions before refreshing", + async () => { + const view = await db.createMaterializedView("copy", "people"); + for (const sourceVersion of [-1, 1.5, Infinity, NaN]) { + await expect(view.refresh({ sourceVersion })).rejects.toThrow( + "non-negative integer", + ); + } + }, + ); + + experimentalTest("quotes bare select names", async () => { await db.createTable("odd_names", [{ "order item": "widget" }]); const view = await db.createMaterializedView("quoted", "odd_names", { select: ["order item"], @@ -168,11 +178,14 @@ describe("materialized views", () => { expect(Number(result.rowsWritten)).toBe(1); }); - it("does not require stable row ids on the source", async () => { - await db.createTable("plain", [{ x: 1 }]); - const view = await db.createMaterializedView("v", "plain"); - expect(await view.table().countRows()).toBe(1); - }); + experimentalTest( + "does not require stable row ids on the source", + async () => { + await db.createTable("plain", [{ x: 1 }]); + const view = await db.createMaterializedView("v", "plain"); + expect(await view.table().countRows()).toBe(1); + }, + ); }); async function collect(items: AsyncIterable): Promise { diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index d848676cd..c9e1efcb6 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -47,6 +47,7 @@ import { register, } from "../lancedb/embedding"; import { Index } from "../lancedb/indices"; +import { experimentalFunctionsEnabled } from "../lancedb/native"; import { BooleanQuery, Occur, @@ -5477,13 +5478,14 @@ describe("LSM convergence and stats", () => { }); describe("computed columns", () => { + const experimentalTest = experimentalFunctionsEnabled() ? it : it.skip; let tmpDir: tmp.DirResult; beforeEach(() => { tmpDir = tmp.dirSync({ unsafeCleanup: true }); }); afterEach(() => tmpDir.removeCallback()); - it("declares a column and fills it on refresh", async () => { + experimentalTest("declares a column and fills it on refresh", async () => { const db = await connect(tmpDir.name); const table = await db.createTable("computed", [{ x: 1 }, { x: 2 }]); @@ -5508,7 +5510,7 @@ describe("computed columns", () => { ); }); - it("returns a job handle from refreshColumnAsync", async () => { + experimentalTest("returns a job handle from refreshColumnAsync", async () => { const db = await connect(tmpDir.name); const table = await db.createTable("computed_job", [{ x: 1 }, { x: 2 }]); @@ -5530,7 +5532,7 @@ describe("computed columns", () => { ); }); - it("fills rows added since the last refresh", async () => { + experimentalTest("fills rows added since the last refresh", async () => { const db = await connect(tmpDir.name); const table = await db.createTable("computed_append", [{ x: 1 }]); diff --git a/nodejs/lancedb/connection.ts b/nodejs/lancedb/connection.ts index 5f067eeef..80fe2c722 100644 --- a/nodejs/lancedb/connection.ts +++ b/nodejs/lancedb/connection.ts @@ -270,6 +270,10 @@ export abstract class Connection { /** * Define a materialized view named `name` over the table `source`. * + * Local creation and refresh require a source build with the + * `experimental-functions` Cargo feature, disabled in published packages. + * Remote Cloud/Enterprise operations do not require this feature. + * * The view is populated before creation returns. Set `withNoData` to create * only its definition and empty backing table. The view is a normal table: * it can be queried, indexed and searched, and it appears in `tableNames`. diff --git a/nodejs/lancedb/materialized_view.ts b/nodejs/lancedb/materialized_view.ts index 7cf1690e0..cc209b14d 100644 --- a/nodejs/lancedb/materialized_view.ts +++ b/nodejs/lancedb/materialized_view.ts @@ -186,6 +186,7 @@ export class MaterializedView { /** * Recompute the view from its source. + * Local refresh requires a source build with `experimental-functions`. * * Every refresh rebuilds the complete view. For local views, * `sourceVersion` refreshes to that source version instead of the latest; diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 4ae1aefb8..4738b6140 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -594,6 +594,10 @@ export abstract class Table { /** * Add new columns with defined values. * + * Local computed-column declaration and refresh require a source build with + * the `experimental-functions` Cargo feature. Published packages disable it; + * remote Cloud/Enterprise operations remain available. + * * The `{ computed }` form stores the expression rather than evaluating it * now: the column is committed with no values, and rows get them from * {@link Table#refreshColumn}. Declaring one therefore costs the same on a @@ -632,6 +636,7 @@ export abstract class Table { /** * Fill the rows of a computed column that hold no value yet. + * Local refresh requires a source build with `experimental-functions`. * * Rows appended since the last refresh are filled by the next one, and * rows whose inputs changed since they were computed are recomputed; @@ -646,6 +651,8 @@ export abstract class Table { /** * Like {@link Table#refreshColumn}, but returns a handle to the refresh * job instead of blocking until it completes. + * Local refresh requires a source build with `experimental-functions`. + * Remote Cloud/Enterprise jobs do not require this feature. * * The job may already be complete when returned; callers must not assume * the column is filled until {@link Job.wait} resolves. Invalid input -- diff --git a/nodejs/src/lib.rs b/nodejs/src/lib.rs index f47bc113e..a92b8db98 100644 --- a/nodejs/src/lib.rs +++ b/nodejs/src/lib.rs @@ -30,6 +30,12 @@ mod session; mod table; mod util; +/// Whether this native binding was built with local experimental Functions. +#[napi] +pub fn experimental_functions_enabled() -> bool { + cfg!(feature = "experimental-functions") +} + #[napi(object)] #[derive(Debug)] pub struct ConnectionOptions { diff --git a/python/CONTRIBUTING.md b/python/CONTRIBUTING.md index 122677364..f86dea706 100644 --- a/python/CONTRIBUTING.md +++ b/python/CONTRIBUTING.md @@ -48,6 +48,18 @@ Build the package make develop ``` +Local SQL computed columns and materialized-view creation/refresh are experimental +and disabled in published packages. To opt in when building from this directory: + +```shell +uv run --extra tests --extra dev maturin develop --features experimental-functions --extras tests,dev +uv run --no-sync pytest python/tests -m experimental_functions +``` + +Rebuild without `--features experimental-functions` to test the default package. +The tests detect the compiled feature; remote operations, metadata protections, +and ordinary eager `add_columns` transforms remain available in both builds. + Format: ```shell diff --git a/python/Cargo.toml b/python/Cargo.toml index b1581b366..f84ab80a3 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -52,3 +52,4 @@ pyo3-build-config = { version = "0.28", features = ["abi3-py310"] } default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/cos", "lancedb/goosefs", "lancedb/metrics-otel"] fp16kernels = ["lancedb/fp16kernels"] remote = ["lancedb/remote"] +experimental-functions = ["lancedb/experimental-functions"] diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 1f3cfa9ed..333fca92e 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -26,6 +26,8 @@ from lance_namespace import ( from .remote import ClientConfig from .types import BaseTokenizerType +_experimental_functions_enabled: bool + IvfHnswPq: type[HnswPq] = HnswPq IvfHnswSq: type[HnswSq] = HnswSq IvfHnswFlat: type[HnswFlat] = HnswFlat diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index b6dfe8fb8..f38608188 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -626,6 +626,10 @@ class DBConnection(EnforceOverrides): ) -> MaterializedView: """Define a materialized view named ``name`` over the table ``source``. + Local creation and refresh require a source build with the + ``experimental-functions`` Cargo feature. Published packages leave it + disabled; remote Cloud/Enterprise operations remain available. + The view is populated before creation returns. Pass ``with_no_data=True`` to create only its definition and empty backing table. The view is a normal table: it can be queried, indexed and @@ -1760,6 +1764,10 @@ class LanceDBConnection(DBConnection): auto_refresh_interval_seconds: Optional[float] = None, ) -> MaterializedView: """Define a materialized view named ``name`` over the table ``source``. + + Local creation and refresh require a source build with the + ``experimental-functions`` Cargo feature. Published packages leave it + disabled; remote Cloud/Enterprise operations remain available. See [DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view]. @@ -1769,13 +1777,13 @@ class LanceDBConnection(DBConnection): >>> db = lancedb.connect("./.lancedb") >>> data = [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}] >>> table = db.create_table("people", data) - >>> view = db.create_materialized_view( + >>> view = db.create_materialized_view( # doctest: +SKIP ... "adults", ... "people", ... select=["name", ("shout", "upper(name)")], ... where="age >= 18", ... ) - >>> view.table.count_rows() + >>> view.table.count_rows() # doctest: +SKIP 1 """ LOOP.run( @@ -2846,6 +2854,10 @@ class AsyncConnection(object): auto_refresh_interval_seconds: Optional[float] = None, ) -> AsyncMaterializedView: """Define a materialized view named ``name`` over the table ``source``. + + Local creation and refresh require a source build with the + ``experimental-functions`` Cargo feature. Published packages leave it + disabled; remote Cloud/Enterprise operations remain available. See [DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view]. """ diff --git a/python/python/lancedb/materialized_view.py b/python/python/lancedb/materialized_view.py index 6342641ff..8efe31f05 100644 --- a/python/python/lancedb/materialized_view.py +++ b/python/python/lancedb/materialized_view.py @@ -2,7 +2,12 @@ # SPDX-FileCopyrightText: Copyright The LanceDB Authors """Materialized views: tables defined by a query over a source table and -maintained by refresh. See ``DBConnection.create_materialized_view``.""" +maintained by refresh. See ``DBConnection.create_materialized_view``. + +Local creation and refresh require the ``experimental-functions`` Cargo feature +in a source build. Published packages disable it. Opening, reading, listing and +dropping existing views and remote Cloud/Enterprise operations remain available. +""" from __future__ import annotations diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 048c46a6a..6df109d6c 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -2988,6 +2988,10 @@ class Table(ABC): Function columns are supported only on LanceDB Cloud and Enterprise. computed: Dict[str, str], optional + Local declaration and refresh require a source build with the + ``experimental-functions`` Cargo feature, which published packages + do not enable. Remote Cloud/Enterprise operations remain available. + A mapping from output column names to SQL expressions derives each output field from its expression. A direct projection of a Blob v2 field inherits Blob v2 semantics; other expressions derive their @@ -3029,11 +3033,11 @@ class Table(ABC): >>> import lancedb >>> db = lancedb.connect("./.lancedb") >>> table = db.create_table("computed_demo", [{"x": 1}, {"x": 2}]) - >>> table.add_columns(computed={"doubled": "x * 2"}) + >>> table.add_columns(computed={"doubled": "x * 2"}) # doctest: +SKIP AddColumnsResult(version=2) - >>> table.refresh_column("doubled") + >>> table.refresh_column("doubled") # doctest: +SKIP RefreshColumnResult(rows_filled=2, version=4) - >>> table.to_arrow().sort_by("x").to_pandas() + >>> table.to_arrow().sort_by("x").to_pandas() # doctest: +SKIP x doubled 0 1 2 1 2 4 @@ -3044,6 +3048,9 @@ class Table(ABC): """ Fill the rows of a computed column that hold no value yet. + Local refresh requires the ``experimental-functions`` Cargo feature + in a source build. Published packages leave it disabled. + Declared with ``add_columns(computed=...)``, a column starts empty and gets its values here. Rows appended since the last refresh are filled by the next one, and rows whose inputs changed since they were computed @@ -3070,6 +3077,9 @@ class Table(ABC): Like :meth:`refresh_column`, but returns a handle to the refresh job instead of blocking until it completes. + Local refresh requires a source build with ``experimental-functions``. + Remote Cloud/Enterprise jobs do not require this feature. + The job may already be complete when returned; callers must not assume the column is filled until :meth:`Job.wait` returns. Invalid input -- an unknown column, or one that is not computed -- raises here rather @@ -3087,13 +3097,13 @@ class Table(ABC): >>> import lancedb >>> db = lancedb.connect("./.lancedb") >>> table = db.create_table("computed_job_demo", [{"x": 1}, {"x": 2}]) - >>> table.add_columns(computed={"doubled": "x * 2"}) + >>> table.add_columns(computed={"doubled": "x * 2"}) # doctest: +SKIP AddColumnsResult(version=2) - >>> job = table.refresh_column_async("doubled") - >>> result = job.wait() - >>> result.rows_assigned + >>> job = table.refresh_column_async("doubled") # doctest: +SKIP + >>> result = job.wait() # doctest: +SKIP + >>> result.rows_assigned # doctest: +SKIP 2 - >>> job.status() + >>> job.status() # doctest: +SKIP 'finished' """ @@ -7195,6 +7205,10 @@ class AsyncTable: Function columns are supported only on LanceDB Cloud and Enterprise. computed: Dict[str, str], optional + Local declaration and refresh require a source build with the + ``experimental-functions`` Cargo feature, which published packages + do not enable. Remote Cloud/Enterprise operations remain available. + A mapping from output column names to SQL expressions derives each output field from its expression. A direct projection of a Blob v2 field inherits Blob v2 semantics; other expressions derive their @@ -7314,6 +7328,9 @@ class AsyncTable: """ Fill the rows of a computed column that hold no value yet. + Local refresh requires the ``experimental-functions`` Cargo feature + in a source build. Published packages leave it disabled. + Declared with ``add_columns(computed=...)``, a column starts empty and gets its values here. Rows appended since the last refresh are filled by the next one, and rows whose inputs changed since they were computed @@ -7384,6 +7401,9 @@ class AsyncTable: Like :meth:`refresh_column`, but returns a handle to the refresh job instead of blocking until it completes. + Local refresh requires a source build with ``experimental-functions``. + Remote Cloud/Enterprise jobs do not require this feature. + The job may already be complete when returned; callers must not assume the column is filled until :meth:`AsyncJob.wait` resolves. Invalid input -- an unknown column, or one that is not computed -- raises here @@ -7409,7 +7429,7 @@ class AsyncTable: ... result = await job.wait() ... assert result.rows_assigned == 1 ... return await job.status() - >>> asyncio.run(refresh_in_background()) + >>> asyncio.run(refresh_in_background()) # doctest: +SKIP 'finished' """ return _typed_job( diff --git a/python/python/tests/conftest.py b/python/python/tests/conftest.py index 8567a9830..86c4f00bb 100644 --- a/python/python/tests/conftest.py +++ b/python/python/tests/conftest.py @@ -9,6 +9,21 @@ import pytest import pytest_asyncio +def pytest_configure(config): + config.addinivalue_line( + "markers", "experimental_functions: requires the experimental-functions build" + ) + + +def pytest_collection_modifyitems(items): + if lancedb._lancedb._experimental_functions_enabled: + return + skip = pytest.mark.skip(reason="requires the experimental-functions Cargo feature") + for item in items: + if item.get_closest_marker("experimental_functions"): + item.add_marker(skip) + + def pandas_string_type(): """Return the PyArrow string type that pandas uses for string columns. diff --git a/python/python/tests/test_experimental_functions.py b/python/python/tests/test_experimental_functions.py new file mode 100644 index 000000000..81de84606 --- /dev/null +++ b/python/python/tests/test_experimental_functions.py @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import lancedb +import pytest + +pytestmark = pytest.mark.skipif( + lancedb._lancedb._experimental_functions_enabled, + reason="tests the default build without experimental-functions", +) + + +@pytest.mark.parametrize("namespace", [False, True]) +def test_default_build_rejects_local_functions(tmp_path, namespace): + db = ( + lancedb.connect_namespace("dir", {"root": str(tmp_path)}) + if namespace + else lancedb.connect(tmp_path) + ) + table = db.create_table("source", [{"x": 1}, {"x": 2}]) + version = table.version + operations = [ + lambda: table.add_columns(computed={"doubled": "x * 2"}), + lambda: table.refresh_column("x"), + lambda: table.refresh_column_async("x"), + lambda: db.create_materialized_view("blocked", "source"), + lambda: db.create_materialized_view("blocked", "source", with_no_data=True), + lambda: db.create_materialized_view_async("blocked", "source"), + ] + for operation in operations: + with pytest.raises(NotImplementedError, match="experimental-functions"): + operation() + assert table.version == version + assert db.table_names() == ["source"] + table.add_columns({"eager": "x * 2"}) + assert table.to_arrow()["eager"].to_pylist() == [2, 4] + + +@pytest.mark.asyncio +async def test_async_default_build_rejects_local_functions(tmp_path): + db = await lancedb.connect_async(tmp_path) + table = await db.create_table("source", [{"x": 1}, {"x": 2}]) + version = await table.version() + operations = [ + lambda: table.add_columns(computed={"doubled": "x * 2"}), + lambda: table.refresh_column("x"), + lambda: table.refresh_column_async("x"), + lambda: db.create_materialized_view("blocked", "source"), + lambda: db.create_materialized_view("blocked", "source", with_no_data=True), + lambda: db.create_materialized_view_async("blocked", "source"), + ] + for operation in operations: + with pytest.raises(NotImplementedError, match="experimental-functions"): + await operation() + assert await table.version() == version + assert await db.table_names() == ["source"] + await table.add_columns({"eager": "x * 2"}) + assert (await table.query().to_arrow())["eager"].to_pylist() == [2, 4] diff --git a/python/python/tests/test_materialized_views.py b/python/python/tests/test_materialized_views.py index e2be9c0d4..337fcad78 100644 --- a/python/python/tests/test_materialized_views.py +++ b/python/python/tests/test_materialized_views.py @@ -173,6 +173,7 @@ def test_sync_remote_create_uses_public_async_connection(): assert calls[-1] == ("drop", "adults", ["analytics"]) +@pytest.mark.experimental_functions def test_create_refresh_and_query(tmp_path): db = make_db(tmp_path) view = db.create_materialized_view( @@ -188,6 +189,7 @@ def test_create_refresh_and_query(tmp_path): assert sorted(row["shout"] for row in rows) == ["ADA", "GRACE"] +@pytest.mark.experimental_functions def test_create_and_refresh_jobs(tmp_path): db = make_db(tmp_path) create_job = db.create_materialized_view_async( @@ -212,6 +214,7 @@ def test_create_and_refresh_jobs(tmp_path): assert "adults" not in list(db.list_materialized_views()) +@pytest.mark.experimental_functions def test_definition_round_trips(tmp_path): db = make_db(tmp_path) db.create_materialized_view("adults", "people", where="age >= 18") @@ -222,6 +225,7 @@ def test_definition_round_trips(tmp_path): ) +@pytest.mark.experimental_functions def test_full_refresh_after_append(tmp_path): db = make_db(tmp_path) view = db.create_materialized_view("copy", "people", with_no_data=True) @@ -236,6 +240,7 @@ def test_full_refresh_after_append(tmp_path): assert view.refresh().mode == "rebuild" +@pytest.mark.experimental_functions def test_full_refresh_after_update(tmp_path): db = make_db(tmp_path) view = db.create_materialized_view("copy", "people", with_no_data=True) @@ -249,6 +254,7 @@ def test_full_refresh_after_update(tmp_path): assert sorted(row["age"] for row in rows) == [8, 36, 85] +@pytest.mark.experimental_functions def test_legacy_storage_source_update_rebuilds(tmp_path): db = lancedb.connect( tmp_path, @@ -265,6 +271,7 @@ def test_legacy_storage_source_update_rebuilds(tmp_path): assert sorted(row["age"] for row in rows) == [8, 36] +@pytest.mark.experimental_functions def test_direct_writes_to_a_view_are_refused(tmp_path): db = make_db(tmp_path) view = db.create_materialized_view("copy", "people") @@ -290,6 +297,7 @@ def test_direct_writes_to_a_view_are_refused(tmp_path): assert view.refresh().rows_written == 4 +@pytest.mark.experimental_functions def test_list_and_not_a_view(tmp_path): db = make_db(tmp_path) db.create_materialized_view("adults", "people", where="age >= 18") @@ -304,6 +312,7 @@ def test_list_and_not_a_view(tmp_path): assert list(db.list_materialized_views()) == [] +@pytest.mark.experimental_functions def test_invalid_expression_fails_at_create(tmp_path): db = make_db(tmp_path) with pytest.raises(Exception, match="missing"): @@ -311,6 +320,7 @@ def test_invalid_expression_fails_at_create(tmp_path): assert "bad" not in db.list_tables().tables +@pytest.mark.experimental_functions @pytest.mark.asyncio async def test_async_create_refresh_and_open(tmp_path): db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS) @@ -332,6 +342,7 @@ async def test_async_create_refresh_and_open(tmp_path): assert [item async for item in db.list_materialized_views()] == ["shouts"] +@pytest.mark.experimental_functions @pytest.mark.asyncio async def test_async_create_and_refresh_jobs(tmp_path): db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS) @@ -358,6 +369,7 @@ async def test_async_create_and_refresh_jobs(tmp_path): assert "adults" not in [item async for item in db.list_materialized_views()] +@pytest.mark.experimental_functions @pytest.mark.asyncio async def test_async_refresh_rebuilds(tmp_path): db = await lancedb.connect_async(tmp_path) @@ -372,6 +384,7 @@ async def test_async_refresh_rebuilds(tmp_path): assert result.rows_written == 2 +@pytest.mark.experimental_functions def test_source_does_not_require_stable_row_ids(tmp_path): db = lancedb.connect(tmp_path) db.create_table("plain", [{"x": 1}]) @@ -379,6 +392,7 @@ def test_source_does_not_require_stable_row_ids(tmp_path): assert view.table.count_rows() == 1 +@pytest.mark.experimental_functions def test_bare_select_names_are_quoted(tmp_path): db = lancedb.connect(tmp_path, storage_options=STABLE_ROW_IDS) db.create_table("odd_names", [{"order item": "widget", "select": 2}]) @@ -396,6 +410,7 @@ def test_bare_select_names_are_quoted(tmp_path): assert rows[0]["select"] == 2 +@pytest.mark.experimental_functions def test_scalar_select_is_one_column(tmp_path): db = make_db(tmp_path) view = db.create_materialized_view("just_name", "people", select="name") @@ -405,6 +420,7 @@ def test_scalar_select_is_one_column(tmp_path): assert sorted(row["name"] for row in rows) == ["ada", "grace", "kid"] +@pytest.mark.experimental_functions @pytest.mark.asyncio async def test_async_scalar_select_is_one_column(tmp_path): db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS) @@ -415,6 +431,7 @@ async def test_async_scalar_select_is_one_column(tmp_path): assert set(rows[0]) == {"name"} +@pytest.mark.experimental_functions def test_limit_above_i64_max_is_refused(tmp_path): db = make_db(tmp_path) with pytest.raises(ValueError, match="exceeds the maximum"): @@ -434,6 +451,7 @@ def _namespace_db(tmp_path): ) +@pytest.mark.experimental_functions def test_namespace_connection_materialized_views(tmp_path): db = _namespace_db(tmp_path) db.create_table( @@ -464,6 +482,7 @@ def test_namespace_connection_materialized_views(tmp_path): assert list(db.list_materialized_views()) == [] +@pytest.mark.experimental_functions @pytest.mark.asyncio async def test_async_namespace_connection_materialized_views(tmp_path): db = lancedb.connect_namespace_async( @@ -557,6 +576,7 @@ def test_stored_queries_and_legacy_layouts_are_read(): read(newer) +@pytest.mark.experimental_functions def test_local_materialized_view_listing_pages_and_resume(tmp_path): db = make_db(tmp_path) db.create_materialized_view("a_view", "people", with_no_data=True) @@ -646,6 +666,7 @@ async def test_async_remote_create_keeps_interval_unspecified(): assert native.calls[0][2]["auto_refresh_interval_seconds"] is None +@pytest.mark.experimental_functions @pytest.mark.parametrize( "options", [ @@ -673,12 +694,14 @@ def test_remote_create_rejects_invalid_interval_without_request(seconds): assert requests == [] +@pytest.mark.experimental_functions def test_namespace_forwards_automatic_refresh_request(tmp_path): db = lancedb.connect_namespace("dir", {"root": str(tmp_path)}) with pytest.raises(NotImplementedError, match="Cloud or Enterprise"): db.create_materialized_view("mv", "source", auto_refresh_enabled=True) +@pytest.mark.experimental_functions @pytest.mark.asyncio async def test_async_namespace_forwards_automatic_refresh_request(tmp_path): db = lancedb.connect_namespace_async("dir", {"root": str(tmp_path)}) @@ -686,6 +709,7 @@ async def test_async_namespace_forwards_automatic_refresh_request(tmp_path): await db.create_materialized_view("mv", "source", auto_refresh_enabled=True) +@pytest.mark.experimental_functions def test_embedded_explicitly_disabled_refresh_is_supported(tmp_path): db = make_db(tmp_path) view = db.create_materialized_view("mv", "people", auto_refresh_enabled=False) diff --git a/python/python/tests/test_namespace.py b/python/python/tests/test_namespace.py index b55da4d57..c70ba3d8e 100644 --- a/python/python/tests/test_namespace.py +++ b/python/python/tests/test_namespace.py @@ -16,6 +16,7 @@ import pyarrow as pa import lancedb from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError from lancedb.listing import AsyncListing, Listing +from lancedb.materialized_view import DEFINITION_FORMAT, DEFINITION_META_KEY from lancedb.namespace import _MAX_QUERY_K, _query_to_namespace_request from lancedb.query import Query from lancedb.table import AsyncTable, LanceTable @@ -359,7 +360,17 @@ class TestNamespaceConnection: def test_list_materialized_views(self): db = lancedb.connect_namespace("dir", {"root": self.temp_dir}) db.create_table("source", [{"id": 1}]) - db.create_materialized_view("view", "source", with_no_data=True) + # Model a stored view from an enabled build: listing must also work + # when local creation is disabled in the current build. + schema = pa.schema( + [("id", pa.int64())], + metadata={ + DEFINITION_META_KEY: json.dumps( + {"format": DEFINITION_FORMAT, "query": "SELECT id FROM source"} + ).encode() + }, + ) + db.create_table("view", schema=schema) names = db.list_materialized_views(page_limit=1) assert isinstance(names, Listing) @@ -872,7 +883,16 @@ class TestAsyncNamespaceConnection: async def test_list_materialized_views(self): db = lancedb.connect_namespace_async("dir", {"root": self.temp_dir}) await db.create_table("source", [{"id": 1}]) - await db.create_materialized_view("view", "source", with_no_data=True) + # Model a stored view independently of the local creation feature. + schema = pa.schema( + [("id", pa.int64())], + metadata={ + DEFINITION_META_KEY: json.dumps( + {"format": DEFINITION_FORMAT, "query": "SELECT id FROM source"} + ).encode() + }, + ) + await db.create_table("view", schema=schema) names = db.list_materialized_views(page_limit=1) assert isinstance(names, AsyncListing) diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 458f419ae..21fce93d3 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -5489,6 +5489,7 @@ async def test_async_search_runs_embedding_on_dedicated_executor( ) +@pytest.mark.experimental_functions def test_computed_column_declare_and_refresh(tmp_path): db = lancedb.connect(tmp_path) table = db.create_table("computed", [{"x": 1}, {"x": 2}]) @@ -5512,6 +5513,7 @@ def test_computed_column_rejects_transforms_and_computed_together(tmp_path): table.add_columns({"a": "x + 1"}, computed={"b": "x * 2"}) +@pytest.mark.experimental_functions def test_computed_column_blob_projection_inherits_semantics(tmp_path): schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) db = lancedb.connect(tmp_path) @@ -5535,6 +5537,7 @@ def test_computed_column_blob_projection_inherits_semantics(tmp_path): assert copied.to_pylist() == [b"hello", b"", None] +@pytest.mark.experimental_functions @pytest.mark.asyncio async def test_computed_column_async(tmp_path): db = await lancedb.connect_async(tmp_path) @@ -5553,6 +5556,7 @@ def test_function_errors_are_remote_only(tmp_path): table.function_errors() +@pytest.mark.experimental_functions def test_refresh_column_async_returns_job(tmp_path): db = lancedb.connect(tmp_path) table = db.create_table("computed_job", [{"x": 1}, {"x": 2}]) @@ -5581,6 +5585,7 @@ def test_refresh_column_async_returns_job(tmp_path): table.refresh_column_async("x") +@pytest.mark.experimental_functions @pytest.mark.asyncio async def test_refresh_column_async_job_async_table(tmp_path): db = await lancedb.connect_async(tmp_path) @@ -5597,6 +5602,7 @@ async def test_refresh_column_async_job_async_table(tmp_path): assert (await table.to_arrow())["tripled"].to_pylist() == [9] +@pytest.mark.experimental_functions @pytest.mark.parametrize("interval", [None, 300]) def test_computed_auto_refresh_options(tmp_path, interval): db = lancedb.connect(tmp_path) @@ -5635,6 +5641,7 @@ def test_auto_refresh_options_require_computed_columns(tmp_path): table.add_columns({"doubled": "x * 2"}, auto_refresh_enabled=True) +@pytest.mark.experimental_functions @pytest.mark.asyncio async def test_async_computed_auto_refresh_uses_server_interval(tmp_path): db = await lancedb.connect_async(tmp_path) diff --git a/python/src/lib.rs b/python/src/lib.rs index 3814f2caf..71662add1 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -132,6 +132,10 @@ pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(expr_lit, m)?)?; m.add_function(wrap_pyfunction!(expr_func, m)?)?; m.add("__version__", env!("CARGO_PKG_VERSION"))?; + m.add( + "_experimental_functions_enabled", + cfg!(feature = "experimental-functions"), + )?; // Give the shared runtime a coordinated, bounded shutdown at normal // process exit -- see `shutdown_runtime` and `runtime::shutdown` for why. py.import("atexit")? diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 9015bcdd7..62029bb92 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -137,6 +137,9 @@ pprof = { version = "0.14", features = ["flamegraph"] } [features] default = [] +# Local SQL computed columns and materialized-view creation/refresh. +# Remote clients and shared server primitives do not require this feature. +experimental-functions = [] # Native spatial indexes and queries. Remote RTree requests do not require this feature. geo = ["lance/geo"] aws = [ diff --git a/rust/lancedb/src/lib.rs b/rust/lancedb/src/lib.rs index 68521ff2c..8b1ad97c3 100644 --- a/rust/lancedb/src/lib.rs +++ b/rust/lancedb/src/lib.rs @@ -31,6 +31,11 @@ //! - `gcs` - Enable Google Cloud Storage object store support. //! - `oss` - Enable Alibaba Cloud OSS object store support. //! - `remote` - Enable remote client to connect to LanceDB cloud. +//! - `experimental-functions` - Opt in to local SQL computed-column declarations +//! and refresh, and local materialized-view creation and refresh. Disabled by +//! default, including in published Python and TypeScript packages. Remote +//! Cloud/Enterprise operations, reading existing data, metadata protections, +//! and ordinary eager `add_columns` transforms do not require this feature. //! - `huggingface` - Enable HuggingFace Hub integration for loading datasets from the Hub. //! - `fp16kernels` - Enable FP16 kernels for faster vector search on CPU. //! - `metrics` - Publish LanceDB's internal metrics through the @@ -229,6 +234,19 @@ pub use materialized_view::{ pub use metrics; pub use table::{FtsToken, Table}; +/// Gate local execution without hiding client types or metadata protections. +pub(crate) fn require_experimental_functions() -> Result<()> { + if !cfg!(feature = "experimental-functions") { + return Err(Error::NotSupported { + message: "local computed columns and materialized-view creation/refresh require \ + building LanceDB with the 'experimental-functions' Cargo feature; \ + official Python and TypeScript packages do not enable it" + .into(), + }); + } + Ok(()) +} + /// Tokenize a full-text search query using an explicit FTS tokenizer configuration. /// /// This does not require a table or FTS index. The tokenizer options are the diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs index 7ee76a160..b7124d930 100644 --- a/rust/lancedb/src/materialized_view.rs +++ b/rust/lancedb/src/materialized_view.rs @@ -9,6 +9,11 @@ //! query this version cannot maintain reads back as unrefreshable, not as a //! plain table. Queries, indexes and search work on the view unchanged; //! writes to its rows or stored columns are refused. +//! +//! Local creation and refresh require the `experimental-functions` Cargo +//! feature. Opening, reading, listing and dropping existing views, remote +//! client operations, and shared server commit primitives remain available +//! without it. mod grouped; pub use grouped::IVF_PARTITION; @@ -1245,6 +1250,7 @@ impl PreparedDeclaration { namespace_path: &[String], name: &str, ) -> Result { + crate::require_experimental_functions()?; ensure_no_source_cycle(&self.database, namespace_path, name, &self.definition).await?; let empty: Vec> = vec![]; @@ -1733,6 +1739,7 @@ impl CreateMaterializedViewBuilder { }) .await; } + crate::require_experimental_functions()?; Ok(Job::spawned(tokio::spawn(async move { self.execute_native().await.map(|_| ()) }))) @@ -1756,6 +1763,7 @@ impl CreateMaterializedViewBuilder { } async fn execute_native(self) -> Result { + crate::require_experimental_functions()?; if self.auto_refresh_enabled == Some(true) || self.auto_refresh_interval_seconds.is_some() { return Err(Error::NotSupported { message: "automatic materialized-view refresh requires Cloud or Enterprise" @@ -1879,6 +1887,7 @@ impl RefreshMaterializedViewBuilder { .refresh_materialized_view_async(self.source_version) .await; } + crate::require_experimental_functions()?; Ok(Job::spawned(tokio::spawn(async move { refresh::execute_refresh(&self.view.table, self.source_version).await }))) @@ -2061,6 +2070,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn refresh_always_rebuilds_without_source_row_ids() { let conn = connect("memory://").execute().await.unwrap(); let source = conn @@ -2124,6 +2137,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn a_view_cannot_lead_back_to_itself() { let conn = connect("memory://").execute().await.unwrap(); let rows = || record_batch!(("x", Int32, [1, 2])).unwrap(); @@ -2159,6 +2176,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn direct_writes_to_a_view_are_refused() { use arrow_schema::Schema; use datafusion::prelude::SessionContext; @@ -2295,6 +2316,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn indexes_and_optimize_still_apply_to_a_view() { use crate::index::Index; use crate::table::{CompactionOptions, OptimizeAction}; @@ -2339,6 +2364,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn computed_columns_of_a_view_are_still_filled_and_dropped() { use crate::table::computed_columns::{ COMPUTED_COLUMN_META_KEY, EXPRESSION_META_KEY, INPUTS_META_KEY, KIND_META_KEY, SQL_KIND, diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs index 6b4bb6d8a..1186ac576 100644 --- a/rust/lancedb/src/materialized_view/refresh.rs +++ b/rust/lancedb/src/materialized_view/refresh.rs @@ -72,6 +72,7 @@ pub(crate) async fn execute_refresh( view: &Table, pinned: Option, ) -> Result { + crate::require_experimental_functions()?; let native = view.as_native().ok_or_else(|| Error::NotSupported { message: "materialized views are supported only on local tables".into(), })?; @@ -576,6 +577,10 @@ mod tests { use crate::connect; #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn raced_first_full_refresh_lands_one_result() { let directory = tempfile::tempdir().unwrap(); let connection = connect(directory.path().to_str().unwrap()) diff --git a/rust/lancedb/src/table/add_columns.rs b/rust/lancedb/src/table/add_columns.rs index 4f9cbb772..0f13718fd 100644 --- a/rust/lancedb/src/table/add_columns.rs +++ b/rust/lancedb/src/table/add_columns.rs @@ -69,6 +69,9 @@ impl AddColumnsBuilder { /// renamed, retyped or dropped while a declaration reads it, since the /// expression names it. /// + /// Local declaration and refresh require the `experimental-functions` + /// Cargo feature. Remote operations do not require this feature. + /// /// On LanceDB Cloud and Enterprise the expression is planned by the /// server, and the refresh runs as a server job -- see /// [`Table::refresh_column_async`](super::Table::refresh_column_async). @@ -252,6 +255,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn computed_options_commit_with_declaration() { let table = table_with_two_columns("options").await; let version = table.version().await.unwrap(); @@ -313,6 +320,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn omitted_refresh_interval_stays_unset() { let table = table_with_two_columns("server_interval").await; let version = table.version().await.unwrap(); diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index fe4c17b1f..942c43d4e 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -2631,6 +2631,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_declare_infers_type_and_inputs() { let table = table_with_ints("declare_infers").await; let initial = table.version().await.unwrap(); @@ -2698,6 +2702,10 @@ mod tests { /// The binding reaches the schema only if `AllNulls` carries per-field /// metadata through the commit. The whole representation rests on it. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_all_nulls_preserves_field_metadata() { let table = table_with_ints("metadata_survives").await; add_computed(&table, &[("doubled".into(), "x * 2".into())]) @@ -2722,6 +2730,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_declared_column_is_all_null() { let table = table_with_ints("declare_is_null").await; add_computed(&table, &[("doubled".into(), "x * 2".into())]) @@ -2746,6 +2758,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_unknown_column_fails_at_declare_time() { let table = table_with_ints("unknown_input").await; let err = add_computed(&table, &[("bad".into(), "missing + 1".into())]) @@ -2758,6 +2774,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_unparsable_expression_fails_at_declare_time() { let table = table_with_ints("bad_syntax").await; let err = add_computed(&table, &[("bad".into(), "x *".into())]) @@ -2778,6 +2798,10 @@ mod tests { /// resolution is missing. When a registry-aware planner exists this /// becomes a supported declaration rather than a new API. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_unregistered_function_is_rejected_for_now() { let table = table_with_ints("udf_not_yet").await; let err = add_computed(&table, &[("vec".into(), "embed(x)".into())]) @@ -2795,6 +2819,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_existing_column_name_is_rejected() { let table = table_with_ints("name_taken").await; let err = add_computed(&table, &[("x".into(), "x * 2".into())]) @@ -2805,6 +2833,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_constant_expression_needs_no_inputs() { let table = table_with_ints("constant").await; add_computed(&table, &[("answer".into(), "42".into())]) @@ -2817,6 +2849,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_multiple_columns_in_one_commit() { let table = table_with_ints("multi").await; let initial = table.version().await.unwrap(); @@ -2839,6 +2875,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_duplicate_declaration_in_one_call_is_rejected() { let table = table_with_ints("dupe").await; let err = add_computed( @@ -2857,6 +2897,10 @@ mod tests { /// A batch may build on itself: one commit, and the later entry's inputs /// name the earlier one. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_declaration_may_read_one_declared_before_it() { let table = table_with_ints("chain").await; let before = table.version().await.unwrap(); @@ -2909,6 +2953,10 @@ mod tests { /// Built-in functions type the column the same way an operator does. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_builtin_function_inference() { let conn = connect("memory://").execute().await.unwrap(); let batch = record_batch!(("name", Utf8, ["ada", "grace"]), ("n", Int32, [-1, 2])).unwrap(); @@ -2959,6 +3007,10 @@ mod tests { /// an ordinary column. Reported as absent it would be refreshable by /// nothing and redeclarable over, silently. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_unrecognized_kind_is_reported_rather_than_hidden() { let table = table_with_ints("foreign_kind").await; super::add_foreign_kind(&table, "embedding", "udf").await; @@ -3002,6 +3054,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_inputs_are_deduplicated_and_sorted() { let conn = connect("memory://").execute().await.unwrap(); let batch = record_batch!(("b", Int32, [1, 2]), ("a", Int32, [3, 4])).unwrap(); @@ -3018,6 +3074,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_dropping_an_input_is_refused() { let table = table_with_ints("drop_input").await; add_computed(&table, &[("doubled".into(), "x * 2".into())]) @@ -3032,6 +3092,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_renaming_an_input_is_refused() { let table = table_with_ints("rename_input").await; add_computed(&table, &[("doubled".into(), "x * 2".into())]) @@ -3050,6 +3114,10 @@ mod tests { /// Nothing resolves against nullability, so it is not a rebinding. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_altering_an_input_nullability_is_allowed() { let table = table_with_ints("nullable_input").await; add_computed(&table, &[("doubled".into(), "x * 2".into())]) @@ -3066,6 +3134,10 @@ mod tests { /// the counting and writing passes, so the declared value is incoherent. /// Refused at declare time. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_volatile_expression_is_refused() { let table = table_with_ints("volatile_expr").await; let err = add_computed(&table, &[("maybe".into(), "random() < 0.5".into())]) @@ -3082,6 +3154,10 @@ mod tests { /// constant, but the stored SQL still names `x`, so the recorded inputs /// must too -- otherwise dropping `x` is allowed and refresh breaks. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_inputs_survive_expression_optimization() { let table = table_with_ints("optimized_inputs").await; add_computed(&table, &[("flag".into(), "true OR x > 0".into())]) @@ -3099,6 +3175,10 @@ mod tests { /// The gate's reproducer: casting a computed column rewrites the field /// without its metadata, silently destroying the declaration. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_retyping_the_computed_column_is_refused() { use arrow_schema::DataType as ArrowDataType; @@ -3123,6 +3203,10 @@ mod tests { /// A declaration cannot be edited, fabricated or erased through field /// metadata: it is validated as a whole at declare time. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_declaration_metadata_is_immutable() { use crate::table::FieldMetadataUpdate; @@ -3170,6 +3254,10 @@ mod tests { /// The gate's reproducer: only refresh materializes a declared column; /// a direct write would store an arbitrary durable value. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_computed_column_cannot_be_written_directly() { let table = table_with_ints("direct_write").await; add_computed(&table, &[("doubled".into(), "x * 2".into())]) @@ -3211,6 +3299,10 @@ mod tests { /// The gate's reproducer: the reciprocal of the declare-under-spec check. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_installing_an_lsm_spec_over_computed_columns_is_refused() { use crate::table::RTEWriteSpec; @@ -3279,6 +3371,10 @@ mod tests { /// The gate's reproducer: SQL INSERT is a write path too. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_sql_insert_cannot_write_a_computed_column() { use datafusion::prelude::SessionContext; @@ -3378,6 +3474,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_sql_insert_omitting_computed_is_allowed() { use datafusion::prelude::SessionContext; @@ -3404,6 +3504,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_nested_computed_field_cannot_be_renamed() { let table = table_with_ints("computed_struct_rename").await; add_computed(&table, &[("payload".into(), "named_struct('a', x)".into())]) @@ -3422,6 +3526,10 @@ mod tests { /// Stale handles must not commit the computed/RTE state in either order. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_stale_handles_cannot_mix_computed_and_lsm() { use crate::table::RTEWriteSpec; @@ -3472,6 +3580,10 @@ mod tests { /// unset, retained SSTable rows survive without a live spec. The catch-up /// flag is the durable marker; declaration refuses on it. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_unset_with_retained_lsm_rows_cannot_admit_a_declaration() { use crate::table::RTEWriteSpec; use arrow_array::{Int64Array, RecordBatchIterator}; @@ -3523,6 +3635,10 @@ mod tests { /// A declaration does not read itself, so it travels with its binding. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_dropping_the_computed_column_is_allowed() { let table = table_with_ints("drop_computed").await; add_computed(&table, &[("doubled".into(), "x * 2".into())]) @@ -4663,6 +4779,10 @@ mod tests { /// downstream binding's immutable input schema must not record them, or /// the first upstream refresh strands the table. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_computed_input_survives_upstream_refresh() { use arrow_array::{Int32Array, Int64Array, RecordBatch}; diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index 8f7f576ff..c5eadeeaf 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -77,6 +77,7 @@ pub(crate) async fn execute_refresh_column( table: &NativeTable, column: &str, ) -> Result { + crate::require_experimental_functions()?; Ok(execute_refresh_column_with_source(table, column) .await? .result) @@ -300,6 +301,7 @@ pub(crate) async fn execute_refresh_column_async( table: &NativeTable, column: &str, ) -> Result> { + crate::require_experimental_functions()?; // Validate before spawning so bad input is reported by this call rather // than only by the job. table.dataset.ensure_mutable()?; @@ -731,6 +733,10 @@ mod tests { /// must not bake zeros from `a`'s placeholder null. It is refused, and /// names the input, until `a` is filled -- after every append too. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_dependent_refresh_refuses_an_unfilled_input() { let table = table_with("dependent_refresh_order", vec![1, 2, 3]).await; table @@ -766,6 +772,10 @@ mod tests { /// declaration metadata and the dependency check: the recorded inputs /// are matched by name, never re-parsed as SQL. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_dependent_refresh_handles_awkward_column_names() { use arrow_array::{Int32Array, StructArray}; use arrow_schema::{DataType, Field, Fields}; @@ -820,6 +830,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_fills_a_declared_column() { let table = table_with("refresh_fills", vec![1, 2, 3]).await; let declared = declare_doubled(&table).await.unwrap(); @@ -848,6 +862,10 @@ mod tests { /// Values written after the last refresh must be reachable by another one. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_fills_rows_appended_since_the_last_refresh() { let table = table_with("refresh_appended", vec![1, 2]).await; declare_doubled(&table).await.unwrap(); @@ -868,6 +886,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_with_nothing_to_fill() { let table = table_with("refresh_noop", vec![1, 2, 3]).await; declare_doubled(&table).await.unwrap(); @@ -884,6 +906,10 @@ mod tests { /// A branch inherits the freshness main recorded: its first refresh /// fills nothing, and only its own appends after that. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_branch_inherits_freshness() { let dir = tempfile::tempdir().unwrap(); let conn = connect(dir.path().to_str().unwrap()) @@ -919,6 +945,10 @@ mod tests { /// A shallow clone inherits the freshness its source recorded. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_shallow_clone_inherits_freshness() { let dir = tempfile::tempdir().unwrap(); let conn = connect(dir.path().to_str().unwrap()) @@ -953,6 +983,10 @@ mod tests { /// settles at once instead of re-selecting the same rows forever: the /// second refresh finds the fragment signed and moves nothing. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_converges_on_a_null_result() { let table = table_with("refresh_null_result", vec![1, 2, 3]).await; let declared = table @@ -976,6 +1010,10 @@ mod tests { /// A filled row whose input moved is recomputed: the update rewrites /// the row into a fragment the stamp never signed, and only that one. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_recomputes_a_row_whose_input_moved() { let table = table_with("refresh_mutation", vec![1, 2]).await; declare_doubled(&table).await.unwrap(); @@ -1009,6 +1047,10 @@ mod tests { /// Each stamp is a sidecar under `_computed/`; pruning old versions /// removes the sidecars only they referenced, and keeps the current one. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_pruning_drops_the_sidecars_of_pruned_versions() { let dir = tempfile::tempdir().unwrap(); let conn = connect(dir.path().to_str().unwrap()) @@ -1050,6 +1092,10 @@ mod tests { /// Absolute timestamp pruning applies the same computed-column sidecar /// cleanup as duration-based pruning. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_absolute_pruning_drops_the_sidecars_of_pruned_versions() { let dir = tempfile::tempdir().unwrap(); let conn = connect(dir.path().to_str().unwrap()) @@ -1087,6 +1133,10 @@ mod tests { /// A deleted row is never computed and the rows that stay keep their /// values: a delete recomputes nothing and stamps nothing. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_delete_recomputes_nothing() { let table = table_with("refresh_delete", vec![1, 2, 3]).await; declare_doubled(&table).await.unwrap(); @@ -1106,6 +1156,10 @@ mod tests { /// signed ones is fresh: the refresh recomputes nothing and only records /// the new fragment. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_compaction_of_signed_fragments_recomputes_nothing() { let table = table_with("refresh_compact_signed", vec![1, 2]).await; declare_doubled(&table).await.unwrap(); @@ -1136,6 +1190,10 @@ mod tests { /// refresh keeps the null-fill contract and enrolls what it read from; /// from then on a moved input is recomputed like any other. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_an_unsigned_column_is_enrolled_by_its_first_refresh() { let table = table_with("refresh_legacy", vec![1, 2]).await; declare_doubled(&table).await.unwrap(); @@ -1188,6 +1246,10 @@ mod tests { /// column as null behind a covering data file. Those rows are still /// unfilled and a later refresh has to reach them. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_update_before_the_first_refresh() { let table = table_with("refresh_update_first", vec![1]).await; declare_doubled(&table).await.unwrap(); @@ -1203,6 +1265,10 @@ mod tests { /// for any of its rows: every live row is recomputed, the moved one /// included. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_compaction_of_an_unsigned_fragment_recomputes_it() { let table = table_with("refresh_mixed", vec![1, 2]).await; declare_doubled(&table).await.unwrap(); @@ -1236,6 +1302,10 @@ mod tests { /// computed column. Compaction cannot certify it, so the product is /// recomputed and the supplied value replaced. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_raw_append_values_are_not_trusted_after_compaction() { use arrow_array::RecordBatchIterator; use lance::Dataset; @@ -1288,6 +1358,10 @@ mod tests { /// An appended fragment holds no values, so compacting it into a signed /// one leaves the product fresh: only the appended rows are filled. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_compaction_with_an_appended_fragment_fills_only_its_rows() { let table = table_with("refresh_preserves", vec![1, 2]).await; declare_doubled(&table).await.unwrap(); @@ -1311,6 +1385,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_leaves_deleted_rows_alone() { let table = table_with("refresh_deleted", vec![1, 2, 3, 4]).await; declare_doubled(&table).await.unwrap(); @@ -1325,6 +1403,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_a_constant_expression() { let table = table_with("refresh_constant", vec![1, 2, 3]).await; table @@ -1341,6 +1423,10 @@ mod tests { /// A name needing quotes reaches the evaluator intact: it is carried as a /// projection alias, never spliced into SQL text. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_a_column_whose_name_needs_quoting() { let table = table_with("refresh_quoted", vec![1, 2, 3]).await; table @@ -1362,6 +1448,10 @@ mod tests { /// the probe buffers only until the first gained value and the rest flows /// through write_columns a batch at a time. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_streams_a_multi_batch_fragment() { let values: Vec = (0..20_000).collect(); let table = table_with("refresh_multi_batch", values.clone()).await; @@ -1381,6 +1471,10 @@ mod tests { /// The gate's reproducer: the commit must reuse the configured session, /// or registrations and caches vanish from the handle after a refresh. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_preserves_the_configured_session() { let session = Arc::new(lance::session::Session::default()); let conn = crate::connect("memory://") @@ -1404,6 +1498,10 @@ mod tests { /// The async form's job settles with the fill visible, like /// create_index's execute_async. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_async_job_waits_for_the_fill() { let table = table_with("refresh_async", vec![1, 2, 3]).await; declare_doubled(&table).await.unwrap(); @@ -1426,6 +1524,10 @@ mod tests { /// Bad input is reported by the call, not by the job. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_async_rejects_bad_input_before_spawning() { let table = table_with("refresh_async_bad", vec![1, 2, 3]).await; @@ -1437,6 +1539,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_async_job_reports_success_to_every_waiter() { let table = table_with("refresh_async_waiters", vec![1, 2]).await; declare_doubled(&table).await.unwrap(); @@ -1449,6 +1555,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_rejects_a_plain_column() { let table = table_with("refresh_plain", vec![1, 2, 3]).await; let err = table.refresh_column("x").await.unwrap_err(); @@ -1456,6 +1566,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_rejects_an_unknown_column() { let table = table_with("refresh_missing", vec![1, 2, 3]).await; let err = table.refresh_column("nope").await.unwrap_err(); @@ -1465,6 +1579,10 @@ mod tests { /// The gate's reproducer: a poison value in a deleted row must not /// abort filling the live rows, since nobody can read it. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_deleted_rows_value_is_never_evaluated() { let table = table_with("refresh_deleted_poison", vec![1, 0]).await; table @@ -1484,6 +1602,10 @@ mod tests { /// input did not move is not: the untouched fragment is never read, so /// its poison input is never reached. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_only_a_moved_rows_value_is_re_evaluated() { let table = table_with("refresh_filled_poison", vec![1, 0]).await; table @@ -1515,6 +1637,10 @@ mod tests { /// The gate's reproducer: the old internal projection alias is an /// ordinary column name; a computed column may use it. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_a_column_named_like_the_old_alias() { let table = table_with("refresh_alias_name", vec![1, 2]).await; table @@ -1537,6 +1663,10 @@ mod tests { /// this pins behaviorally; the memory bound is structural -- the fill /// stream retains no batches at all. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_fills_a_late_gain_fragment() { let values: Vec = (0..20_000).collect(); let table = table_with("refresh_late_gain", values).await; @@ -1562,6 +1692,10 @@ mod tests { /// The gate's reproducer: a nested input declares, refreshes, and guards /// its root against invalidating schema changes. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_a_nested_input_declares_and_refreshes() { use arrow_array::{Int32Array, StructArray}; use arrow_schema::{DataType, Field, Fields}; @@ -1636,6 +1770,10 @@ mod tests { /// schema_evolution tests); refresh's own check covers a dataset another /// writer left in that state. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_refuses_a_foreign_lsm_state() { use crate::table::RTEWriteSpec; @@ -1669,6 +1807,10 @@ mod tests { /// After catch-up activation and unset, no spec remains but the catch-up /// flag still marks retained SSTable rows; refresh refuses on the flag. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_refuses_retained_catchup_state() { use crate::table::RTEWriteSpec; @@ -1719,6 +1861,10 @@ mod tests { /// A declaration of a kind this version cannot evaluate is refused by /// name, rather than mistaken for a plain column or fed to the SQL path. #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_rejects_a_kind_it_cannot_evaluate() { let table = table_with("refresh_foreign", vec![1, 2, 3]).await; super::super::computed_columns::add_foreign_kind(&table, "embedding", "udf").await; @@ -1754,6 +1900,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_inherits_and_publishes_blob_output() { use arrow_array::UInt64Array; use lance_arrow::{ @@ -1887,6 +2037,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_inherits_nested_struct_blob_input() { use arrow_array::{Int32Array, StructArray, UInt64Array}; use arrow_schema::{DataType, Field, Fields, Schema}; @@ -1952,6 +2106,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_preserves_list_shape_when_materializing_blob_input() { use arrow_array::{Int32Array, ListArray}; use arrow_buffer::{OffsetBuffer, ScalarBuffer}; @@ -2013,6 +2171,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "experimental-functions"), + ignore = "requires experimental-functions" + )] async fn test_refresh_inherits_external_blob_input() { use arrow_array::{Int32Array, StringArray, UInt64Array}; use arrow_schema::{DataType, Field, Schema}; diff --git a/rust/lancedb/src/table/schema_evolution.rs b/rust/lancedb/src/table/schema_evolution.rs index f80e89ee7..d78fc5102 100644 --- a/rust/lancedb/src/table/schema_evolution.rs +++ b/rust/lancedb/src/table/schema_evolution.rs @@ -133,6 +133,8 @@ pub(crate) async fn execute_declare( ) -> Result { use lance::dataset::mem_wal::DatasetMemWalExt; + crate::require_experimental_functions()?; + // An RTE write spec keeps visible rows in tiers refresh cannot reach; // checked against latest committed state, not this handle's snapshot. table.checkout_latest().await?; @@ -491,12 +493,14 @@ mod tests { .execute() .await .unwrap(); - table - .add_columns() - .computed("derived", "spare * 2") - .execute() - .await - .unwrap(); + if cfg!(feature = "experimental-functions") { + table + .add_columns() + .computed("derived", "spare * 2") + .execute() + .await + .unwrap(); + } table .update_field_metadata(&[metadata_update("eager")]) .await @@ -510,7 +514,10 @@ mod tests { let schema = table.schema().await.unwrap(); ensure_supported_function_metadata(&schema).unwrap(); assert_eq!(function_bindings(&schema).unwrap().len(), 1); - assert!(schema.field_with_name("derived").is_ok()); + assert_eq!( + schema.field_with_name("derived").is_ok(), + cfg!(feature = "experimental-functions") + ); assert!(schema.field_with_name("moved").is_err()); } @@ -578,14 +585,16 @@ mod tests { .await .unwrap_err(), ); - bound( - table - .add_columns() - .computed(column, "1") - .execute() - .await - .unwrap_err(), - ); + if cfg!(feature = "experimental-functions") { + bound( + table + .add_columns() + .computed(column, "1") + .execute() + .await + .unwrap_err(), + ); + } } bound( table diff --git a/rust/lancedb/tests/experimental_functions.rs b/rust/lancedb/tests/experimental_functions.rs new file mode 100644 index 000000000..f7cd022e3 --- /dev/null +++ b/rust/lancedb/tests/experimental_functions.rs @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +#![cfg(not(feature = "experimental-functions"))] +#![recursion_limit = "256"] + +use std::collections::HashMap; + +use arrow_array::record_batch; +use futures::TryStreamExt; +use lancedb::materialized_view::{ + DEFINITION_META_KEY, definition_metadata_from_sql, prepare_declaration, +}; +use lancedb::table::NewColumnTransform; +use lancedb::table::computed_columns::{ + COMPUTED_COLUMN_META_KEY, EXPRESSION_META_KEY, INPUTS_META_KEY, KIND_META_KEY, computed_columns, +}; +use lancedb::{Error, MaterializedView, Result, connect}; + +fn assert_disabled(result: Result) { + match result { + Err(Error::NotSupported { message }) => { + assert!(message.contains("experimental-functions"), "{message}"); + } + _ => panic!("expected the experimental-functions feature error"), + } +} + +#[tokio::test] +async fn computed_operations_fail_before_mutation_or_job_creation() { + let db = connect("memory://").execute().await.unwrap(); + let table = db + .create_table("source", record_batch!(("x", Int32, [1, 2])).unwrap()) + .execute() + .await + .unwrap(); + let version = table.version().await.unwrap(); + assert_disabled( + table + .add_columns() + .computed("doubled", "x * 2") + .execute() + .await, + ); + assert_disabled(table.refresh_column("x").await); + assert_disabled(table.refresh_column_async("x").await); + assert_eq!(table.version().await.unwrap(), version); + assert_eq!(table.schema().await.unwrap().fields().len(), 1); + + // The eager SQL transform remains an ordinary schema-evolution operation. + table + .add_columns() + .transform(NewColumnTransform::SqlExpressions(vec![( + "eager".into(), + "x * 2".into(), + )])) + .execute() + .await + .unwrap(); + assert_eq!( + table + .count_rows(Some("eager = x * 2".into())) + .await + .unwrap(), + 2 + ); +} + +#[tokio::test] +async fn existing_computed_metadata_remains_readable_and_protected() { + let db = connect("memory://").execute().await.unwrap(); + let table = db + .create_table( + "source", + record_batch!(("x", Int32, [1, 2]), ("doubled", Int32, [2, 4])).unwrap(), + ) + .execute() + .await + .unwrap(); + // Model a table previously written by an enabled build, without enabling + // the feature in this test binary through Cargo feature unification. + let wrapper = table.dataset().unwrap(); + let mut dataset = wrapper.get().await.unwrap().as_ref().clone(); + let field_id = dataset.schema().field("doubled").unwrap().id as u32; + dataset + .replace_field_metadata(vec![( + field_id, + HashMap::from([ + (COMPUTED_COLUMN_META_KEY.into(), "true".into()), + (KIND_META_KEY.into(), "sql".into()), + (EXPRESSION_META_KEY.into(), "x * 2".into()), + (INPUTS_META_KEY.into(), "[\"x\"]".into()), + ]), + )]) + .await + .unwrap(); + wrapper.update(dataset); + let version = table.version().await.unwrap(); + assert_eq!(computed_columns(&table.schema().await.unwrap()).len(), 1); + assert_eq!( + table + .count_rows(Some("doubled = x * 2".into())) + .await + .unwrap(), + 2 + ); + assert_disabled(table.refresh_column("doubled").await); + assert_disabled(table.refresh_column_async("doubled").await); + assert!( + table + .update() + .column("doubled", "42") + .execute() + .await + .is_err() + ); + assert!(table.drop_columns(&["x"]).await.is_err()); + assert_eq!(table.version().await.unwrap(), version); + table.drop_columns(&["doubled"]).await.unwrap(); +} + +#[tokio::test] +async fn views_cannot_be_created_or_refreshed_but_existing_data_is_accessible() { + let db = connect("memory://").execute().await.unwrap(); + let source = db + .create_table("source", record_batch!(("x", Int32, [1, 2])).unwrap()) + .execute() + .await + .unwrap(); + for with_no_data in [false, true] { + assert_disabled( + db.create_materialized_view("blocked", "source") + .with_no_data(with_no_data) + .execute() + .await, + ); + assert_disabled( + db.create_materialized_view("blocked", "source") + .with_no_data(with_no_data) + .execute_async() + .await, + ); + } + let prepared = prepare_declaration(&source, None, None, None) + .await + .unwrap(); + assert_disabled(prepared.clone().create("blocked").await); + assert_disabled(prepared.create_in(&[], "blocked").await); + assert!(matches!( + db.open_table("blocked").execute().await, + Err(Error::TableNotFound { .. }) + )); + + let table = db + .create_table("existing", record_batch!(("x", Int32, [1, 2])).unwrap()) + .execute() + .await + .unwrap(); + let wrapper = table.dataset().unwrap(); + let mut dataset = wrapper.get().await.unwrap().as_ref().clone(); + dataset + .update_schema_metadata(vec![( + DEFINITION_META_KEY.into(), + Some(definition_metadata_from_sql("SELECT x FROM source")), + )]) + .await + .unwrap(); + wrapper.update(dataset); + let version = table.version().await.unwrap(); + let view = MaterializedView::from_table(table.clone()).await.unwrap(); + assert_disabled(view.refresh().execute().await); + assert_disabled(view.refresh().execute_async().await); + assert_eq!(table.version().await.unwrap(), version); + assert_eq!(view.table().count_rows(None).await.unwrap(), 2); + assert_eq!( + db.open_materialized_view("existing") + .await + .unwrap() + .definition_sql(), + "SELECT x FROM source" + ); + assert_eq!( + db.list_materialized_views(Default::default()) + .try_collect::>() + .await + .unwrap(), + vec!["existing"] + ); + assert!( + table + .add(record_batch!(("x", Int32, [3])).unwrap()) + .execute() + .await + .is_err() + ); + assert!(table.delete("true").await.is_err()); + db.drop_materialized_view("existing", &[]).await.unwrap(); +}