diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index 3fa3b08db..763c68fc8 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -71,7 +71,12 @@ Add new columns with defined values. #### Parameters -* **newColumnTransforms**: `Field`<`any`> \| `Field`<`any`>[] \| `Schema`<`any`> \| [`AddColumnsSql`](../interfaces/AddColumnsSql.md)[] +* **newColumnTransforms**: + \| `Field`<`any`> + \| `Field`<`any`>[] + \| `Schema`<`any`> + \| [`AddColumnsSql`](../interfaces/AddColumnsSql.md)[] + \| `object` Either: - An array of objects with column names and SQL expressions to calculate values - A single Arrow Field defining one column with its data type (column will be initialized with null values) @@ -718,6 +723,28 @@ for await (const batch of table.query()) { *** +### refreshColumn() + +```ts +abstract refreshColumn(column): Promise +``` + +Compute and store values for a computed column's unfilled rows. + +#### Parameters + +* **column**: `string` + The name of the computed column to fill. + +#### Returns + +`Promise`<[`RefreshColumnResult`](../interfaces/RefreshColumnResult.md)> + +A promise that resolves to the +number of rows filled and the new version number of the table. + +*** + ### restore() ```ts diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 7455a81ce..bd2ca54b5 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -105,6 +105,7 @@ - [OptimizeOptions](interfaces/OptimizeOptions.md) - [OptimizeStats](interfaces/OptimizeStats.md) - [QueryExecutionOptions](interfaces/QueryExecutionOptions.md) +- [RefreshColumnResult](interfaces/RefreshColumnResult.md) - [RemovalStats](interfaces/RemovalStats.md) - [RenameTableOptions](interfaces/RenameTableOptions.md) - [RestNamespaceConfig](interfaces/RestNamespaceConfig.md) diff --git a/docs/src/js/interfaces/RefreshColumnResult.md b/docs/src/js/interfaces/RefreshColumnResult.md new file mode 100644 index 000000000..d2854fda6 --- /dev/null +++ b/docs/src/js/interfaces/RefreshColumnResult.md @@ -0,0 +1,23 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / RefreshColumnResult + +# Interface: RefreshColumnResult + +## Properties + +### rowsFilled + +```ts +rowsFilled: number; +``` + +*** + +### version + +```ts +version: number; +``` diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index d263d9cab..bc495d24b 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -3340,3 +3340,45 @@ describe("LSM merge insert", () => { await expect(table.query().useLsm(true).toArray()).rejects.toThrow(); }); }); + +describe("computed columns", () => { + let tmpDir: tmp.DirResult; + beforeEach(() => { + tmpDir = tmp.dirSync({ unsafeCleanup: true }); + }); + afterEach(() => tmpDir.removeCallback()); + + it("declares a column and fills it on refresh", async () => { + const db = await connect(tmpDir.name); + const table = await db.createTable("computed", [{ x: 1 }, { x: 2 }]); + + await table.addColumns({ + computed: [{ name: "doubled", valueSql: "x * 2" }], + }); + let rows = await table.query().toArray(); + expect(rows.map((r) => r.doubled)).toEqual([null, null]); + + const result = await table.refreshColumn("doubled"); + expect(result.rowsFilled).toBe(2); + + rows = await table.query().toArray(); + expect(rows.map((r) => r.doubled).sort()).toEqual([2, 4]); + }); + + it("fills rows added since the last refresh", async () => { + const db = await connect(tmpDir.name); + const table = await db.createTable("computed_append", [{ x: 1 }]); + + await table.addColumns({ + computed: [{ name: "doubled", valueSql: "x * 2" }], + }); + await table.refreshColumn("doubled"); + await table.add([{ x: 5 }]); + + const result = await table.refreshColumn("doubled"); + expect(result.rowsFilled).toBe(1); + + const rows = await table.query().toArray(); + expect(rows.map((r) => r.doubled).sort()).toEqual([10, 2]); + }); +}); diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index 319222421..9f2e97989 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -50,6 +50,7 @@ export { MergeResult, AddResult, AddColumnsResult, + RefreshColumnResult, AlterColumnsResult, UpdateFieldMetadataResult, DeleteResult, diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 04705475b..141cefd46 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -33,6 +33,7 @@ import { Job, Branches as NativeBranches, OptimizeStats, + RefreshColumnResult, TableStatistics, Tags, UpdateFieldMetadataResult, @@ -534,9 +535,22 @@ export abstract class Table { * containing the new version number of the table after adding the columns. */ abstract addColumns( - newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema, + newColumnTransforms: + | AddColumnsSql[] + | Field + | Field[] + | Schema + | { computed: AddColumnsSql[] }, ): Promise; + /** + * Compute and store values for a computed column's unfilled rows. + * @param {string} column The name of the computed column to fill. + * @returns {Promise} A promise that resolves to the + * number of rows filled and the new version number of the table. + */ + abstract refreshColumn(column: string): Promise; + /** * Alter the name or nullability of columns. * @param {ColumnAlteration[]} columnAlterations One or more alterations to @@ -1088,8 +1102,22 @@ export class LocalTable extends Table { // TODO: Support BatchUDF async addColumns( - newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema, + newColumnTransforms: + | AddColumnsSql[] + | Field + | Field[] + | Schema + | { computed: AddColumnsSql[] }, ): Promise { + // Columns defined by an expression are declared, not materialized here. + if ( + typeof newColumnTransforms === "object" && + !Array.isArray(newColumnTransforms) && + "computed" in newColumnTransforms + ) { + return await this.inner.addComputedColumns(newColumnTransforms.computed); + } + // Handle single Field -> convert to array of Fields if (newColumnTransforms instanceof Field) { newColumnTransforms = [newColumnTransforms]; @@ -1124,6 +1152,10 @@ export class LocalTable extends Table { throw new Error("Invalid input type for addColumns"); } + async refreshColumn(column: string): Promise { + return await this.inner.refreshColumn(column); + } + async alterColumns( columnAlterations: ColumnAlteration[], ): Promise { diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index c4ece20e2..40ed7d9f0 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -347,6 +347,30 @@ impl Table { Ok(res.into()) } + #[napi(catch_unwind)] + pub async fn add_computed_columns( + &self, + columns: Vec, + ) -> napi::Result { + let table = self.inner_ref()?; + let mut builder = table.add_columns(); + for column in columns { + builder = builder.computed(column.name, column.value_sql); + } + let res = builder.execute().await.default_error()?; + Ok(res.into()) + } + + #[napi(catch_unwind)] + pub async fn refresh_column(&self, column: String) -> napi::Result { + let res = self + .inner_ref()? + .refresh_column(column) + .await + .default_error()?; + Ok(res.into()) + } + #[napi(catch_unwind)] pub async fn add_columns_with_schema( &self, @@ -1196,6 +1220,21 @@ pub struct AddColumnsResult { pub version: i64, } +#[napi(object)] +pub struct RefreshColumnResult { + pub rows_filled: i64, + pub version: i64, +} + +impl From for RefreshColumnResult { + fn from(value: lancedb::table::RefreshColumnResult) -> Self { + Self { + rows_filled: value.rows_filled as i64, + version: value.version as i64, + } + } +} + impl From for AddColumnsResult { fn from(value: lancedb::table::AddColumnsResult) -> Self { Self {