diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index 763c68fc8..278559cc4 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -69,6 +69,19 @@ abstract addColumns(newColumnTransforms): Promise Add new columns with defined values. +The `{ computed }` form stores the expression rather than evaluating it +now: the column is committed with no values, and rows get them from +[Table#refreshColumn](Table.md#refreshcolumn). Declaring one therefore costs the same on a +large table as on an empty one. + +A refresh does not revisit rows it has already filled, so mutating an +input leaves the value computed at fill time; recomputing means dropping +the column and declaring it again. While a declaration reads a column, +that column cannot be renamed, retyped or dropped. + +Computed columns are local-only: LanceDB Cloud and Enterprise reject a +declaration. + #### Parameters * **newColumnTransforms**: @@ -82,6 +95,7 @@ Add new columns with defined values. - A single Arrow Field defining one column with its data type (column will be initialized with null values) - An array of Arrow Fields defining columns with their data types (columns will be initialized with null values) - An Arrow Schema defining columns with their data types (columns will be initialized with null values) + - `{ computed }`, declaring columns defined by a SQL expression whose type and inputs are derived from it #### Returns @@ -90,6 +104,13 @@ Add new columns with defined values. A promise that resolves to an object containing the new version number of the table after adding the columns. +#### Example + +```ts +await table.addColumns({ computed: [{ name: "doubled", valueSql: "x * 2" }] }); +const { rowsFilled } = await table.refreshColumn("doubled"); +``` + *** ### alterColumns() @@ -729,7 +750,11 @@ for await (const batch of table.query()) { abstract refreshColumn(column): Promise ``` -Compute and store values for a computed column's unfilled rows. +Fill the rows of a computed column that hold no value yet. + +Rows appended since the last refresh are filled by the next one; rows +already filled are left as they are, so the call is idempotent and does +not observe a mutated input. Local tables only. #### Parameters diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 141cefd46..5b8d00076 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -526,13 +526,32 @@ export abstract class Table { abstract vectorSearch(vector: IntoVector | MultiVector): VectorQuery; /** * Add new columns with defined values. + * + * The `{ computed }` form stores the expression rather than evaluating it + * now: the column is committed with no values, and rows get them from + * {@link Table#refreshColumn}. Declaring one therefore costs the same on a + * large table as on an empty one. + * + * A refresh does not revisit rows it has already filled, so mutating an + * input leaves the value computed at fill time; recomputing means dropping + * the column and declaring it again. While a declaration reads a column, + * that column cannot be renamed, retyped or dropped. + * + * Computed columns are local-only: LanceDB Cloud and Enterprise reject a + * declaration. * @param {AddColumnsSql[] | Field | Field[] | Schema} newColumnTransforms Either: * - An array of objects with column names and SQL expressions to calculate values * - A single Arrow Field defining one column with its data type (column will be initialized with null values) * - An array of Arrow Fields defining columns with their data types (columns will be initialized with null values) * - An Arrow Schema defining columns with their data types (columns will be initialized with null values) + * - `{ computed }`, declaring columns defined by a SQL expression whose type and inputs are derived from it * @returns {Promise} A promise that resolves to an object * containing the new version number of the table after adding the columns. + * @example + * ```ts + * await table.addColumns({ computed: [{ name: "doubled", valueSql: "x * 2" }] }); + * const { rowsFilled } = await table.refreshColumn("doubled"); + * ``` */ abstract addColumns( newColumnTransforms: @@ -544,7 +563,11 @@ export abstract class Table { ): Promise; /** - * Compute and store values for a computed column's unfilled rows. + * Fill the rows of a computed column that hold no value yet. + * + * Rows appended since the last refresh are filled by the next one; rows + * already filled are left as they are, so the call is idempotent and does + * not observe a mutated input. Local tables only. * @param {string} column The name of the computed column to fill. * @returns {Promise} A promise that resolves to the * number of rows filled and the new version number of the table. diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 3ca245e55..db25c4ebc 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -1938,11 +1938,68 @@ class Table(ABC): Alternatively, a pyarrow Field or Schema can be provided to add new columns with the specified data types. The new columns will be initialized with null values. + computed: Dict[str, str], optional + A map of column name to a SQL expression defining the column. The + column's type and inputs are derived from the expression, so no + data type is supplied. + + Unlike ``transforms``, the expression is stored rather than + evaluated now: the column is committed with no values, and rows get + them from [`refresh_column`][lancedb.table.Table.refresh_column]. + Declaring one therefore costs the same on a large table as on an + empty one. + + A refresh does not revisit rows it has already filled, so mutating + an input leaves the value computed at fill time; recomputing means + dropping the column and declaring it again. While a declaration + reads a column, that column cannot be renamed, retyped or dropped. + + Local tables only; LanceDB Cloud and Enterprise raise + ``NotImplementedError``. Cannot be combined with ``transforms``. Returns ------- AddColumnsResult version: the new version number of the table after adding columns. + + Examples + -------- + >>> import lancedb + >>> db = lancedb.connect("./.lancedb") + >>> table = db.create_table("computed_demo", [{"x": 1}, {"x": 2}]) + >>> table.add_columns(computed={"doubled": "x * 2"}) + AddColumnsResult(version=2) + >>> table.refresh_column("doubled") + RefreshColumnResult(rows_filled=2, version=3) + >>> table.to_arrow().sort_by("x").to_pandas() + x doubled + 0 1 2 + 1 2 4 + """ + + @abstractmethod + def refresh_column(self, column: str) -> "RefreshColumnResult": + """ + Fill the rows of a computed column that hold no value yet. + + Declared with ``add_columns(computed=...)``, a column starts empty and + gets its values here. Rows appended since the last refresh are filled + by the next one; rows already filled are left as they are, so the call + is idempotent and does not observe a mutated input. + + Local tables only; LanceDB Cloud and Enterprise raise + ``NotImplementedError``. + + Parameters + ---------- + column: str + The name of the computed column to fill. + + Returns + ------- + RefreshColumnResult + rows_filled: the number of rows given a value. + version: the new version number of the table. """ @abstractmethod @@ -5896,6 +5953,21 @@ class AsyncTable: each row in the table, and can reference existing columns. Alternatively, you can pass a pyarrow field or schema to add new columns with NULLs. + computed: Dict[str, str], optional + A map of column name to a SQL expression defining the column. The + column's type and inputs are derived from the expression. + + Unlike ``transforms``, the expression is stored rather than + evaluated now: the column is committed with no values, and rows get + them from + [`refresh_column`][lancedb.table.AsyncTable.refresh_column]. + + A refresh does not revisit rows it has already filled, so mutating + an input leaves the value computed at fill time. While a + declaration reads a column, that column cannot be renamed, retyped + or dropped. + + Local tables only. Cannot be combined with ``transforms``. Returns ------- @@ -5924,7 +5996,15 @@ class AsyncTable: async def refresh_column(self, column: str) -> RefreshColumnResult: """ - Compute and store values for a computed column's unfilled rows. + Fill the rows of a computed column that hold no value yet. + + Declared with ``add_columns(computed=...)``, a column starts empty and + gets its values here. Rows appended since the last refresh are filled + by the next one; rows already filled are left as they are, so the call + is idempotent and does not observe a mutated input. + + Local tables only; LanceDB Cloud and Enterprise raise + ``NotImplementedError``. Parameters ---------- diff --git a/rust/lancedb/src/table/add_columns.rs b/rust/lancedb/src/table/add_columns.rs index 8c95ef669..5659ae1b0 100644 --- a/rust/lancedb/src/table/add_columns.rs +++ b/rust/lancedb/src/table/add_columns.rs @@ -50,6 +50,34 @@ impl AddColumnsBuilder { /// Add a column defined by `expression`, evaluated by a later refresh /// rather than by this commit. Its type and inputs are derived from the /// expression. + /// + /// The column is committed with no values, so declaring one costs the same + /// on an empty table as on a large one. Rows get values from + /// [`Table::refresh_column`](super::Table::refresh_column), which fills + /// every fragment that has none -- including fragments appended since the + /// last refresh. + /// + /// Refresh does not revisit a fragment it has filled, so mutating an input + /// leaves the value computed at fill time; recomputing means dropping the + /// column and declaring it again. An input cannot be renamed, retyped or + /// dropped while a declaration reads it, since the expression names it. + /// + /// Local tables only: LanceDB Cloud and Enterprise reject a declaration + /// with `NotSupported`. + /// + /// ``` + /// # use lancedb::Table; + /// # async fn declare(table: &Table) -> Result<(), Box> { + /// table + /// .add_columns() + /// .computed("doubled", "x * 2") + /// .execute() + /// .await?; + /// let filled = table.refresh_column("doubled").await?; + /// println!("filled {} rows", filled.rows_filled); + /// # Ok(()) + /// # } + /// ``` pub fn computed(mut self, name: impl Into, expression: impl Into) -> Self { self.computed.push((name.into(), expression.into())); self