// SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors import { Table as ArrowTable, Data, DataType, Field, IntoVector, MultiVector, Schema, dataTypeToJson, fromDataToBuffer, fromTableToBuffer, isMultiVector, makeEmptyTable, tableFromIPC, } from "./arrow"; import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry"; import { IndexOptions } from "./indices"; import { MergeInsertBuilder } from "./merge"; import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, IndexConfig, IndexStatistics, Job, LsmStats, Branches as NativeBranches, OptimizeStats, RefreshColumnResult, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable, } from "./native"; import { FullTextQuery, Query, TakeQuery, VectorQuery, instanceOfFullTextQuery, } from "./query"; import { sanitizeType } from "./sanitize"; import { IntoSql, toSQL } from "./util"; export { IndexConfig } from "./native"; export { BucketStats, GenerationStats, LsmStats, MemtableStats, } from "./native"; /** * Progress snapshot for a write operation, delivered to the `progress` * callback passed to {@link Table.add}. */ export interface WriteProgress { /** Number of rows written so far. */ outputRows: number; /** Number of bytes written so far. */ outputBytes: number; /** * Total rows expected, when the input source reports it. * * Always set on the final callback (the one with `done: true`), falling * back to the actual number of rows written when the source could not * report a row count up front. */ totalRows?: number; /** Wall-clock seconds since the write started. */ elapsedSeconds: number; /** Number of parallel write tasks currently in flight. */ activeTasks: number; /** Total number of parallel write tasks (the write parallelism). */ totalTasks: number; /** `true` for the final callback; `false` otherwise. */ done: boolean; } /** * Options for adding data to a table. */ export interface AddDataOptions { /** * If "append" (the default) then the new data will be added to the table * * If "overwrite" then the new data will replace the existing data in the table. */ mode: "append" | "overwrite"; /** * Optional callback invoked periodically with write progress. * * The callback is fired once per batch written and once more with * `done: true` when the write completes. Calls are dispatched * asynchronously to the JS event loop and never block the write — a slow * callback will queue events rather than back-pressure the writer. * * Errors thrown from the callback are logged with `console.warn` and * swallowed — they do not abort the write. * * @example * ```ts * await table.add(data, { * progress: (p) => { * console.log(`${p.outputRows}/${p.totalRows ?? "?"} rows`); * }, * }); * ``` */ progress: (progress: WriteProgress) => void; } export interface UpdateOptions { /** * A filter that limits the scope of the update. * * This should be an SQL filter expression. * * Only rows that satisfy the expression will be updated. * * For example, this could be 'my_col == 0' to replace all instances * of 0 in a column with some other default value. */ where: string; } export interface OptimizeOptions { /** * If set then all versions older than the given date * be removed. The current version will never be removed. * The default is 7 days * @example * // Delete all versions older than 1 day * const olderThan = new Date(); * olderThan.setDate(olderThan.getDate() - 1)); * tbl.optimize({cleanupOlderThan: olderThan}); * * // Delete all versions except the current version * tbl.optimize({cleanupOlderThan: new Date()}); */ cleanupOlderThan: Date; /** * Because they may be part of an in-progress transaction, files newer than * 7 days old are not deleted by default. If you are sure that there are no * in-progress transactions, then you can set this to true to delete all * files older than `cleanupOlderThan`. * * **WARNING**: This should only be set to true if you can guarantee that * no other process is currently working on this dataset. Otherwise the * dataset could be put into a corrupted state. */ deleteUnverified: boolean; } export interface Version { version: number; timestamp: Date; metadata: Record; } /** Token produced by the tokenizer configured on a full-text search index. */ export interface FtsToken { /** Token text after tokenizer filters have been applied. */ text: string; /** Token position used by full-text query matching. */ position: number; } export type TokenizeTableOptions = | { /** FTS-indexed column whose tokenizer should be used. */ column: string; indexName?: never; } | { /** Name of the FTS index whose tokenizer should be used. */ indexName: string; column?: never; }; /** * Specification selecting Lance's MemWAL LSM-style write path for * `mergeInsert`. * * `specType` is `"bucket"`, `"identity"`, or `"unsharded"`. For `"bucket"`, * `column` and `numBuckets` are required; for `"identity"`, `column` is * required and must be a deterministic function of the unenforced primary * key (every row with a given primary key must always produce the same * `column` value, or upserts of that key can land in different shards and a * stale version can win). */ export interface LsmWriteSpec { /** One of `"bucket"`, `"identity"`, or `"unsharded"`. */ specType: "bucket" | "identity" | "unsharded"; /** Bucket and identity variants: the sharding column. */ column?: string; /** Bucket variant: the number of buckets, in `[1, 1024]`. */ numBuckets?: number; /** * Indexes the MemWAL keeps up to date. Omit to maintain every supported * index, resolved on install — a snapshot, so indexes created later are not * maintained. Pass `[]` for none. */ maintainedIndexes?: string[]; /** Default `ShardWriter` configuration recorded in the MemWAL index. */ writerConfigDefaults?: Record; } /** * A Table is a collection of Records in a LanceDB Database. * * A Table object is expected to be long lived and reused for multiple operations. * Table objects will cache a certain amount of index data in memory. This cache * will be freed when the Table is garbage collected. To eagerly free the cache you * can call the `close` method. Once the Table is closed, it cannot be used for any * further operations. * * Tables are created using the methods {@link Connection#createTable} * and {@link Connection#createEmptyTable}. Existing tables are opened * using {@link Connection#openTable}. * * Closing a table is optional. It not closed, it will be closed when it is garbage * collected. * * @hideconstructor */ export abstract class Table { [Symbol.for("nodejs.util.inspect.custom")](): string { return this.display(); } /** Returns the name of the table */ abstract get name(): string; /** Return true if the table has not been closed */ abstract isOpen(): boolean; /** * Close the table, releasing any underlying resources. * * It is safe to call this method multiple times. * * Any attempt to use the table after it is closed will result in an error. */ abstract close(): void; /** Return a brief description of the table */ abstract display(): string; /** Get the schema of the table. */ abstract schema(): Promise; /** * Insert records into this Table. * @param {Data} data Records to be inserted into the Table * @returns {Promise} A promise that resolves to an object * containing the new version number of the table */ abstract add( data: Data, options?: Partial, ): Promise; /** * Update existing records in the Table * @param opts.values The values to update. The keys are the column names and the values * are the values to set. * @returns {Promise} A promise that resolves to an object containing * the number of rows updated and the new version number * @example * ```ts * table.update({where:"x = 2", values:{"vector": [10, 10]}}) * ``` */ abstract update( opts: { values: Map | Record; } & Partial, ): Promise; /** * Update existing records in the Table * @param opts.valuesSql The values to update. The keys are the column names and the values * are the values to set. The values are SQL expressions. * @returns {Promise} A promise that resolves to an object containing * the number of rows updated and the new version number * @example * ```ts * table.update({where:"x = 2", valuesSql:{"x": "x + 1"}}) * ``` */ abstract update( opts: { valuesSql: Map | Record; } & Partial, ): Promise; /** * Update existing records in the Table * * An update operation can be used to adjust existing values. Use the * returned builder to specify which columns to update. The new value * can be a literal value (e.g. replacing nulls with some default value) * or an expression applied to the old value (e.g. incrementing a value) * * An optional condition can be specified (e.g. "only update if the old * value is 0") * * Note: if your condition is something like "some_id_column == 7" and * you are updating many rows (with different ids) then you will get * better performance with a single [`merge_insert`] call instead of * repeatedly calilng this method. * @param {Map | Record} updates - the * columns to update * @returns {Promise} A promise that resolves to an object * containing the number of rows updated and the new version number * * Keys in the map should specify the name of the column to update. * Values in the map provide the new value of the column. These can * be SQL literal strings (e.g. "7" or "'foo'") or they can be expressions * based on the row being updated (e.g. "my_col + 1") * @param {Partial} options - additional options to control * the update behavior */ abstract update( updates: Map | Record, options?: Partial, ): Promise; /** Count the total number of rows in the dataset. */ abstract countRows(filter?: string): Promise; /** * Delete the rows that satisfy the predicate. * @returns {Promise} A promise that resolves to an object * containing the new version number of the table */ abstract delete(predicate: string): Promise; /** * Create an index to speed up queries. * * Indices can be created on vector columns or scalar columns. * Indices on vector columns will speed up vector searches. * Indices on scalar columns will speed up filtering (in both * vector and non-vector searches) * * We currently don't support custom named indexes. * The index name will always be `${column}_idx`. * * @example * // If the column has a vector (fixed size list) data type then * // an IvfPq vector index will be created. * const table = await conn.openTable("my_table"); * await table.createIndex("vector"); * @example * // For advanced control over vector index creation you can specify * // the index type and options. * const table = await conn.openTable("my_table"); * await table.createIndex("vector", { * config: lancedb.Index.ivfPq({ * numPartitions: 128, * numSubVectors: 16, * }), * }); * @example * // Or create a Scalar index * await table.createIndex("my_float_col"); */ abstract createIndex( column: string, options?: Partial, ): Promise; /** * Create an index, returning a handle to the indexing job. * * The job may already be complete when returned; callers must not assume * the index exists until {@link Job.wait} resolves. */ abstract createIndexAsync( column: string, options?: Partial, ): Promise; /** * Drop an index from the table. * * @param name The name of the index. * * This does not delete the index from disk, it just removes it from the table. * To delete the index, run {@link Table#optimize} after dropping the index. * * Use {@link Table.listIndices} to find the names of the indices. */ abstract dropIndex(name: string): Promise; /** * Prewarm an index in the table. * * @param name The name of the index. * * This will load the index into memory. This may reduce the cold-start time for * future queries. If the index does not fit in the cache then this call may be * wasteful. */ abstract prewarmIndex(name: string): Promise; /** * Prewarm one or more columns of data in the table. * * @param columns The columns to prewarm. If undefined, all columns are prewarmed. * * This will load the column data into the page cache so that future queries that * read those columns avoid the initial cold-start latency. This call initiates * prewarming and returns once the request is accepted; the warming itself may * continue in the background. Calling it on already-prewarmed columns is a * no-op on the server. * * Prewarming is generally useful for columns used in filters or projections. * Large columns (e.g. high-dimensional vectors or binary data) may not be * practical to prewarm. * * This feature is currently only supported on remote tables. */ abstract prewarmData(columns?: string[]): Promise; /** * Waits for asynchronous indexing to complete on the table. * * @param indexNames The name of the indices to wait for * @param timeoutSeconds The number of seconds to wait before timing out * * This will raise an error if the indices are not created and fully indexed within the timeout. */ abstract waitForIndex( indexNames: string[], timeoutSeconds: number, ): Promise; /** * Create a {@link Query} Builder. * * Queries allow you to search your existing data. By default the query will * return all the data in the table in no particular order. The builder * returned by this method can be used to control the query using filtering, * vector similarity, sorting, and more. * * Note: By default, all columns are returned. For best performance, you should * only fetch the columns you need. * * When appropriate, various indices and statistics based pruning will be used to * accelerate the query. * @example * // SQL-style filtering * // * // This query will return up to 1000 rows whose value in the `id` column * // is greater than 5. LanceDb supports a broad set of filtering functions. * for await (const batch of table * .query() * .where("id > 1") * .select(["id"]) * .limit(20)) { * console.log(batch); * } * @example * // Vector Similarity Search * // * // This example will find the 10 rows whose value in the "vector" column are * // closest to the query vector [1.0, 2.0, 3.0]. If an index has been created * // on the "vector" column then this will perform an ANN search. * // * // The `refineFactor` and `nprobes` methods are used to control the recall / * // latency tradeoff of the search. * for await (const batch of table * .query() * .where("id > 1") * .select(["id"]) * .limit(20)) { * console.log(batch); * } * @example * // Scan the full dataset * // * // This query will return everything in the table in no particular order. * for await (const batch of table.query()) { * console.log(batch); * } * @returns {Query} A builder that can be used to parameterize the query */ abstract query(): Query; /** * Create a query that returns a subset of the rows in the table. * @param offsets The offsets of the rows to return. * @returns A builder that can be used to parameterize the query. */ abstract takeOffsets(offsets: number[]): TakeQuery; /** * Create a query that returns a subset of the rows in the table. * @param rowIds The row ids of the rows to return. * * Row ids returned by `withRowId()` are `bigint`, so `bigint[]` is supported. * For convenience / backwards compatibility, `number[]` is also accepted (for * small row ids that fit in a safe integer). * @returns A builder that can be used to parameterize the query. */ abstract takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery; /** * Create a search query to find the nearest neighbors * of the given query * @param {string | IntoVector} query - the query, a vector or string * @param {string} queryType - the type of the query, "vector", "fts", or "auto" * @param {string | string[]} ftsColumns - the columns to search in for full text search * for now, only one column can be searched at a time. * * when "auto" is used, if the query is a string and an embedding function is defined, it will be treated as a vector query * if the query is a string and no embedding function is defined, it will be treated as a full text search query */ abstract search( query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[], ): VectorQuery | Query; /** * Search the table with a given query vector. * * This is a convenience method for preparing a vector query and * is the same thing as calling `nearestTo` on the builder returned * by `query`. @see {@link Query#nearestTo} for more details. */ abstract vectorSearch(vector: IntoVector | MultiVector): VectorQuery; /** * Add new columns with defined values. * * The `{ computed }` form stores the expression rather than evaluating it * now: the column is committed with no values, and rows get them from * {@link Table#refreshColumn}. Declaring one therefore costs the same on a * large table as on an empty one. * * A refresh does not revisit rows it has already filled, so mutating an * input leaves the value computed at fill time; recomputing means dropping * the column and declaring it again. While a declaration reads a column, * that column cannot be renamed, retyped or dropped. * * On LanceDB Cloud and Enterprise the expression is planned by the * server, and the refresh runs as a server job -- see * {@link Table#refreshColumnAsync}. * @param {AddColumnsSql[] | Field | Field[] | Schema} newColumnTransforms Either: * - An array of objects with column names and SQL expressions to calculate values * - A single Arrow Field defining one column with its data type (column will be initialized with null values) * - An array of Arrow Fields defining columns with their data types (columns will be initialized with null values) * - An Arrow Schema defining columns with their data types (columns will be initialized with null values) * - `{ computed }`, declaring columns defined by a SQL expression whose type and inputs are derived from it * @returns {Promise} A promise that resolves to an object * containing the new version number of the table after adding the columns. * @example * ```ts * await table.addColumns({ computed: [{ name: "doubled", valueSql: "x * 2" }] }); * const { rowsFilled } = await table.refreshColumn("doubled"); * ``` */ abstract addColumns( newColumnTransforms: | AddColumnsSql[] | Field | Field[] | Schema | { computed: AddColumnsSql[] }, ): Promise; /** * Fill the rows of a computed column that hold no value yet. * * Rows appended since the last refresh are filled by the next one; rows * already filled are left as they are, so the call is idempotent and does * not observe a mutated input. Local tables only: a remote refresh runs * as a server job, through {@link Table#refreshColumnAsync}. * @param {string} column The name of the computed column to fill. * @returns {Promise} A promise that resolves to the * number of rows filled and the new version number of the table. */ abstract refreshColumn(column: string): Promise; /** * Like {@link Table#refreshColumn}, but returns a handle to the refresh * job instead of blocking until it completes. * * The job may already be complete when returned; callers must not assume * the column is filled until {@link Job.wait} resolves. Invalid input -- * an unknown column, or one that is not computed -- rejects here rather * than failing the job. On local tables the job runs in-process; on * LanceDB Cloud and Enterprise it is the server's backfill job. * @param {string} column The name of the computed column to fill. * @example * ```ts * const job = await table.refreshColumnAsync("doubled"); * await job.wait(); * console.log(await job.status()); // "finished" * ``` */ abstract refreshColumnAsync(column: string): Promise; /** * Alter the name or nullability of columns. * @param {ColumnAlteration[]} columnAlterations One or more alterations to * apply to columns. * @returns {Promise} A promise that resolves to an object * containing the new version number of the table after altering the columns. */ abstract alterColumns( columnAlterations: ColumnAlteration[], ): Promise; /** * Update per-field (column) metadata. * @param {FieldMetadataUpdate[]} updates One or more per-field updates. Each * update's metadata is merged into the field's existing metadata by default; * a value of `null` deletes that key, and `replace: true` swaps the whole map. * @returns {Promise} resolves to the new table version. */ abstract updateFieldMetadata( updates: FieldMetadataUpdate[], ): Promise; /** * Drop one or more columns from the dataset * * This is a metadata-only operation and does not remove the data from the * underlying storage. In order to remove the data, you must subsequently * call ``compact_files`` to rewrite the data without the removed columns and * then call ``cleanup_files`` to remove the old files. * @param {string[]} columnNames The names of the columns to drop. These can * be nested column references (e.g. "a.b.c") or top-level column names * (e.g. "a"). * @returns {Promise} A promise that resolves to an object * containing the new version number of the table after dropping the columns. */ abstract dropColumns(columnNames: string[]): Promise; /** * Set the unenforced primary key for this table to a single column. * * "Unenforced" means LanceDB does not check uniqueness on writes; the * column is recorded in the schema as the primary key for use by features * such as `merge_insert`. Only single-column primary keys are supported, * and the key cannot be changed once set. * @param {string | string[]} columns The primary key column. A one-element * array is also accepted; passing more than one column is rejected. * @returns {Promise} */ abstract setUnenforcedPrimaryKey(columns: string | string[]): Promise; /** * Install an {@link LsmWriteSpec} on this table, selecting Lance's MemWAL * LSM-style write path for future `mergeInsert` calls. * * `LsmWriteSpec` chooses one of three sharding strategies via `specType`: * * - `"bucket"` — hash-bucket writes by the single-column unenforced primary * key (`column` and `numBuckets` required). * - `"identity"` — shard by the raw value of a scalar `column`. * - `"unsharded"` — route every write to a single shard. * * All variants require the table to have an unenforced primary key * ({@link Table#setUnenforcedPrimaryKey}); bucket sharding additionally * requires it to be the single column being bucketed. * * Omitting `maintainedIndexes` maintains every index on the table, resolved * here, failing if one cannot be maintained — name them to install anyway. * Naming them pins an exact set, and a still-building index is rejected * rather than quietly omitted. * @param {LsmWriteSpec} spec The sharding spec to install. * @returns {Promise} * @example * ```ts * await table.setUnenforcedPrimaryKey("id"); * await table.setLsmWriteSpec({ * specType: "bucket", * column: "id", * numBuckets: 16, * maintainedIndexes: ["id_idx"], * }); * ``` */ abstract setLsmWriteSpec(spec: LsmWriteSpec): Promise; /** * Remove the {@link LsmWriteSpec} from this table, reverting to the standard * `mergeInsert` write path. * * Errors if no spec is currently set. * @returns {Promise} */ abstract unsetLsmWriteSpec(): Promise; /** * Read the {@link LsmWriteSpec} currently installed on this table. * * Resolves to `undefined` when the MemWAL LSM write path is not enabled (no * spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}). * The returned spec mirrors what was passed to * {@link Table#setLsmWriteSpec}, except that `maintainedIndexes` always * reports the concrete list resolved when the spec was set — `undefined` * never round-trips. * @returns {Promise} */ abstract getLsmWriteSpec(): Promise; /** * Drain and close any cached MemWAL shard writers held for this table. * * When an {@link LsmWriteSpec} is installed, `mergeInsert` opens MemWAL * shard writers and caches them for reuse across calls. This closes them, * flushing pending data; writers reopen lazily on the next `mergeInsert`. * It is a no-op when no writers are cached. * @returns {Promise} */ abstract closeLsmWriters(): Promise; /** * Seal every bucket's active memtable into a new L0 generation. * * Returns once the seal is committed. Sealing an empty memtable is a no-op, * so this is safe to call repeatedly. * @returns {Promise} */ abstract flushLsm(): Promise; /** * Trigger a background L0 → base compaction pass per bucket. * * Returns once the passes are *dispatched*, not once they finish — watch * {@link Table#getLsmStats} for progress, or use * {@link Table#checkpointLsm} to wait for convergence. * @returns {Promise} */ abstract compactLsm(): Promise; /** * Converge this table's LSM write path into its base table. * * Seals once, then triggers compaction and polls until the L0 that existed * at the start is gone. The target set is fixed at the start, so * generations created *during* the checkpoint are ignored — that is what * lets it terminate under write load, and what makes it best-effort: it * converges the fresh tier as of some instant. Idempotent, abandonable at * any point, and safe to run on a cadence. * * There is no liveness bound — the compactor pool is shared across tables, * so a checkpoint queued behind unrelated work looks exactly like one that * is merging. The caller owns the deadline. * @returns {Promise} * @example * ```ts * const before = await table.getLsmStats(); * await table.checkpointLsm(); * const after = await table.getLsmStats(); * ``` */ abstract checkpointLsm(): Promise; /** * Read live per-bucket LSM state. * * Answers "how far behind is my fresh tier", "which bucket is hot", and * "why is my fresh-tier vector search brute-force". Mutates no table state. * * Resolves to `undefined` only when the LSM write path is not enabled. * @param {boolean} includeGenerationRows Also count rows per L0 generation. * Off by default because each count opens an uncached Lance dataset. * @returns {Promise} */ abstract getLsmStats( includeGenerationRows?: boolean, ): Promise; /** Retrieve the version of the table */ abstract version(): Promise; /** * Checks out a specific version of the table _This is an in-place operation._ * * This allows viewing previous versions of the table. If you wish to * keep writing to the dataset starting from an old version, then use * the `restore` function. * * Calling this method will set the table into time-travel mode. If you * wish to return to standard mode, call `checkoutLatest`. * @param {number | string} version The version to checkout, could be version number or tag * @example * ```typescript * import * as lancedb from "@lancedb/lancedb" * const db = await lancedb.connect("./.lancedb"); * const table = await db.createTable("my_table", [ * { vector: [1.1, 0.9], type: "vector" }, * ]); * * console.log(await table.version()); // 1 * console.log(table.display()); * await table.add([{ vector: [0.5, 0.2], type: "vector" }]); * await table.checkout(1); * console.log(await table.version()); // 2 * ``` */ abstract checkout(version: number | string): Promise; /** * Checkout the latest version of the table. _This is an in-place operation._ * * The table will be set back into standard mode, and will track the latest * version of the table. */ abstract checkoutLatest(): Promise; /** * List all the versions of the table */ abstract listVersions(): Promise; /** * Get a tags manager for this table. * * Tags allow you to label specific versions of a table with a human-readable name. * The returned tags manager can be used to list, create, update, or delete tags. * * @returns {Tags} A tags manager for this table * @example * ```typescript * const tagsManager = await table.tags(); * await tagsManager.create("v1", 1); * const tags = await tagsManager.list(); * console.log(tags); // { "v1": { version: 1, manifestSize: ... } } * ``` */ abstract tags(): Promise; /** * Get the branch manager for this table. * * Branches are isolated, writable lines of history forked from another * branch (or version). Writes on a branch do not affect `main`. */ abstract branches(): Promise; /** * The branch this table handle is scoped to, or `null` for the main branch. * * A handle returned by {@link Branches.create} or {@link Branches.checkout} * reports the branch it targets; a handle opened normally reports `null`. */ abstract currentBranch(): string | null; /** * Restore the table to the currently checked out version * * This operation will fail if checkout has not been called previously * * This operation will overwrite the latest version of the table with a * previous version. Any changes made since the checked out version will * no longer be visible. * * Once the operation concludes the table will no longer be in a checked * out state and the read_consistency_interval, if any, will apply. */ abstract restore(): Promise; /** * Optimize the on-disk data and indices for better performance. * * Modeled after ``VACUUM`` in PostgreSQL. * * Optimization covers three operations: * * - Compaction: Merges small files into larger ones * - Prune: Removes old versions of the dataset * - Index: Optimizes the indices, adding new data to existing indices * * * The frequency an application should call optimize is based on the frequency of * data modifications. If data is frequently added, deleted, or updated then * optimize should be run frequently. A good rule of thumb is to run optimize if * you have added or modified 100,000 or more records or run more than 20 data * modification operations. */ abstract optimize(options?: Partial): Promise; /** List all indices that have been created with {@link Table.createIndex} */ abstract listIndices(): Promise; /** * Tokenize a full-text search query using the tokenizer configured on an FTS index. * * Specify exactly one of `column` or `indexName`. * * Model-backed tokenizers such as `jieba/*` and `lindera/*` are rebuilt in * the client process from index metadata. For remote tables, this means the * same tokenizer model files must also exist locally. */ abstract tokenize( query: string, options: TokenizeTableOptions, ): Promise; /** Return the table as an arrow table */ abstract toArrow(): Promise; abstract mergeInsert(on: string | string[]): MergeInsertBuilder; /** List all the stats of a specified index * * @param {string} name The name of the index. * @returns {IndexStatistics | undefined} The stats of the index. If the index does not exist, it will return undefined * * Use {@link Table.listIndices} to find the names of the indices. */ abstract indexStats(name: string): Promise; /** Returns table and fragment statistics * * @returns {TableStatistics} The table and fragment statistics * */ abstract stats(): Promise; /** * Get the initial storage options that were passed in when opening this table. * * For dynamically refreshed options (e.g., credential vending), use * {@link Table.latestStorageOptions}. * * Warning: This is an internal API and the return value is subject to change. * * @returns The storage options, or undefined if no storage options were configured. */ abstract initialStorageOptions(): Promise< Record | null | undefined >; /** * Get the latest storage options, refreshing from provider if configured. * * This method is useful for credential vending scenarios where storage options * may be refreshed dynamically. If no dynamic provider is configured, this * returns the initial static options. * * Warning: This is an internal API and the return value is subject to change. * * @returns The storage options, or undefined if no storage options were configured. */ abstract latestStorageOptions(): Promise< Record | null | undefined >; } export class LocalTable extends Table { private readonly inner: _NativeTable; constructor(inner: _NativeTable) { super(); this.inner = inner; } get name(): string { return this.inner.name; } isOpen(): boolean { return this.inner.isOpen(); } close(): void { this.inner.close(); } display(): string { return this.inner.display(); } private async getEmbeddingFunctions(): Promise< Map > { const schema = await this.schema(); const registry = getRegistry(); return registry.parseFunctions(schema.metadata); } /** Get the schema of the table. */ async schema(): Promise { const schemaBuf = await this.inner.schema(); const tbl = tableFromIPC(schemaBuf); return tbl.schema; } async add(data: Data, options?: Partial): Promise { const mode = options?.mode ?? "append"; const schema = await this.schema(); const buffer = await fromDataToBuffer(data, undefined, schema); // Wrap the user callback so a thrown error doesn't surface as an // unhandled exception (the callback fires from a napi threadsafe // function — exceptions there crash the process). const userProgress = options?.progress; const progress = userProgress ? (p: WriteProgress) => { try { userProgress(p); } catch (e) { console.warn("Table.add progress callback threw:", e); } } : undefined; return await this.inner.add(buffer, mode, progress); } async update( optsOrUpdates: | (Map | Record) | ({ values: Map | Record; } & Partial) | ({ valuesSql: Map | Record; } & Partial), options?: Partial, ): Promise { const isValues = "values" in optsOrUpdates && typeof optsOrUpdates.values !== "string"; const isValuesSql = "valuesSql" in optsOrUpdates && typeof optsOrUpdates.valuesSql !== "string"; const isMap = (obj: unknown): obj is Map => { return obj instanceof Map; }; let predicate; let columns: [string, string][]; switch (true) { case isMap(optsOrUpdates): columns = Array.from(optsOrUpdates.entries()); predicate = options?.where; break; case isValues && isMap(optsOrUpdates.values): columns = Array.from(optsOrUpdates.values.entries()).map(([k, v]) => [ k, toSQL(v), ]); predicate = optsOrUpdates.where; break; case isValues && !isMap(optsOrUpdates.values): columns = Object.entries(optsOrUpdates.values).map(([k, v]) => [ k, toSQL(v), ]); predicate = optsOrUpdates.where; break; case isValuesSql && isMap(optsOrUpdates.valuesSql): columns = Array.from(optsOrUpdates.valuesSql.entries()); predicate = optsOrUpdates.where; break; case isValuesSql && !isMap(optsOrUpdates.valuesSql): columns = Object.entries(optsOrUpdates.valuesSql).map(([k, v]) => [ k, v, ]); predicate = optsOrUpdates.where; break; default: columns = Object.entries(optsOrUpdates as Record); predicate = options?.where; } return await this.inner.update(predicate, columns); } async countRows(filter?: string): Promise { return await this.inner.countRows(filter); } async delete(predicate: string): Promise { return await this.inner.delete(predicate); } async createIndex(column: string, options?: Partial) { // Bit of a hack to get around the fact that TS has no package-scope. // biome-ignore lint/suspicious/noExplicitAny: skip const nativeIndex = (options?.config as any)?.inner; await this.inner.createIndex( nativeIndex, column, options?.replace, options?.waitTimeoutSeconds, options?.name, options?.train, ); } async createIndexAsync( column: string, options?: Partial, ): Promise { // biome-ignore lint/suspicious/noExplicitAny: skip const nativeIndex = (options?.config as any)?.inner; return await this.inner.createIndexAsync( nativeIndex, column, options?.replace, options?.waitTimeoutSeconds, options?.name, options?.train, ); } async dropIndex(name: string): Promise { await this.inner.dropIndex(name); } async prewarmIndex(name: string): Promise { await this.inner.prewarmIndex(name); } async prewarmData(columns?: string[]): Promise { await this.inner.prewarmData(columns); } async waitForIndex( indexNames: string[], timeoutSeconds: number, ): Promise { await this.inner.waitForIndex(indexNames, timeoutSeconds); } takeOffsets(offsets: number[]): TakeQuery { return new TakeQuery(this.inner.takeOffsets(offsets)); } takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery { const ids = rowIds.map((id) => { if (typeof id === "bigint") { return id; } if (!Number.isInteger(id)) { throw new Error("Row id must be an integer (or bigint)"); } if (id < 0) { throw new Error("Row id cannot be negative"); } if (!Number.isSafeInteger(id)) { throw new Error("Row id is too large for number; use bigint instead"); } return BigInt(id); }); return new TakeQuery(this.inner.takeRowIds(ids)); } query(): Query { return new Query(this.inner); } search( query: string | IntoVector | MultiVector | FullTextQuery, queryType: string = "auto", ftsColumns?: string | string[], ): VectorQuery | Query { if (typeof query !== "string" && !instanceOfFullTextQuery(query)) { if (queryType === "fts") { throw new Error("Cannot perform full text search on a vector query"); } return this.vectorSearch(query); } // If the query is a string, we need to determine if it is a vector query or a full text search query if (queryType === "fts") { return this.query().fullTextSearch(query, { columns: ftsColumns, }); } // The query type is auto or vector // fall back to full text search if no embedding functions are defined and the query is a string if ( queryType === "auto" && (getRegistry().length() === 0 || instanceOfFullTextQuery(query)) ) { return this.query().fullTextSearch(query, { columns: ftsColumns, }); } const queryPromise = this.getEmbeddingFunctions().then( async (functions) => { // TODO: Support multiple embedding functions const embeddingFunc: EmbeddingFunctionConfig | undefined = functions .values() .next().value; if (!embeddingFunc) { return Promise.reject( new Error("No embedding functions are defined in the table"), ); } return await embeddingFunc.function.computeQueryEmbeddings(query); }, ); return this.query().nearestTo(queryPromise); } vectorSearch(vector: IntoVector | MultiVector): VectorQuery { if (isMultiVector(vector)) { const query = this.query().nearestTo(vector[0]); for (const v of vector.slice(1)) { query.addQueryVector(v); } return query; } return this.query().nearestTo(vector); } // TODO: Support BatchUDF async addColumns( newColumnTransforms: | AddColumnsSql[] | Field | Field[] | Schema | { computed: AddColumnsSql[] }, ): Promise { // Columns defined by an expression are declared, not materialized here. if ( typeof newColumnTransforms === "object" && !Array.isArray(newColumnTransforms) && "computed" in newColumnTransforms ) { return await this.inner.addComputedColumns(newColumnTransforms.computed); } // Handle single Field -> convert to array of Fields if (newColumnTransforms instanceof Field) { newColumnTransforms = [newColumnTransforms]; } // Handle array of Fields -> convert to Schema if ( Array.isArray(newColumnTransforms) && newColumnTransforms.length > 0 && newColumnTransforms[0] instanceof Field ) { const fields = newColumnTransforms as Field[]; newColumnTransforms = new Schema(fields); } // Handle Schema -> use schema-based approach if (newColumnTransforms instanceof Schema) { const schema = newColumnTransforms; // Convert schema to buffer using Arrow IPC format const emptyTable = makeEmptyTable(schema); const schemaBuf = await fromTableToBuffer(emptyTable); return await this.inner.addColumnsWithSchema(schemaBuf); } // Handle SQL expressions (existing functionality) if (Array.isArray(newColumnTransforms)) { return await this.inner.addColumns( newColumnTransforms as AddColumnsSql[], ); } throw new Error("Invalid input type for addColumns"); } async refreshColumn(column: string): Promise { return await this.inner.refreshColumn(column); } async refreshColumnAsync(column: string): Promise { return await this.inner.refreshColumnAsync(column); } async alterColumns( columnAlterations: ColumnAlteration[], ): Promise { const processedAlterations = columnAlterations.map((alteration) => { if (typeof alteration.dataType === "string") { return { ...alteration, dataType: JSON.stringify({ type: alteration.dataType }), }; } else if (alteration.dataType === undefined) { return { ...alteration, dataType: undefined, }; } else { const dataType = sanitizeType(alteration.dataType); return { ...alteration, dataType: JSON.stringify(dataTypeToJson(dataType)), }; } }); return await this.inner.alterColumns(processedAlterations); } async updateFieldMetadata( updates: FieldMetadataUpdate[], ): Promise { return await this.inner.updateFieldMetadata(updates); } async dropColumns(columnNames: string[]): Promise { return await this.inner.dropColumns(columnNames); } async setUnenforcedPrimaryKey(columns: string | string[]): Promise { const cols = typeof columns === "string" ? [columns] : columns; return await this.inner.setUnenforcedPrimaryKey(cols); } async setLsmWriteSpec(spec: LsmWriteSpec): Promise { return await this.inner.setLsmWriteSpec(spec); } async unsetLsmWriteSpec(): Promise { return await this.inner.unsetLsmWriteSpec(); } async getLsmWriteSpec(): Promise { // The native binding types `specType` as a plain `string`; narrow it back // to the public union. The Rust `From` impl only ever emits one of the // three valid values, so the cast is safe. return ((await this.inner.getLsmWriteSpec()) ?? undefined) as | LsmWriteSpec | undefined; } async closeLsmWriters(): Promise { return await this.inner.closeLsmWriters(); } async flushLsm(): Promise { return await this.inner.flushLsm(); } async compactLsm(): Promise { return await this.inner.compactLsm(); } async checkpointLsm(): Promise { return await this.inner.checkpointLsm(); } async getLsmStats( includeGenerationRows: boolean = false, ): Promise { return (await this.inner.getLsmStats(includeGenerationRows)) ?? undefined; } async version(): Promise { return await this.inner.version(); } async checkout(version: number | string): Promise { if (typeof version === "string") { return this.inner.checkoutTag(version); } return this.inner.checkout(version); } async checkoutLatest(): Promise { await this.inner.checkoutLatest(); } async listVersions(): Promise { return (await this.inner.listVersions()).map((version) => ({ version: version.version, timestamp: new Date(version.timestamp / 1000), metadata: version.metadata, })); } async restore(): Promise { await this.inner.restore(); } async tags(): Promise { return await this.inner.tags(); } async branches(): Promise { return new Branches(await this.inner.branches()); } currentBranch(): string | null { return this.inner.currentBranch() ?? null; } async optimize(options?: Partial): Promise { let cleanupOlderThanMs; if ( options?.cleanupOlderThan !== undefined && options?.cleanupOlderThan !== null ) { cleanupOlderThanMs = new Date().getTime() - options.cleanupOlderThan.getTime(); } return await this.inner.optimize( cleanupOlderThanMs, options?.deleteUnverified, ); } async listIndices(): Promise { return await this.inner.listIndices(); } async tokenize( query: string, options: TokenizeTableOptions, ): Promise { return await this.inner.tokenize( query, options?.column, options?.indexName, ); } async toArrow(): Promise { return await this.query().toArrow(); } async indexStats(name: string): Promise { const stats = await this.inner.indexStats(name); if (stats === null) { return undefined; } return stats; } async stats(): Promise { return await this.inner.stats(); } async initialStorageOptions(): Promise< Record | null | undefined > { return await this.inner.initialStorageOptions(); } async latestStorageOptions(): Promise< Record | null | undefined > { return await this.inner.latestStorageOptions(); } mergeInsert(on: string | string[]): MergeInsertBuilder { on = Array.isArray(on) ? on : [on]; return new MergeInsertBuilder(this.inner.mergeInsert(on), this.schema()); } /** * Check if the table uses the new manifest path scheme. * * This function will return true if the table uses the V2 manifest * path scheme. */ async usesV2ManifestPaths(): Promise { return await this.inner.usesV2ManifestPaths(); } /** * Migrate the table to use the new manifest path scheme. * * This function will rename all V1 manifests to V2 manifest paths. * These paths provide more efficient opening of datasets with many versions * on object stores. * * This function is idempotent, and can be run multiple times without * changing the state of the object store. * * However, it should not be run while other concurrent operations are happening. * And it should also run until completion before resuming other operations. */ async migrateManifestPathsV2(): Promise { await this.inner.migrateManifestPathsV2(); } } /** * A definition of a column alteration. The alteration changes the column at * `path` to have the new name `name`, to be nullable if `nullable` is true, * and to have the data type `data_type`. At least one of `rename` or `nullable` * must be provided. */ export interface ColumnAlteration { /** * The path to the column to alter. This is a dot-separated path to the column. * If it is a top-level column then it is just the name of the column. If it is * a nested column then it is the path to the column, e.g. "a.b.c" for a column * `c` nested inside a column `b` nested inside a column `a`. */ path: string; /** * The new name of the column. If not provided then the name will not be changed. * This must be distinct from the names of all other columns in the table. */ rename?: string; /** * A new data type for the column. If not provided then the data type will not be changed. * Changing data types is limited to casting to the same general type. For example, these * changes are valid: * * `int32` -> `int64` (integers) * * `double` -> `float` (floats) * * `string` -> `large_string` (strings) * But these changes are not: * * `int32` -> `double` (mix integers and floats) * * `string` -> `int32` (mix strings and integers) */ dataType?: string | DataType; /** Set the new nullability. Note that a nullable column cannot be made non-nullable. */ nullable?: boolean; } /** A per-field metadata update, addressed by dot-path. */ export interface FieldMetadataUpdate { /** * Dot-separated path to the field. For a top-level column this is just its * name; for a nested field it's the path, e.g. "a.b.c". */ path: string; /** * Metadata key/value pairs. Merged into the field's existing metadata by * default; a value of `null` deletes that key. */ metadata: Record; /** If true, replace the field's entire metadata map instead of merging. */ replace?: boolean; } /** Summary of a column in a branch diff. */ export interface BranchColumnSummary { name: string; dataType: string; nullable: boolean; } /** A column whose definition differs between main and the branch. */ export interface BranchColumnChange { name: string; main: BranchColumnSummary; branch: BranchColumnSummary; } /** Summary of an index in a branch diff. */ export interface BranchIndexSummary { indexName: string; columns: string[]; indexType?: string; status: string; } /** Row-level comparison between main and the branch. */ export interface BranchRowCountSummary { unchanged: number; newOnBase: number; newOnBranch: number; staleRecompute: number; inputsChanged: number; deltaAvailable: boolean; } /** A reason why a branch cannot currently be merged. */ export interface MergeBlocker { code: string; message: string; } /** Read-only comparison of a branch against main. */ export interface BranchDiff { fromBranch: string; parentVersion: number; mainVersion: number; branchVersion: number; baseMoved: boolean; rowCountMain: number; rowCountBranch: number; rowSummary: BranchRowCountSummary; addedColumns: BranchColumnSummary[]; removedColumns: BranchColumnSummary[]; changedColumns: BranchColumnChange[]; addedIndexes: BranchIndexSummary[]; removedIndexes: BranchIndexSummary[]; mergeable: boolean; mergeBlockers: MergeBlocker[]; } /** Changes that would be, or were, promoted by a branch merge. */ export interface MergePreview { promotedColumns: string[]; } /** Result of previewing or attempting a branch merge. */ export interface MergeBranchResult { status: "ready" | "rejected" | "notImplemented" | "merged" | "unknown"; diff: BranchDiff; preview: MergePreview; mainVersionAfter?: number; } /** * Branch manager for a {@link Table}. * * Unlike tags, `create` and `checkout` return a new {@link Table} handle scoped * to the branch; writes on it do not affect `main`. */ export class Branches { #inner: NativeBranches; /** * Construct a Branches manager. Internal use only. * @hidden */ constructor(inner: NativeBranches) { this.#inner = inner; } /** List all branches, mapping name to branch metadata. */ async list(): Promise> { return await this.#inner.list(); } /** * Create a branch and return a handle scoped to it. * * @param name Name of the new branch. * @param fromRef Source branch to fork from. Defaults to `main`. * @param fromVersion A specific version on `fromRef`. Defaults to latest. */ async create( name: string, fromRef?: string, fromVersion?: number, ): Promise { return new LocalTable(await this.#inner.create(name, fromRef, fromVersion)); } /** * Check out an existing branch and return a handle scoped to it. * * With `version` set, the returned handle is pinned to that version of the * branch (a read-only, detached view); otherwise it tracks the branch's * latest and stays writable. */ async checkout(name: string, version?: number): Promise
{ return new LocalTable(await this.#inner.checkout(name, version)); } /** Delete a branch. */ async delete(name: string): Promise { return await this.#inner.delete(name); } /** Compare a branch against main without modifying either branch. */ async diff(fromBranch: string): Promise { return (await this.#inner.diff(fromBranch)) as unknown as BranchDiff; } /** * Merge a branch into main. * * Set `dryRun` to `true` to preview the merge. A rejected merge resolves * with `status: "rejected"` instead of throwing. * * @param fromBranch Branch to merge from. * @param dryRun When true, only preview the merge. Defaults to false. */ async merge( fromBranch: string, dryRun: boolean = false, ): Promise { return (await this.#inner.merge( fromBranch, dryRun, )) as unknown as MergeBranchResult; } }