mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-19 12:38:38 +00:00
928c3dde2d
LanceDB Cloud and Enterprise support computed columns through the REST
API,
so declaration dispatches per backend: local tables plan the expression
themselves, remote ones send {name, computed} entries for the server to
plan. A remote refresh is the server's backfill job --
refresh_column_async
submits it and returns a handle whose successful wait establishes a
read-freshness baseline on the submitting handle, unless a checkout has
pinned the handle by the time the job completes; the blocking form
refuses
rather than invent a fill count the server does not report.
Declaration entries are built from the namespace client's
AddColumnsEntry
model (lance-namespace 0.11.0, via the lance beta.13 pin), so the
payload
shape is compile-checked against the published contract.
---
<sub>Stack created with <a
href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a
href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
1597 lines
52 KiB
TypeScript
1597 lines
52 KiB
TypeScript
// SPDX-License-Identifier: Apache-2.0
|
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
|
|
import {
|
|
Table as ArrowTable,
|
|
Data,
|
|
DataType,
|
|
Field,
|
|
IntoVector,
|
|
MultiVector,
|
|
Schema,
|
|
dataTypeToJson,
|
|
fromDataToBuffer,
|
|
fromTableToBuffer,
|
|
isMultiVector,
|
|
makeEmptyTable,
|
|
tableFromIPC,
|
|
} from "./arrow";
|
|
|
|
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
|
|
import { IndexOptions } from "./indices";
|
|
import { MergeInsertBuilder } from "./merge";
|
|
import {
|
|
AddColumnsResult,
|
|
AddColumnsSql,
|
|
AddResult,
|
|
AlterColumnsResult,
|
|
BranchContents,
|
|
DeleteResult,
|
|
DropColumnsResult,
|
|
IndexConfig,
|
|
IndexStatistics,
|
|
Job,
|
|
Branches as NativeBranches,
|
|
OptimizeStats,
|
|
RefreshColumnResult,
|
|
TableStatistics,
|
|
Tags,
|
|
UpdateFieldMetadataResult,
|
|
UpdateResult,
|
|
Table as _NativeTable,
|
|
} from "./native";
|
|
import {
|
|
FullTextQuery,
|
|
Query,
|
|
TakeQuery,
|
|
VectorQuery,
|
|
instanceOfFullTextQuery,
|
|
} from "./query";
|
|
import { sanitizeType } from "./sanitize";
|
|
import { IntoSql, toSQL } from "./util";
|
|
export { IndexConfig } from "./native";
|
|
|
|
/**
|
|
* Progress snapshot for a write operation, delivered to the `progress`
|
|
* callback passed to {@link Table.add}.
|
|
*/
|
|
export interface WriteProgress {
|
|
/** Number of rows written so far. */
|
|
outputRows: number;
|
|
/** Number of bytes written so far. */
|
|
outputBytes: number;
|
|
/**
|
|
* Total rows expected, when the input source reports it.
|
|
*
|
|
* Always set on the final callback (the one with `done: true`), falling
|
|
* back to the actual number of rows written when the source could not
|
|
* report a row count up front.
|
|
*/
|
|
totalRows?: number;
|
|
/** Wall-clock seconds since the write started. */
|
|
elapsedSeconds: number;
|
|
/** Number of parallel write tasks currently in flight. */
|
|
activeTasks: number;
|
|
/** Total number of parallel write tasks (the write parallelism). */
|
|
totalTasks: number;
|
|
/** `true` for the final callback; `false` otherwise. */
|
|
done: boolean;
|
|
}
|
|
|
|
/**
|
|
* Options for adding data to a table.
|
|
*/
|
|
export interface AddDataOptions {
|
|
/**
|
|
* If "append" (the default) then the new data will be added to the table
|
|
*
|
|
* If "overwrite" then the new data will replace the existing data in the table.
|
|
*/
|
|
mode: "append" | "overwrite";
|
|
|
|
/**
|
|
* Optional callback invoked periodically with write progress.
|
|
*
|
|
* The callback is fired once per batch written and once more with
|
|
* `done: true` when the write completes. Calls are dispatched
|
|
* asynchronously to the JS event loop and never block the write — a slow
|
|
* callback will queue events rather than back-pressure the writer.
|
|
*
|
|
* Errors thrown from the callback are logged with `console.warn` and
|
|
* swallowed — they do not abort the write.
|
|
*
|
|
* @example
|
|
* ```ts
|
|
* await table.add(data, {
|
|
* progress: (p) => {
|
|
* console.log(`${p.outputRows}/${p.totalRows ?? "?"} rows`);
|
|
* },
|
|
* });
|
|
* ```
|
|
*/
|
|
progress: (progress: WriteProgress) => void;
|
|
}
|
|
|
|
export interface UpdateOptions {
|
|
/**
|
|
* A filter that limits the scope of the update.
|
|
*
|
|
* This should be an SQL filter expression.
|
|
*
|
|
* Only rows that satisfy the expression will be updated.
|
|
*
|
|
* For example, this could be 'my_col == 0' to replace all instances
|
|
* of 0 in a column with some other default value.
|
|
*/
|
|
where: string;
|
|
}
|
|
|
|
export interface OptimizeOptions {
|
|
/**
|
|
* If set then all versions older than the given date
|
|
* be removed. The current version will never be removed.
|
|
* The default is 7 days
|
|
* @example
|
|
* // Delete all versions older than 1 day
|
|
* const olderThan = new Date();
|
|
* olderThan.setDate(olderThan.getDate() - 1));
|
|
* tbl.optimize({cleanupOlderThan: olderThan});
|
|
*
|
|
* // Delete all versions except the current version
|
|
* tbl.optimize({cleanupOlderThan: new Date()});
|
|
*/
|
|
cleanupOlderThan: Date;
|
|
/**
|
|
* Because they may be part of an in-progress transaction, files newer than
|
|
* 7 days old are not deleted by default. If you are sure that there are no
|
|
* in-progress transactions, then you can set this to true to delete all
|
|
* files older than `cleanupOlderThan`.
|
|
*
|
|
* **WARNING**: This should only be set to true if you can guarantee that
|
|
* no other process is currently working on this dataset. Otherwise the
|
|
* dataset could be put into a corrupted state.
|
|
*/
|
|
deleteUnverified: boolean;
|
|
}
|
|
|
|
export interface Version {
|
|
version: number;
|
|
timestamp: Date;
|
|
metadata: Record<string, string>;
|
|
}
|
|
|
|
/** Token produced by the tokenizer configured on a full-text search index. */
|
|
export interface FtsToken {
|
|
/** Token text after tokenizer filters have been applied. */
|
|
text: string;
|
|
/** Token position used by full-text query matching. */
|
|
position: number;
|
|
}
|
|
|
|
export type TokenizeTableOptions =
|
|
| {
|
|
/** FTS-indexed column whose tokenizer should be used. */
|
|
column: string;
|
|
indexName?: never;
|
|
}
|
|
| {
|
|
/** Name of the FTS index whose tokenizer should be used. */
|
|
indexName: string;
|
|
column?: never;
|
|
};
|
|
|
|
/**
|
|
* Specification selecting Lance's MemWAL LSM-style write path for
|
|
* `mergeInsert`.
|
|
*
|
|
* `specType` is `"bucket"`, `"identity"`, or `"unsharded"`. For `"bucket"`,
|
|
* `column` and `numBuckets` are required; for `"identity"`, `column` is
|
|
* required and must be a deterministic function of the unenforced primary
|
|
* key (every row with a given primary key must always produce the same
|
|
* `column` value, or upserts of that key can land in different shards and a
|
|
* stale version can win).
|
|
*/
|
|
export interface LsmWriteSpec {
|
|
/** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
|
|
specType: "bucket" | "identity" | "unsharded";
|
|
/** Bucket and identity variants: the sharding column. */
|
|
column?: string;
|
|
/** Bucket variant: the number of buckets, in `[1, 1024]`. */
|
|
numBuckets?: number;
|
|
/**
|
|
* Indexes the MemWAL keeps up to date. Omit to maintain every supported
|
|
* index, resolved on install — a snapshot, so indexes created later are not
|
|
* maintained. Pass `[]` for none.
|
|
*/
|
|
maintainedIndexes?: string[];
|
|
/** Default `ShardWriter` configuration recorded in the MemWAL index. */
|
|
writerConfigDefaults?: Record<string, string>;
|
|
}
|
|
|
|
/**
|
|
* A Table is a collection of Records in a LanceDB Database.
|
|
*
|
|
* A Table object is expected to be long lived and reused for multiple operations.
|
|
* Table objects will cache a certain amount of index data in memory. This cache
|
|
* will be freed when the Table is garbage collected. To eagerly free the cache you
|
|
* can call the `close` method. Once the Table is closed, it cannot be used for any
|
|
* further operations.
|
|
*
|
|
* Tables are created using the methods {@link Connection#createTable}
|
|
* and {@link Connection#createEmptyTable}. Existing tables are opened
|
|
* using {@link Connection#openTable}.
|
|
*
|
|
* Closing a table is optional. It not closed, it will be closed when it is garbage
|
|
* collected.
|
|
*
|
|
* @hideconstructor
|
|
*/
|
|
export abstract class Table {
|
|
[Symbol.for("nodejs.util.inspect.custom")](): string {
|
|
return this.display();
|
|
}
|
|
/** Returns the name of the table */
|
|
abstract get name(): string;
|
|
|
|
/** Return true if the table has not been closed */
|
|
abstract isOpen(): boolean;
|
|
/**
|
|
* Close the table, releasing any underlying resources.
|
|
*
|
|
* It is safe to call this method multiple times.
|
|
*
|
|
* Any attempt to use the table after it is closed will result in an error.
|
|
*/
|
|
abstract close(): void;
|
|
/** Return a brief description of the table */
|
|
abstract display(): string;
|
|
/** Get the schema of the table. */
|
|
abstract schema(): Promise<Schema>;
|
|
/**
|
|
* Insert records into this Table.
|
|
* @param {Data} data Records to be inserted into the Table
|
|
* @returns {Promise<AddResult>} A promise that resolves to an object
|
|
* containing the new version number of the table
|
|
*/
|
|
abstract add(
|
|
data: Data,
|
|
options?: Partial<AddDataOptions>,
|
|
): Promise<AddResult>;
|
|
/**
|
|
* Update existing records in the Table
|
|
* @param opts.values The values to update. The keys are the column names and the values
|
|
* are the values to set.
|
|
* @returns {Promise<UpdateResult>} A promise that resolves to an object containing
|
|
* the number of rows updated and the new version number
|
|
* @example
|
|
* ```ts
|
|
* table.update({where:"x = 2", values:{"vector": [10, 10]}})
|
|
* ```
|
|
*/
|
|
abstract update(
|
|
opts: {
|
|
values: Map<string, IntoSql> | Record<string, IntoSql>;
|
|
} & Partial<UpdateOptions>,
|
|
): Promise<UpdateResult>;
|
|
/**
|
|
* Update existing records in the Table
|
|
* @param opts.valuesSql The values to update. The keys are the column names and the values
|
|
* are the values to set. The values are SQL expressions.
|
|
* @returns {Promise<UpdateResult>} A promise that resolves to an object containing
|
|
* the number of rows updated and the new version number
|
|
* @example
|
|
* ```ts
|
|
* table.update({where:"x = 2", valuesSql:{"x": "x + 1"}})
|
|
* ```
|
|
*/
|
|
abstract update(
|
|
opts: {
|
|
valuesSql: Map<string, string> | Record<string, string>;
|
|
} & Partial<UpdateOptions>,
|
|
): Promise<UpdateResult>;
|
|
/**
|
|
* Update existing records in the Table
|
|
*
|
|
* An update operation can be used to adjust existing values. Use the
|
|
* returned builder to specify which columns to update. The new value
|
|
* can be a literal value (e.g. replacing nulls with some default value)
|
|
* or an expression applied to the old value (e.g. incrementing a value)
|
|
*
|
|
* An optional condition can be specified (e.g. "only update if the old
|
|
* value is 0")
|
|
*
|
|
* Note: if your condition is something like "some_id_column == 7" and
|
|
* you are updating many rows (with different ids) then you will get
|
|
* better performance with a single [`merge_insert`] call instead of
|
|
* repeatedly calilng this method.
|
|
* @param {Map<string, string> | Record<string, string>} updates - the
|
|
* columns to update
|
|
* @returns {Promise<UpdateResult>} A promise that resolves to an object
|
|
* containing the number of rows updated and the new version number
|
|
*
|
|
* Keys in the map should specify the name of the column to update.
|
|
* Values in the map provide the new value of the column. These can
|
|
* be SQL literal strings (e.g. "7" or "'foo'") or they can be expressions
|
|
* based on the row being updated (e.g. "my_col + 1")
|
|
* @param {Partial<UpdateOptions>} options - additional options to control
|
|
* the update behavior
|
|
*/
|
|
abstract update(
|
|
updates: Map<string, string> | Record<string, string>,
|
|
options?: Partial<UpdateOptions>,
|
|
): Promise<UpdateResult>;
|
|
|
|
/** Count the total number of rows in the dataset. */
|
|
abstract countRows(filter?: string): Promise<number>;
|
|
/**
|
|
* Delete the rows that satisfy the predicate.
|
|
* @returns {Promise<DeleteResult>} A promise that resolves to an object
|
|
* containing the new version number of the table
|
|
*/
|
|
abstract delete(predicate: string): Promise<DeleteResult>;
|
|
/**
|
|
* Create an index to speed up queries.
|
|
*
|
|
* Indices can be created on vector columns or scalar columns.
|
|
* Indices on vector columns will speed up vector searches.
|
|
* Indices on scalar columns will speed up filtering (in both
|
|
* vector and non-vector searches)
|
|
*
|
|
* We currently don't support custom named indexes.
|
|
* The index name will always be `${column}_idx`.
|
|
*
|
|
* @example
|
|
* // If the column has a vector (fixed size list) data type then
|
|
* // an IvfPq vector index will be created.
|
|
* const table = await conn.openTable("my_table");
|
|
* await table.createIndex("vector");
|
|
* @example
|
|
* // For advanced control over vector index creation you can specify
|
|
* // the index type and options.
|
|
* const table = await conn.openTable("my_table");
|
|
* await table.createIndex("vector", {
|
|
* config: lancedb.Index.ivfPq({
|
|
* numPartitions: 128,
|
|
* numSubVectors: 16,
|
|
* }),
|
|
* });
|
|
* @example
|
|
* // Or create a Scalar index
|
|
* await table.createIndex("my_float_col");
|
|
*/
|
|
abstract createIndex(
|
|
column: string,
|
|
options?: Partial<IndexOptions>,
|
|
): Promise<void>;
|
|
|
|
/**
|
|
* Create an index, returning a handle to the indexing job.
|
|
*
|
|
* The job may already be complete when returned; callers must not assume
|
|
* the index exists until {@link Job.wait} resolves.
|
|
*/
|
|
abstract createIndexAsync(
|
|
column: string,
|
|
options?: Partial<IndexOptions>,
|
|
): Promise<Job>;
|
|
|
|
/**
|
|
* Drop an index from the table.
|
|
*
|
|
* @param name The name of the index.
|
|
*
|
|
* This does not delete the index from disk, it just removes it from the table.
|
|
* To delete the index, run {@link Table#optimize} after dropping the index.
|
|
*
|
|
* Use {@link Table.listIndices} to find the names of the indices.
|
|
*/
|
|
abstract dropIndex(name: string): Promise<void>;
|
|
|
|
/**
|
|
* Prewarm an index in the table.
|
|
*
|
|
* @param name The name of the index.
|
|
*
|
|
* This will load the index into memory. This may reduce the cold-start time for
|
|
* future queries. If the index does not fit in the cache then this call may be
|
|
* wasteful.
|
|
*/
|
|
abstract prewarmIndex(name: string): Promise<void>;
|
|
|
|
/**
|
|
* Prewarm one or more columns of data in the table.
|
|
*
|
|
* @param columns The columns to prewarm. If undefined, all columns are prewarmed.
|
|
*
|
|
* This will load the column data into the page cache so that future queries that
|
|
* read those columns avoid the initial cold-start latency. This call initiates
|
|
* prewarming and returns once the request is accepted; the warming itself may
|
|
* continue in the background. Calling it on already-prewarmed columns is a
|
|
* no-op on the server.
|
|
*
|
|
* Prewarming is generally useful for columns used in filters or projections.
|
|
* Large columns (e.g. high-dimensional vectors or binary data) may not be
|
|
* practical to prewarm.
|
|
*
|
|
* This feature is currently only supported on remote tables.
|
|
*/
|
|
abstract prewarmData(columns?: string[]): Promise<void>;
|
|
|
|
/**
|
|
* Waits for asynchronous indexing to complete on the table.
|
|
*
|
|
* @param indexNames The name of the indices to wait for
|
|
* @param timeoutSeconds The number of seconds to wait before timing out
|
|
*
|
|
* This will raise an error if the indices are not created and fully indexed within the timeout.
|
|
*/
|
|
abstract waitForIndex(
|
|
indexNames: string[],
|
|
timeoutSeconds: number,
|
|
): Promise<void>;
|
|
|
|
/**
|
|
* Create a {@link Query} Builder.
|
|
*
|
|
* Queries allow you to search your existing data. By default the query will
|
|
* return all the data in the table in no particular order. The builder
|
|
* returned by this method can be used to control the query using filtering,
|
|
* vector similarity, sorting, and more.
|
|
*
|
|
* Note: By default, all columns are returned. For best performance, you should
|
|
* only fetch the columns you need.
|
|
*
|
|
* When appropriate, various indices and statistics based pruning will be used to
|
|
* accelerate the query.
|
|
* @example
|
|
* // SQL-style filtering
|
|
* //
|
|
* // This query will return up to 1000 rows whose value in the `id` column
|
|
* // is greater than 5. LanceDb supports a broad set of filtering functions.
|
|
* for await (const batch of table
|
|
* .query()
|
|
* .where("id > 1")
|
|
* .select(["id"])
|
|
* .limit(20)) {
|
|
* console.log(batch);
|
|
* }
|
|
* @example
|
|
* // Vector Similarity Search
|
|
* //
|
|
* // This example will find the 10 rows whose value in the "vector" column are
|
|
* // closest to the query vector [1.0, 2.0, 3.0]. If an index has been created
|
|
* // on the "vector" column then this will perform an ANN search.
|
|
* //
|
|
* // The `refineFactor` and `nprobes` methods are used to control the recall /
|
|
* // latency tradeoff of the search.
|
|
* for await (const batch of table
|
|
* .query()
|
|
* .where("id > 1")
|
|
* .select(["id"])
|
|
* .limit(20)) {
|
|
* console.log(batch);
|
|
* }
|
|
* @example
|
|
* // Scan the full dataset
|
|
* //
|
|
* // This query will return everything in the table in no particular order.
|
|
* for await (const batch of table.query()) {
|
|
* console.log(batch);
|
|
* }
|
|
* @returns {Query} A builder that can be used to parameterize the query
|
|
*/
|
|
abstract query(): Query;
|
|
|
|
/**
|
|
* Create a query that returns a subset of the rows in the table.
|
|
* @param offsets The offsets of the rows to return.
|
|
* @returns A builder that can be used to parameterize the query.
|
|
*/
|
|
abstract takeOffsets(offsets: number[]): TakeQuery;
|
|
|
|
/**
|
|
* Create a query that returns a subset of the rows in the table.
|
|
* @param rowIds The row ids of the rows to return.
|
|
*
|
|
* Row ids returned by `withRowId()` are `bigint`, so `bigint[]` is supported.
|
|
* For convenience / backwards compatibility, `number[]` is also accepted (for
|
|
* small row ids that fit in a safe integer).
|
|
* @returns A builder that can be used to parameterize the query.
|
|
*/
|
|
abstract takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery;
|
|
|
|
/**
|
|
* Create a search query to find the nearest neighbors
|
|
* of the given query
|
|
* @param {string | IntoVector} query - the query, a vector or string
|
|
* @param {string} queryType - the type of the query, "vector", "fts", or "auto"
|
|
* @param {string | string[]} ftsColumns - the columns to search in for full text search
|
|
* for now, only one column can be searched at a time.
|
|
*
|
|
* when "auto" is used, if the query is a string and an embedding function is defined, it will be treated as a vector query
|
|
* if the query is a string and no embedding function is defined, it will be treated as a full text search query
|
|
*/
|
|
abstract search(
|
|
query: string | IntoVector | MultiVector | FullTextQuery,
|
|
queryType?: string,
|
|
ftsColumns?: string | string[],
|
|
): VectorQuery | Query;
|
|
/**
|
|
* Search the table with a given query vector.
|
|
*
|
|
* This is a convenience method for preparing a vector query and
|
|
* is the same thing as calling `nearestTo` on the builder returned
|
|
* by `query`. @see {@link Query#nearestTo} for more details.
|
|
*/
|
|
abstract vectorSearch(vector: IntoVector | MultiVector): VectorQuery;
|
|
/**
|
|
* Add new columns with defined values.
|
|
*
|
|
* The `{ computed }` form stores the expression rather than evaluating it
|
|
* now: the column is committed with no values, and rows get them from
|
|
* {@link Table#refreshColumn}. Declaring one therefore costs the same on a
|
|
* large table as on an empty one.
|
|
*
|
|
* A refresh does not revisit rows it has already filled, so mutating an
|
|
* input leaves the value computed at fill time; recomputing means dropping
|
|
* the column and declaring it again. While a declaration reads a column,
|
|
* that column cannot be renamed, retyped or dropped.
|
|
*
|
|
* On LanceDB Cloud and Enterprise the expression is planned by the
|
|
* server, and the refresh runs as a server job -- see
|
|
* {@link Table#refreshColumnAsync}.
|
|
* @param {AddColumnsSql[] | Field | Field[] | Schema} newColumnTransforms Either:
|
|
* - An array of objects with column names and SQL expressions to calculate values
|
|
* - A single Arrow Field defining one column with its data type (column will be initialized with null values)
|
|
* - An array of Arrow Fields defining columns with their data types (columns will be initialized with null values)
|
|
* - An Arrow Schema defining columns with their data types (columns will be initialized with null values)
|
|
* - `{ computed }`, declaring columns defined by a SQL expression whose type and inputs are derived from it
|
|
* @returns {Promise<AddColumnsResult>} A promise that resolves to an object
|
|
* containing the new version number of the table after adding the columns.
|
|
* @example
|
|
* ```ts
|
|
* await table.addColumns({ computed: [{ name: "doubled", valueSql: "x * 2" }] });
|
|
* const { rowsFilled } = await table.refreshColumn("doubled");
|
|
* ```
|
|
*/
|
|
abstract addColumns(
|
|
newColumnTransforms:
|
|
| AddColumnsSql[]
|
|
| Field
|
|
| Field[]
|
|
| Schema
|
|
| { computed: AddColumnsSql[] },
|
|
): Promise<AddColumnsResult>;
|
|
|
|
/**
|
|
* Fill the rows of a computed column that hold no value yet.
|
|
*
|
|
* Rows appended since the last refresh are filled by the next one; rows
|
|
* already filled are left as they are, so the call is idempotent and does
|
|
* not observe a mutated input. Local tables only: a remote refresh runs
|
|
* as a server job, through {@link Table#refreshColumnAsync}.
|
|
* @param {string} column The name of the computed column to fill.
|
|
* @returns {Promise<RefreshColumnResult>} A promise that resolves to the
|
|
* number of rows filled and the new version number of the table.
|
|
*/
|
|
abstract refreshColumn(column: string): Promise<RefreshColumnResult>;
|
|
|
|
/**
|
|
* Like {@link Table#refreshColumn}, but returns a handle to the refresh
|
|
* job instead of blocking until it completes.
|
|
*
|
|
* The job may already be complete when returned; callers must not assume
|
|
* the column is filled until {@link Job.wait} resolves. Invalid input --
|
|
* an unknown column, or one that is not computed -- rejects here rather
|
|
* than failing the job. On local tables the job runs in-process; on
|
|
* LanceDB Cloud and Enterprise it is the server's backfill job.
|
|
* @param {string} column The name of the computed column to fill.
|
|
* @example
|
|
* ```ts
|
|
* const job = await table.refreshColumnAsync("doubled");
|
|
* await job.wait();
|
|
* console.log(await job.status()); // "finished"
|
|
* ```
|
|
*/
|
|
abstract refreshColumnAsync(column: string): Promise<Job>;
|
|
|
|
/**
|
|
* Alter the name or nullability of columns.
|
|
* @param {ColumnAlteration[]} columnAlterations One or more alterations to
|
|
* apply to columns.
|
|
* @returns {Promise<AlterColumnsResult>} A promise that resolves to an object
|
|
* containing the new version number of the table after altering the columns.
|
|
*/
|
|
abstract alterColumns(
|
|
columnAlterations: ColumnAlteration[],
|
|
): Promise<AlterColumnsResult>;
|
|
|
|
/**
|
|
* Update per-field (column) metadata.
|
|
* @param {FieldMetadataUpdate[]} updates One or more per-field updates. Each
|
|
* update's metadata is merged into the field's existing metadata by default;
|
|
* a value of `null` deletes that key, and `replace: true` swaps the whole map.
|
|
* @returns {Promise<UpdateFieldMetadataResult>} resolves to the new table version.
|
|
*/
|
|
abstract updateFieldMetadata(
|
|
updates: FieldMetadataUpdate[],
|
|
): Promise<UpdateFieldMetadataResult>;
|
|
|
|
/**
|
|
* Drop one or more columns from the dataset
|
|
*
|
|
* This is a metadata-only operation and does not remove the data from the
|
|
* underlying storage. In order to remove the data, you must subsequently
|
|
* call ``compact_files`` to rewrite the data without the removed columns and
|
|
* then call ``cleanup_files`` to remove the old files.
|
|
* @param {string[]} columnNames The names of the columns to drop. These can
|
|
* be nested column references (e.g. "a.b.c") or top-level column names
|
|
* (e.g. "a").
|
|
* @returns {Promise<DropColumnsResult>} A promise that resolves to an object
|
|
* containing the new version number of the table after dropping the columns.
|
|
*/
|
|
abstract dropColumns(columnNames: string[]): Promise<DropColumnsResult>;
|
|
/**
|
|
* Set the unenforced primary key for this table to a single column.
|
|
*
|
|
* "Unenforced" means LanceDB does not check uniqueness on writes; the
|
|
* column is recorded in the schema as the primary key for use by features
|
|
* such as `merge_insert`. Only single-column primary keys are supported,
|
|
* and the key cannot be changed once set.
|
|
* @param {string | string[]} columns The primary key column. A one-element
|
|
* array is also accepted; passing more than one column is rejected.
|
|
* @returns {Promise<void>}
|
|
*/
|
|
abstract setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
|
|
/**
|
|
* Install an {@link LsmWriteSpec} on this table, selecting Lance's MemWAL
|
|
* LSM-style write path for future `mergeInsert` calls.
|
|
*
|
|
* `LsmWriteSpec` chooses one of three sharding strategies via `specType`:
|
|
*
|
|
* - `"bucket"` — hash-bucket writes by the single-column unenforced primary
|
|
* key (`column` and `numBuckets` required).
|
|
* - `"identity"` — shard by the raw value of a scalar `column`.
|
|
* - `"unsharded"` — route every write to a single shard.
|
|
*
|
|
* All variants require the table to have an unenforced primary key
|
|
* ({@link Table#setUnenforcedPrimaryKey}); bucket sharding additionally
|
|
* requires it to be the single column being bucketed.
|
|
*
|
|
* Omitting `maintainedIndexes` maintains every index on the table, resolved
|
|
* here, failing if one cannot be maintained — name them to install anyway.
|
|
* Naming them pins an exact set, and a still-building index is rejected
|
|
* rather than quietly omitted.
|
|
* @param {LsmWriteSpec} spec The sharding spec to install.
|
|
* @returns {Promise<void>}
|
|
* @example
|
|
* ```ts
|
|
* await table.setUnenforcedPrimaryKey("id");
|
|
* await table.setLsmWriteSpec({
|
|
* specType: "bucket",
|
|
* column: "id",
|
|
* numBuckets: 16,
|
|
* maintainedIndexes: ["id_idx"],
|
|
* });
|
|
* ```
|
|
*/
|
|
abstract setLsmWriteSpec(spec: LsmWriteSpec): Promise<void>;
|
|
/**
|
|
* Remove the {@link LsmWriteSpec} from this table, reverting to the standard
|
|
* `mergeInsert` write path.
|
|
*
|
|
* Errors if no spec is currently set.
|
|
* @returns {Promise<void>}
|
|
*/
|
|
abstract unsetLsmWriteSpec(): Promise<void>;
|
|
/**
|
|
* Read the {@link LsmWriteSpec} currently installed on this table.
|
|
*
|
|
* Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
|
|
* spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}).
|
|
* The returned spec mirrors what was passed to
|
|
* {@link Table#setLsmWriteSpec}, except that `maintainedIndexes` always
|
|
* reports the concrete list resolved when the spec was set — `undefined`
|
|
* never round-trips.
|
|
* @returns {Promise<LsmWriteSpec | undefined>}
|
|
*/
|
|
abstract getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
|
|
/**
|
|
* Drain and close any cached MemWAL shard writers held for this table.
|
|
*
|
|
* When an {@link LsmWriteSpec} is installed, `mergeInsert` opens MemWAL
|
|
* shard writers and caches them for reuse across calls. This closes them,
|
|
* flushing pending data; writers reopen lazily on the next `mergeInsert`.
|
|
* It is a no-op when no writers are cached.
|
|
* @returns {Promise<void>}
|
|
*/
|
|
abstract closeLsmWriters(): Promise<void>;
|
|
/** Retrieve the version of the table */
|
|
|
|
abstract version(): Promise<number>;
|
|
/**
|
|
* Checks out a specific version of the table _This is an in-place operation._
|
|
*
|
|
* This allows viewing previous versions of the table. If you wish to
|
|
* keep writing to the dataset starting from an old version, then use
|
|
* the `restore` function.
|
|
*
|
|
* Calling this method will set the table into time-travel mode. If you
|
|
* wish to return to standard mode, call `checkoutLatest`.
|
|
* @param {number | string} version The version to checkout, could be version number or tag
|
|
* @example
|
|
* ```typescript
|
|
* import * as lancedb from "@lancedb/lancedb"
|
|
* const db = await lancedb.connect("./.lancedb");
|
|
* const table = await db.createTable("my_table", [
|
|
* { vector: [1.1, 0.9], type: "vector" },
|
|
* ]);
|
|
*
|
|
* console.log(await table.version()); // 1
|
|
* console.log(table.display());
|
|
* await table.add([{ vector: [0.5, 0.2], type: "vector" }]);
|
|
* await table.checkout(1);
|
|
* console.log(await table.version()); // 2
|
|
* ```
|
|
*/
|
|
abstract checkout(version: number | string): Promise<void>;
|
|
|
|
/**
|
|
* Checkout the latest version of the table. _This is an in-place operation._
|
|
*
|
|
* The table will be set back into standard mode, and will track the latest
|
|
* version of the table.
|
|
*/
|
|
abstract checkoutLatest(): Promise<void>;
|
|
|
|
/**
|
|
* List all the versions of the table
|
|
*/
|
|
abstract listVersions(): Promise<Version[]>;
|
|
|
|
/**
|
|
* Get a tags manager for this table.
|
|
*
|
|
* Tags allow you to label specific versions of a table with a human-readable name.
|
|
* The returned tags manager can be used to list, create, update, or delete tags.
|
|
*
|
|
* @returns {Tags} A tags manager for this table
|
|
* @example
|
|
* ```typescript
|
|
* const tagsManager = await table.tags();
|
|
* await tagsManager.create("v1", 1);
|
|
* const tags = await tagsManager.list();
|
|
* console.log(tags); // { "v1": { version: 1, manifestSize: ... } }
|
|
* ```
|
|
*/
|
|
abstract tags(): Promise<Tags>;
|
|
|
|
/**
|
|
* Get the branch manager for this table.
|
|
*
|
|
* Branches are isolated, writable lines of history forked from another
|
|
* branch (or version). Writes on a branch do not affect `main`.
|
|
*/
|
|
abstract branches(): Promise<Branches>;
|
|
|
|
/**
|
|
* The branch this table handle is scoped to, or `null` for the main branch.
|
|
*
|
|
* A handle returned by {@link Branches.create} or {@link Branches.checkout}
|
|
* reports the branch it targets; a handle opened normally reports `null`.
|
|
*/
|
|
abstract currentBranch(): string | null;
|
|
|
|
/**
|
|
* Restore the table to the currently checked out version
|
|
*
|
|
* This operation will fail if checkout has not been called previously
|
|
*
|
|
* This operation will overwrite the latest version of the table with a
|
|
* previous version. Any changes made since the checked out version will
|
|
* no longer be visible.
|
|
*
|
|
* Once the operation concludes the table will no longer be in a checked
|
|
* out state and the read_consistency_interval, if any, will apply.
|
|
*/
|
|
abstract restore(): Promise<void>;
|
|
/**
|
|
* Optimize the on-disk data and indices for better performance.
|
|
*
|
|
* Modeled after ``VACUUM`` in PostgreSQL.
|
|
*
|
|
* Optimization covers three operations:
|
|
*
|
|
* - Compaction: Merges small files into larger ones
|
|
* - Prune: Removes old versions of the dataset
|
|
* - Index: Optimizes the indices, adding new data to existing indices
|
|
*
|
|
*
|
|
* The frequency an application should call optimize is based on the frequency of
|
|
* data modifications. If data is frequently added, deleted, or updated then
|
|
* optimize should be run frequently. A good rule of thumb is to run optimize if
|
|
* you have added or modified 100,000 or more records or run more than 20 data
|
|
* modification operations.
|
|
*/
|
|
abstract optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
|
|
/** List all indices that have been created with {@link Table.createIndex} */
|
|
abstract listIndices(): Promise<IndexConfig[]>;
|
|
/**
|
|
* Tokenize a full-text search query using the tokenizer configured on an FTS index.
|
|
*
|
|
* Specify exactly one of `column` or `indexName`.
|
|
*
|
|
* Model-backed tokenizers such as `jieba/*` and `lindera/*` are rebuilt in
|
|
* the client process from index metadata. For remote tables, this means the
|
|
* same tokenizer model files must also exist locally.
|
|
*/
|
|
abstract tokenize(
|
|
query: string,
|
|
options: TokenizeTableOptions,
|
|
): Promise<FtsToken[]>;
|
|
/** Return the table as an arrow table */
|
|
abstract toArrow(): Promise<ArrowTable>;
|
|
|
|
abstract mergeInsert(on: string | string[]): MergeInsertBuilder;
|
|
|
|
/** List all the stats of a specified index
|
|
*
|
|
* @param {string} name The name of the index.
|
|
* @returns {IndexStatistics | undefined} The stats of the index. If the index does not exist, it will return undefined
|
|
*
|
|
* Use {@link Table.listIndices} to find the names of the indices.
|
|
*/
|
|
abstract indexStats(name: string): Promise<IndexStatistics | undefined>;
|
|
|
|
/** Returns table and fragment statistics
|
|
*
|
|
* @returns {TableStatistics} The table and fragment statistics
|
|
*
|
|
*/
|
|
abstract stats(): Promise<TableStatistics>;
|
|
|
|
/**
|
|
* Get the initial storage options that were passed in when opening this table.
|
|
*
|
|
* For dynamically refreshed options (e.g., credential vending), use
|
|
* {@link Table.latestStorageOptions}.
|
|
*
|
|
* Warning: This is an internal API and the return value is subject to change.
|
|
*
|
|
* @returns The storage options, or undefined if no storage options were configured.
|
|
*/
|
|
abstract initialStorageOptions(): Promise<
|
|
Record<string, string> | null | undefined
|
|
>;
|
|
|
|
/**
|
|
* Get the latest storage options, refreshing from provider if configured.
|
|
*
|
|
* This method is useful for credential vending scenarios where storage options
|
|
* may be refreshed dynamically. If no dynamic provider is configured, this
|
|
* returns the initial static options.
|
|
*
|
|
* Warning: This is an internal API and the return value is subject to change.
|
|
*
|
|
* @returns The storage options, or undefined if no storage options were configured.
|
|
*/
|
|
abstract latestStorageOptions(): Promise<
|
|
Record<string, string> | null | undefined
|
|
>;
|
|
}
|
|
|
|
export class LocalTable extends Table {
|
|
private readonly inner: _NativeTable;
|
|
|
|
constructor(inner: _NativeTable) {
|
|
super();
|
|
this.inner = inner;
|
|
}
|
|
get name(): string {
|
|
return this.inner.name;
|
|
}
|
|
isOpen(): boolean {
|
|
return this.inner.isOpen();
|
|
}
|
|
|
|
close(): void {
|
|
this.inner.close();
|
|
}
|
|
|
|
display(): string {
|
|
return this.inner.display();
|
|
}
|
|
|
|
private async getEmbeddingFunctions(): Promise<
|
|
Map<string, EmbeddingFunctionConfig>
|
|
> {
|
|
const schema = await this.schema();
|
|
const registry = getRegistry();
|
|
return registry.parseFunctions(schema.metadata);
|
|
}
|
|
|
|
/** Get the schema of the table. */
|
|
async schema(): Promise<Schema> {
|
|
const schemaBuf = await this.inner.schema();
|
|
const tbl = tableFromIPC(schemaBuf);
|
|
return tbl.schema;
|
|
}
|
|
|
|
async add(data: Data, options?: Partial<AddDataOptions>): Promise<AddResult> {
|
|
const mode = options?.mode ?? "append";
|
|
const schema = await this.schema();
|
|
|
|
const buffer = await fromDataToBuffer(data, undefined, schema);
|
|
// Wrap the user callback so a thrown error doesn't surface as an
|
|
// unhandled exception (the callback fires from a napi threadsafe
|
|
// function — exceptions there crash the process).
|
|
const userProgress = options?.progress;
|
|
const progress = userProgress
|
|
? (p: WriteProgress) => {
|
|
try {
|
|
userProgress(p);
|
|
} catch (e) {
|
|
console.warn("Table.add progress callback threw:", e);
|
|
}
|
|
}
|
|
: undefined;
|
|
return await this.inner.add(buffer, mode, progress);
|
|
}
|
|
|
|
async update(
|
|
optsOrUpdates:
|
|
| (Map<string, string> | Record<string, string>)
|
|
| ({
|
|
values: Map<string, IntoSql> | Record<string, IntoSql>;
|
|
} & Partial<UpdateOptions>)
|
|
| ({
|
|
valuesSql: Map<string, string> | Record<string, string>;
|
|
} & Partial<UpdateOptions>),
|
|
options?: Partial<UpdateOptions>,
|
|
): Promise<UpdateResult> {
|
|
const isValues =
|
|
"values" in optsOrUpdates && typeof optsOrUpdates.values !== "string";
|
|
const isValuesSql =
|
|
"valuesSql" in optsOrUpdates &&
|
|
typeof optsOrUpdates.valuesSql !== "string";
|
|
const isMap = (obj: unknown): obj is Map<string, string> => {
|
|
return obj instanceof Map;
|
|
};
|
|
|
|
let predicate;
|
|
let columns: [string, string][];
|
|
switch (true) {
|
|
case isMap(optsOrUpdates):
|
|
columns = Array.from(optsOrUpdates.entries());
|
|
predicate = options?.where;
|
|
break;
|
|
case isValues && isMap(optsOrUpdates.values):
|
|
columns = Array.from(optsOrUpdates.values.entries()).map(([k, v]) => [
|
|
k,
|
|
toSQL(v),
|
|
]);
|
|
predicate = optsOrUpdates.where;
|
|
break;
|
|
case isValues && !isMap(optsOrUpdates.values):
|
|
columns = Object.entries(optsOrUpdates.values).map(([k, v]) => [
|
|
k,
|
|
toSQL(v),
|
|
]);
|
|
predicate = optsOrUpdates.where;
|
|
break;
|
|
|
|
case isValuesSql && isMap(optsOrUpdates.valuesSql):
|
|
columns = Array.from(optsOrUpdates.valuesSql.entries());
|
|
predicate = optsOrUpdates.where;
|
|
break;
|
|
case isValuesSql && !isMap(optsOrUpdates.valuesSql):
|
|
columns = Object.entries(optsOrUpdates.valuesSql).map(([k, v]) => [
|
|
k,
|
|
v,
|
|
]);
|
|
predicate = optsOrUpdates.where;
|
|
break;
|
|
default:
|
|
columns = Object.entries(optsOrUpdates as Record<string, string>);
|
|
predicate = options?.where;
|
|
}
|
|
return await this.inner.update(predicate, columns);
|
|
}
|
|
|
|
async countRows(filter?: string): Promise<number> {
|
|
return await this.inner.countRows(filter);
|
|
}
|
|
|
|
async delete(predicate: string): Promise<DeleteResult> {
|
|
return await this.inner.delete(predicate);
|
|
}
|
|
|
|
async createIndex(column: string, options?: Partial<IndexOptions>) {
|
|
// Bit of a hack to get around the fact that TS has no package-scope.
|
|
// biome-ignore lint/suspicious/noExplicitAny: skip
|
|
const nativeIndex = (options?.config as any)?.inner;
|
|
await this.inner.createIndex(
|
|
nativeIndex,
|
|
column,
|
|
options?.replace,
|
|
options?.waitTimeoutSeconds,
|
|
options?.name,
|
|
options?.train,
|
|
);
|
|
}
|
|
|
|
async createIndexAsync(
|
|
column: string,
|
|
options?: Partial<IndexOptions>,
|
|
): Promise<Job> {
|
|
// biome-ignore lint/suspicious/noExplicitAny: skip
|
|
const nativeIndex = (options?.config as any)?.inner;
|
|
return await this.inner.createIndexAsync(
|
|
nativeIndex,
|
|
column,
|
|
options?.replace,
|
|
options?.waitTimeoutSeconds,
|
|
options?.name,
|
|
options?.train,
|
|
);
|
|
}
|
|
|
|
async dropIndex(name: string): Promise<void> {
|
|
await this.inner.dropIndex(name);
|
|
}
|
|
|
|
async prewarmIndex(name: string): Promise<void> {
|
|
await this.inner.prewarmIndex(name);
|
|
}
|
|
|
|
async prewarmData(columns?: string[]): Promise<void> {
|
|
await this.inner.prewarmData(columns);
|
|
}
|
|
|
|
async waitForIndex(
|
|
indexNames: string[],
|
|
timeoutSeconds: number,
|
|
): Promise<void> {
|
|
await this.inner.waitForIndex(indexNames, timeoutSeconds);
|
|
}
|
|
|
|
takeOffsets(offsets: number[]): TakeQuery {
|
|
return new TakeQuery(this.inner.takeOffsets(offsets));
|
|
}
|
|
|
|
takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery {
|
|
const ids = rowIds.map((id) => {
|
|
if (typeof id === "bigint") {
|
|
return id;
|
|
}
|
|
if (!Number.isInteger(id)) {
|
|
throw new Error("Row id must be an integer (or bigint)");
|
|
}
|
|
if (id < 0) {
|
|
throw new Error("Row id cannot be negative");
|
|
}
|
|
if (!Number.isSafeInteger(id)) {
|
|
throw new Error("Row id is too large for number; use bigint instead");
|
|
}
|
|
return BigInt(id);
|
|
});
|
|
|
|
return new TakeQuery(this.inner.takeRowIds(ids));
|
|
}
|
|
|
|
query(): Query {
|
|
return new Query(this.inner);
|
|
}
|
|
|
|
search(
|
|
query: string | IntoVector | MultiVector | FullTextQuery,
|
|
queryType: string = "auto",
|
|
ftsColumns?: string | string[],
|
|
): VectorQuery | Query {
|
|
if (typeof query !== "string" && !instanceOfFullTextQuery(query)) {
|
|
if (queryType === "fts") {
|
|
throw new Error("Cannot perform full text search on a vector query");
|
|
}
|
|
return this.vectorSearch(query);
|
|
}
|
|
|
|
// If the query is a string, we need to determine if it is a vector query or a full text search query
|
|
if (queryType === "fts") {
|
|
return this.query().fullTextSearch(query, {
|
|
columns: ftsColumns,
|
|
});
|
|
}
|
|
|
|
// The query type is auto or vector
|
|
// fall back to full text search if no embedding functions are defined and the query is a string
|
|
if (
|
|
queryType === "auto" &&
|
|
(getRegistry().length() === 0 || instanceOfFullTextQuery(query))
|
|
) {
|
|
return this.query().fullTextSearch(query, {
|
|
columns: ftsColumns,
|
|
});
|
|
}
|
|
|
|
const queryPromise = this.getEmbeddingFunctions().then(
|
|
async (functions) => {
|
|
// TODO: Support multiple embedding functions
|
|
const embeddingFunc: EmbeddingFunctionConfig | undefined = functions
|
|
.values()
|
|
.next().value;
|
|
if (!embeddingFunc) {
|
|
return Promise.reject(
|
|
new Error("No embedding functions are defined in the table"),
|
|
);
|
|
}
|
|
return await embeddingFunc.function.computeQueryEmbeddings(query);
|
|
},
|
|
);
|
|
|
|
return this.query().nearestTo(queryPromise);
|
|
}
|
|
|
|
vectorSearch(vector: IntoVector | MultiVector): VectorQuery {
|
|
if (isMultiVector(vector)) {
|
|
const query = this.query().nearestTo(vector[0]);
|
|
for (const v of vector.slice(1)) {
|
|
query.addQueryVector(v);
|
|
}
|
|
return query;
|
|
}
|
|
|
|
return this.query().nearestTo(vector);
|
|
}
|
|
|
|
// TODO: Support BatchUDF
|
|
|
|
async addColumns(
|
|
newColumnTransforms:
|
|
| AddColumnsSql[]
|
|
| Field
|
|
| Field[]
|
|
| Schema
|
|
| { computed: AddColumnsSql[] },
|
|
): Promise<AddColumnsResult> {
|
|
// Columns defined by an expression are declared, not materialized here.
|
|
if (
|
|
typeof newColumnTransforms === "object" &&
|
|
!Array.isArray(newColumnTransforms) &&
|
|
"computed" in newColumnTransforms
|
|
) {
|
|
return await this.inner.addComputedColumns(newColumnTransforms.computed);
|
|
}
|
|
|
|
// Handle single Field -> convert to array of Fields
|
|
if (newColumnTransforms instanceof Field) {
|
|
newColumnTransforms = [newColumnTransforms];
|
|
}
|
|
|
|
// Handle array of Fields -> convert to Schema
|
|
if (
|
|
Array.isArray(newColumnTransforms) &&
|
|
newColumnTransforms.length > 0 &&
|
|
newColumnTransforms[0] instanceof Field
|
|
) {
|
|
const fields = newColumnTransforms as Field[];
|
|
newColumnTransforms = new Schema(fields);
|
|
}
|
|
|
|
// Handle Schema -> use schema-based approach
|
|
if (newColumnTransforms instanceof Schema) {
|
|
const schema = newColumnTransforms;
|
|
// Convert schema to buffer using Arrow IPC format
|
|
const emptyTable = makeEmptyTable(schema);
|
|
const schemaBuf = await fromTableToBuffer(emptyTable);
|
|
return await this.inner.addColumnsWithSchema(schemaBuf);
|
|
}
|
|
|
|
// Handle SQL expressions (existing functionality)
|
|
if (Array.isArray(newColumnTransforms)) {
|
|
return await this.inner.addColumns(
|
|
newColumnTransforms as AddColumnsSql[],
|
|
);
|
|
}
|
|
|
|
throw new Error("Invalid input type for addColumns");
|
|
}
|
|
|
|
async refreshColumn(column: string): Promise<RefreshColumnResult> {
|
|
return await this.inner.refreshColumn(column);
|
|
}
|
|
|
|
async refreshColumnAsync(column: string): Promise<Job> {
|
|
return await this.inner.refreshColumnAsync(column);
|
|
}
|
|
|
|
async alterColumns(
|
|
columnAlterations: ColumnAlteration[],
|
|
): Promise<AlterColumnsResult> {
|
|
const processedAlterations = columnAlterations.map((alteration) => {
|
|
if (typeof alteration.dataType === "string") {
|
|
return {
|
|
...alteration,
|
|
dataType: JSON.stringify({ type: alteration.dataType }),
|
|
};
|
|
} else if (alteration.dataType === undefined) {
|
|
return {
|
|
...alteration,
|
|
dataType: undefined,
|
|
};
|
|
} else {
|
|
const dataType = sanitizeType(alteration.dataType);
|
|
return {
|
|
...alteration,
|
|
dataType: JSON.stringify(dataTypeToJson(dataType)),
|
|
};
|
|
}
|
|
});
|
|
|
|
return await this.inner.alterColumns(processedAlterations);
|
|
}
|
|
|
|
async updateFieldMetadata(
|
|
updates: FieldMetadataUpdate[],
|
|
): Promise<UpdateFieldMetadataResult> {
|
|
return await this.inner.updateFieldMetadata(updates);
|
|
}
|
|
|
|
async dropColumns(columnNames: string[]): Promise<DropColumnsResult> {
|
|
return await this.inner.dropColumns(columnNames);
|
|
}
|
|
|
|
async setUnenforcedPrimaryKey(columns: string | string[]): Promise<void> {
|
|
const cols = typeof columns === "string" ? [columns] : columns;
|
|
return await this.inner.setUnenforcedPrimaryKey(cols);
|
|
}
|
|
|
|
async setLsmWriteSpec(spec: LsmWriteSpec): Promise<void> {
|
|
return await this.inner.setLsmWriteSpec(spec);
|
|
}
|
|
|
|
async unsetLsmWriteSpec(): Promise<void> {
|
|
return await this.inner.unsetLsmWriteSpec();
|
|
}
|
|
|
|
async getLsmWriteSpec(): Promise<LsmWriteSpec | undefined> {
|
|
// The native binding types `specType` as a plain `string`; narrow it back
|
|
// to the public union. The Rust `From` impl only ever emits one of the
|
|
// three valid values, so the cast is safe.
|
|
return ((await this.inner.getLsmWriteSpec()) ?? undefined) as
|
|
| LsmWriteSpec
|
|
| undefined;
|
|
}
|
|
|
|
async closeLsmWriters(): Promise<void> {
|
|
return await this.inner.closeLsmWriters();
|
|
}
|
|
|
|
async version(): Promise<number> {
|
|
return await this.inner.version();
|
|
}
|
|
|
|
async checkout(version: number | string): Promise<void> {
|
|
if (typeof version === "string") {
|
|
return this.inner.checkoutTag(version);
|
|
}
|
|
return this.inner.checkout(version);
|
|
}
|
|
|
|
async checkoutLatest(): Promise<void> {
|
|
await this.inner.checkoutLatest();
|
|
}
|
|
|
|
async listVersions(): Promise<Version[]> {
|
|
return (await this.inner.listVersions()).map((version) => ({
|
|
version: version.version,
|
|
timestamp: new Date(version.timestamp / 1000),
|
|
metadata: version.metadata,
|
|
}));
|
|
}
|
|
|
|
async restore(): Promise<void> {
|
|
await this.inner.restore();
|
|
}
|
|
|
|
async tags(): Promise<Tags> {
|
|
return await this.inner.tags();
|
|
}
|
|
|
|
async branches(): Promise<Branches> {
|
|
return new Branches(await this.inner.branches());
|
|
}
|
|
|
|
currentBranch(): string | null {
|
|
return this.inner.currentBranch() ?? null;
|
|
}
|
|
|
|
async optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats> {
|
|
let cleanupOlderThanMs;
|
|
if (
|
|
options?.cleanupOlderThan !== undefined &&
|
|
options?.cleanupOlderThan !== null
|
|
) {
|
|
cleanupOlderThanMs =
|
|
new Date().getTime() - options.cleanupOlderThan.getTime();
|
|
}
|
|
return await this.inner.optimize(
|
|
cleanupOlderThanMs,
|
|
options?.deleteUnverified,
|
|
);
|
|
}
|
|
|
|
async listIndices(): Promise<IndexConfig[]> {
|
|
return await this.inner.listIndices();
|
|
}
|
|
|
|
async tokenize(
|
|
query: string,
|
|
options: TokenizeTableOptions,
|
|
): Promise<FtsToken[]> {
|
|
return await this.inner.tokenize(
|
|
query,
|
|
options?.column,
|
|
options?.indexName,
|
|
);
|
|
}
|
|
|
|
async toArrow(): Promise<ArrowTable> {
|
|
return await this.query().toArrow();
|
|
}
|
|
|
|
async indexStats(name: string): Promise<IndexStatistics | undefined> {
|
|
const stats = await this.inner.indexStats(name);
|
|
if (stats === null) {
|
|
return undefined;
|
|
}
|
|
return stats;
|
|
}
|
|
|
|
async stats(): Promise<TableStatistics> {
|
|
return await this.inner.stats();
|
|
}
|
|
|
|
async initialStorageOptions(): Promise<
|
|
Record<string, string> | null | undefined
|
|
> {
|
|
return await this.inner.initialStorageOptions();
|
|
}
|
|
|
|
async latestStorageOptions(): Promise<
|
|
Record<string, string> | null | undefined
|
|
> {
|
|
return await this.inner.latestStorageOptions();
|
|
}
|
|
|
|
mergeInsert(on: string | string[]): MergeInsertBuilder {
|
|
on = Array.isArray(on) ? on : [on];
|
|
return new MergeInsertBuilder(this.inner.mergeInsert(on), this.schema());
|
|
}
|
|
|
|
/**
|
|
* Check if the table uses the new manifest path scheme.
|
|
*
|
|
* This function will return true if the table uses the V2 manifest
|
|
* path scheme.
|
|
*/
|
|
async usesV2ManifestPaths(): Promise<boolean> {
|
|
return await this.inner.usesV2ManifestPaths();
|
|
}
|
|
|
|
/**
|
|
* Migrate the table to use the new manifest path scheme.
|
|
*
|
|
* This function will rename all V1 manifests to V2 manifest paths.
|
|
* These paths provide more efficient opening of datasets with many versions
|
|
* on object stores.
|
|
*
|
|
* This function is idempotent, and can be run multiple times without
|
|
* changing the state of the object store.
|
|
*
|
|
* However, it should not be run while other concurrent operations are happening.
|
|
* And it should also run until completion before resuming other operations.
|
|
*/
|
|
async migrateManifestPathsV2(): Promise<void> {
|
|
await this.inner.migrateManifestPathsV2();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* A definition of a column alteration. The alteration changes the column at
|
|
* `path` to have the new name `name`, to be nullable if `nullable` is true,
|
|
* and to have the data type `data_type`. At least one of `rename` or `nullable`
|
|
* must be provided.
|
|
*/
|
|
export interface ColumnAlteration {
|
|
/**
|
|
* The path to the column to alter. This is a dot-separated path to the column.
|
|
* If it is a top-level column then it is just the name of the column. If it is
|
|
* a nested column then it is the path to the column, e.g. "a.b.c" for a column
|
|
* `c` nested inside a column `b` nested inside a column `a`.
|
|
*/
|
|
path: string;
|
|
/**
|
|
* The new name of the column. If not provided then the name will not be changed.
|
|
* This must be distinct from the names of all other columns in the table.
|
|
*/
|
|
rename?: string;
|
|
/**
|
|
* A new data type for the column. If not provided then the data type will not be changed.
|
|
* Changing data types is limited to casting to the same general type. For example, these
|
|
* changes are valid:
|
|
* * `int32` -> `int64` (integers)
|
|
* * `double` -> `float` (floats)
|
|
* * `string` -> `large_string` (strings)
|
|
* But these changes are not:
|
|
* * `int32` -> `double` (mix integers and floats)
|
|
* * `string` -> `int32` (mix strings and integers)
|
|
*/
|
|
dataType?: string | DataType;
|
|
/** Set the new nullability. Note that a nullable column cannot be made non-nullable. */
|
|
nullable?: boolean;
|
|
}
|
|
|
|
/** A per-field metadata update, addressed by dot-path. */
|
|
export interface FieldMetadataUpdate {
|
|
/**
|
|
* Dot-separated path to the field. For a top-level column this is just its
|
|
* name; for a nested field it's the path, e.g. "a.b.c".
|
|
*/
|
|
path: string;
|
|
/**
|
|
* Metadata key/value pairs. Merged into the field's existing metadata by
|
|
* default; a value of `null` deletes that key.
|
|
*/
|
|
metadata: Record<string, string | null>;
|
|
/** If true, replace the field's entire metadata map instead of merging. */
|
|
replace?: boolean;
|
|
}
|
|
|
|
/** Summary of a column in a branch diff. */
|
|
export interface BranchColumnSummary {
|
|
name: string;
|
|
dataType: string;
|
|
nullable: boolean;
|
|
}
|
|
|
|
/** A column whose definition differs between main and the branch. */
|
|
export interface BranchColumnChange {
|
|
name: string;
|
|
main: BranchColumnSummary;
|
|
branch: BranchColumnSummary;
|
|
}
|
|
|
|
/** Summary of an index in a branch diff. */
|
|
export interface BranchIndexSummary {
|
|
indexName: string;
|
|
columns: string[];
|
|
indexType?: string;
|
|
status: string;
|
|
}
|
|
|
|
/** Row-level comparison between main and the branch. */
|
|
export interface BranchRowCountSummary {
|
|
unchanged: number;
|
|
newOnBase: number;
|
|
newOnBranch: number;
|
|
staleRecompute: number;
|
|
inputsChanged: number;
|
|
deltaAvailable: boolean;
|
|
}
|
|
|
|
/** A reason why a branch cannot currently be merged. */
|
|
export interface MergeBlocker {
|
|
code: string;
|
|
message: string;
|
|
}
|
|
|
|
/** Read-only comparison of a branch against main. */
|
|
export interface BranchDiff {
|
|
fromBranch: string;
|
|
parentVersion: number;
|
|
mainVersion: number;
|
|
branchVersion: number;
|
|
baseMoved: boolean;
|
|
rowCountMain: number;
|
|
rowCountBranch: number;
|
|
rowSummary: BranchRowCountSummary;
|
|
addedColumns: BranchColumnSummary[];
|
|
removedColumns: BranchColumnSummary[];
|
|
changedColumns: BranchColumnChange[];
|
|
addedIndexes: BranchIndexSummary[];
|
|
removedIndexes: BranchIndexSummary[];
|
|
mergeable: boolean;
|
|
mergeBlockers: MergeBlocker[];
|
|
}
|
|
|
|
/** Changes that would be, or were, promoted by a branch merge. */
|
|
export interface MergePreview {
|
|
promotedColumns: string[];
|
|
}
|
|
|
|
/** Result of previewing or attempting a branch merge. */
|
|
export interface MergeBranchResult {
|
|
status: "ready" | "rejected" | "notImplemented" | "merged" | "unknown";
|
|
diff: BranchDiff;
|
|
preview: MergePreview;
|
|
mainVersionAfter?: number;
|
|
}
|
|
|
|
/**
|
|
* Branch manager for a {@link Table}.
|
|
*
|
|
* Unlike tags, `create` and `checkout` return a new {@link Table} handle scoped
|
|
* to the branch; writes on it do not affect `main`.
|
|
*/
|
|
export class Branches {
|
|
#inner: NativeBranches;
|
|
|
|
/**
|
|
* Construct a Branches manager. Internal use only.
|
|
* @hidden
|
|
*/
|
|
constructor(inner: NativeBranches) {
|
|
this.#inner = inner;
|
|
}
|
|
|
|
/** List all branches, mapping name to branch metadata. */
|
|
async list(): Promise<Record<string, BranchContents>> {
|
|
return await this.#inner.list();
|
|
}
|
|
|
|
/**
|
|
* Create a branch and return a handle scoped to it.
|
|
*
|
|
* @param name Name of the new branch.
|
|
* @param fromRef Source branch to fork from. Defaults to `main`.
|
|
* @param fromVersion A specific version on `fromRef`. Defaults to latest.
|
|
*/
|
|
async create(
|
|
name: string,
|
|
fromRef?: string,
|
|
fromVersion?: number,
|
|
): Promise<Table> {
|
|
return new LocalTable(await this.#inner.create(name, fromRef, fromVersion));
|
|
}
|
|
|
|
/**
|
|
* Check out an existing branch and return a handle scoped to it.
|
|
*
|
|
* With `version` set, the returned handle is pinned to that version of the
|
|
* branch (a read-only, detached view); otherwise it tracks the branch's
|
|
* latest and stays writable.
|
|
*/
|
|
async checkout(name: string, version?: number): Promise<Table> {
|
|
return new LocalTable(await this.#inner.checkout(name, version));
|
|
}
|
|
|
|
/** Delete a branch. */
|
|
async delete(name: string): Promise<void> {
|
|
return await this.#inner.delete(name);
|
|
}
|
|
|
|
/** Compare a branch against main without modifying either branch. */
|
|
async diff(fromBranch: string): Promise<BranchDiff> {
|
|
return (await this.#inner.diff(fromBranch)) as unknown as BranchDiff;
|
|
}
|
|
|
|
/**
|
|
* Merge a branch into main.
|
|
*
|
|
* Set `dryRun` to `true` to preview the merge. A rejected merge resolves
|
|
* with `status: "rejected"` instead of throwing.
|
|
*
|
|
* @param fromBranch Branch to merge from.
|
|
* @param dryRun When true, only preview the merge. Defaults to false.
|
|
*/
|
|
async merge(
|
|
fromBranch: string,
|
|
dryRun: boolean = false,
|
|
): Promise<MergeBranchResult> {
|
|
return (await this.#inner.merge(
|
|
fromBranch,
|
|
dryRun,
|
|
)) as unknown as MergeBranchResult;
|
|
}
|
|
}
|