lancedb/nodejs/lancedb/table.ts

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors

import {
  Table as ArrowTable,
  Data,
  DataType,
  Field,
  IntoVector,
  MultiVector,
  Schema,
  dataTypeToJson,
  fromDataToBuffer,
  fromTableToBuffer,
  isMultiVector,
  makeEmptyTable,
  tableFromIPC,
} from "./arrow";

import { BlobFile, BlobReadOptions, runWithSignal } from "./blob";
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
import { IndexOptions } from "./indices";
import { Job } from "./job";
import { validateNonNegativeInteger } from "./materialized_view";
import { MergeInsertBuilder } from "./merge";
import {
  AddColumnsResult,
  AddColumnsSql,
  AddResult,
  AlterColumnsResult,
  BranchContents,
  DeleteResult,
  DropColumnsResult,
  FunctionErrors,
  FunctionErrorsOptions,
  IndexConfig,
  IndexStatistics,
  Branches as NativeBranches,
  OptimizeStats,
  RTEStats,
  RefreshColumnOptions,
  RefreshColumnResult,
  RefreshMaterializedViewResult,
  TableStatistics,
  Tags,
  UpdateFieldMetadataResult,
  UpdateResult,
  Table as _NativeTable,
} from "./native";
import {
  AutoQuery,
  FullTextQuery,
  Query,
  TakeQuery,
  VectorQuery,
  createAutoQuery,
  instanceOfFullTextQuery,
} from "./query";
import { sanitizeType } from "./sanitize";
import { IntoSql, toSQL } from "./util";
export { IndexConfig } from "./native";
export {
  TableShardStats,
  SsTableStats,
  RTEStats,
  MemtableStats,
} from "./native";

/**
 * Progress snapshot for a write operation, delivered to the `progress`
 * callback passed to {@link Table.add}.
 */
export interface WriteProgress {
  /** Number of rows written so far. */
  outputRows: number;
  /** Number of bytes written so far. */
  outputBytes: number;
  /**
   * Total rows expected, when the input source reports it.
   *
   * Always set on the final callback (the one with `done: true`), falling
   * back to the actual number of rows written when the source could not
   * report a row count up front.
   */
  totalRows?: number;
  /** Wall-clock seconds since the write started. */
  elapsedSeconds: number;
  /** Number of parallel write tasks currently in flight. */
  activeTasks: number;
  /** Total number of parallel write tasks (the write parallelism). */
  totalTasks: number;
  /** `true` for the final callback; `false` otherwise. */
  done: boolean;
}

/**
 * Options for adding data to a table.
 */
export interface AddDataOptions {
  /**
   * If "append" (the default) then the new data will be added to the table
   *
   * If "overwrite" then the new data will replace the existing data in the table.
   */
  mode: "append" | "overwrite";

  /**
   * Optional callback invoked periodically with write progress.
   *
   * The callback is fired once per batch written and once more with
   * `done: true` when the write completes. Calls are dispatched
   * asynchronously to the JS event loop and never block the write — a slow
   * callback will queue events rather than back-pressure the writer.
   *
   * Errors thrown from the callback are logged with `console.warn` and
   * swallowed — they do not abort the write.
   *
   * @example
   * ```ts
   * await table.add(data, {
   *   progress: (p) => {
   *     console.log(`${p.outputRows}/${p.totalRows ?? "?"} rows`);
   *   },
   * });
   * ```
   */
  progress: (progress: WriteProgress) => void;

  /**
   * Whether blob URIs outside registered external bases may be written.
   *
   * Defaults to `false`. This option is supported only in LanceDB Embedded;
   * LanceDB Enterprise returns an error when it is enabled. An enabled write
   * stores the absolute URI reference without registering a base or copying
   * the external object into the database. The object must remain accessible
   * when the blob is read later.
   */
  allowExternalBlobOutsideBases?: boolean;
}

export interface UpdateOptions {
  /**
   * A filter that limits the scope of the update.
   *
   * This should be an SQL filter expression.
   *
   * Only rows that satisfy the expression will be updated.
   *
   * For example, this could be 'my_col == 0' to replace all instances
   * of 0 in a column with some other default value.
   */
  where: string;
}

/** Per-invocation compaction settings. Values must be safe integers. */
export interface CompactionOptions {
  /** Positive target rows per output fragment; also affects candidate selection. */
  targetRowsPerFragment?: number;
  /** Positive live-source-row budget across whole tasks; an oversized first task can produce no work. */
  maxSourceRows?: number;
  /** Scanner I/O read-ahead budget in bytes; zero disables byte-based I/O backpressure. */
  ioBufferSize?: number;
  /** Positive scanner batch size in rows. */
  batchSize?: number;
}

export interface OptimizeOptions {
  /**
   * Compaction settings for this invocation. Omitted remote values inherit table
   * tuning, then deployment/built-in defaults.
   * Remote tables require a server supporting these SQL options; settings do
   * not change table policy. This is not a runtime or total-memory limit.
   * @example
   * await tbl.optimize({ compactionOptions: {
   *   targetRowsPerFragment: 1000, maxSourceRows: 5000,
   *   ioBufferSize: 64 * 1024 * 1024, batchSize: 128,
   * } });
   */
  compactionOptions?: CompactionOptions;
  /**
   * If set then all versions older than the given date
   * be removed.  The current version will never be removed.
   * The default is 7 days
   * @example
   * // Delete all versions older than 1 day
   * const olderThan = new Date();
   * olderThan.setDate(olderThan.getDate() - 1));
   * tbl.optimize({cleanupOlderThan: olderThan});
   *
   * // Delete versions committed before this point. Versions created by the
   * // optimize call itself are newer than the cutoff and will be retained.
   * tbl.optimize({cleanupOlderThan: new Date()});
   */
  cleanupOlderThan: Date;
  /**
   * Because they may be part of an in-progress transaction, files newer than
   * 7 days old are not deleted by default. If you are sure that there are no
   * in-progress transactions, then you can set this to true to delete all
   * files older than `cleanupOlderThan`.
   *
   * **WARNING**: This should only be set to true if you can guarantee that
   * no other process is currently working on this dataset. Otherwise the
   * dataset could be put into a corrupted state.
   */
  deleteUnverified: boolean;
}

export interface Version {
  version: number;
  timestamp: Date;
  metadata: Record<string, string>;
}

/** Token produced by the tokenizer configured on a full-text search index. */
export interface FtsToken {
  /** Token text after tokenizer filters have been applied. */
  text: string;
  /** Token position used by full-text query matching. */
  position: number;
}

export type TokenizeTableOptions =
  | {
      /** FTS-indexed column whose tokenizer should be used. */
      column: string;
      indexName?: never;
    }
  | {
      /** Name of the FTS index whose tokenizer should be used. */
      indexName: string;
      column?: never;
    };

/**
 * Specification enabling the realtime engine (RTE), which routes
 * `mergeInsert` through Lance's MemWAL LSM-style write path.
 *
 * `specType` is `"bucket"`, `"identity"`, or `"unsharded"`. For `"bucket"`,
 * `column` and `numBuckets` are required; for `"identity"`, `column` is
 * required and must be a deterministic function of the unenforced primary
 * key (every row with a given primary key must always produce the same
 * `column` value, or upserts of that key can land in different shards and a
 * stale version can win).
 */
export interface RTEWriteSpec {
  /** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
  specType: "bucket" | "identity" | "unsharded";
  /** Bucket and identity variants: the sharding column. */
  column?: string;
  /** Bucket variant: the number of buckets, in `[1, 1024]`. */
  numBuckets?: number;
  /**
   * Indexes the RTE keeps up to date. Omit for every index the table has,
   * including ones created later. Pass `[]` for none.
   */
  maintainedIndexes?: string[];
  /** Default `ShardWriter` configuration recorded in the MemWAL index. */
  writerConfigDefaults?: Record<string, string>;
}

/** Exact immutable Function revision attached to a table output. */
export interface FunctionColumnVersionStatus {
  name: string;
  namespacePath: string[];
  objectId: string;
  location: string;
  version: string;
  manifestDigest: string;
}

/** Latest matching refresh Job projected for a Function output. */
export interface FunctionColumnLastRefreshJob {
  jobId: string;
  state: string;
  automatic?: boolean;
  completedAtMillis?: bigint;
}

/** Automatic-refresh projection for a Function output. */
export interface FunctionColumnAutoRefreshStatus {
  enabled: boolean;
  intervalSeconds?: bigint;
  activeJobId?: string;
  state:
    | "disabled"
    | "running"
    | "ready"
    | "up_to_date"
    | "waiting_interval"
    | "waiting_backoff"
    | "blocked"
    | "unobservable";
  failureStreak: number;
  effectiveCooldownMillis?: bigint;
  blockedReason?:
    | "table_has_mem_wal"
    | "inputs_not_settled"
    | "dependency_in_flight"
    | "table_capacity"
    | "cancelled"
    | "preparation_backoff";
  nextRunAtMillis: bigint | null;
  nextRunAtReason?: string;
}

/** Exact four-way status of one Function output at one table version. */
export interface FunctionColumnStatus {
  column: string;
  bindingId: string;
  function: FunctionColumnVersionStatus;
  tableVersion: bigint;
  rows: bigint;
  fresh: bigint;
  stale: bigint;
  neverComputed: bigint;
  failed: bigint;
  lastRefreshJob?: FunctionColumnLastRefreshJob;
  autoRefresh: FunctionColumnAutoRefreshStatus;
}

/**
 * A Table is a collection of Records in a LanceDB Database.
 *
 * A Table object is expected to be long lived and reused for multiple operations.
 * Table objects will cache a certain amount of index data in memory.  This cache
 * will be freed when the Table is garbage collected.  To eagerly free the cache you
 * can call the `close` method.  Once the Table is closed, it cannot be used for any
 * further operations.
 *
 * Tables are created using the methods {@link Connection#createTable}
 * and {@link Connection#createEmptyTable}. Existing tables are opened
 * using {@link Connection#openTable}.
 *
 * Closing a table is optional.  It not closed, it will be closed when it is garbage
 * collected.
 *
 * @hideconstructor
 */
export abstract class Table {
  [Symbol.for("nodejs.util.inspect.custom")](): string {
    return this.display();
  }
  /** Returns the name of the table */
  abstract get name(): string;

  /** Return true if the table has not been closed */
  abstract isOpen(): boolean;
  /**
   * Close the table, releasing any underlying resources.
   *
   * It is safe to call this method multiple times.
   *
   * Any attempt to use the table after it is closed will result in an error.
   */
  abstract close(): void;
  /** Return a brief description of the table */
  abstract display(): string;
  /** Get the schema of the table. */
  abstract schema(): Promise<Schema>;
  /**
   * Insert records into this Table.
   * @param {Data} data Records to be inserted into the Table
   * @returns {Promise<AddResult>} A promise that resolves to an object
   * containing the new version number of the table
   */
  abstract add(
    data: Data,
    options?: Partial<AddDataOptions>,
  ): Promise<AddResult>;
  /**
   * Update existing records in the Table
   * @param opts.values The values to update. The keys are the column names and the values
   * are the values to set.
   * @returns {Promise<UpdateResult>} A promise that resolves to an object containing
   * the number of rows updated and the new version number
   * @example
   * ```ts
   * table.update({where:"x = 2", values:{"vector": [10, 10]}})
   * ```
   */
  abstract update(
    opts: {
      values: Map<string, IntoSql> | Record<string, IntoSql>;
    } & Partial<UpdateOptions>,
  ): Promise<UpdateResult>;
  /**
   * Update existing records in the Table
   * @param opts.valuesSql The values to update. The keys are the column names and the values
   * are the values to set. The values are SQL expressions.
   * @returns {Promise<UpdateResult>} A promise that resolves to an object containing
   * the number of rows updated and the new version number
   * @example
   * ```ts
   * table.update({where:"x = 2", valuesSql:{"x": "x + 1"}})
   * ```
   */
  abstract update(
    opts: {
      valuesSql: Map<string, string> | Record<string, string>;
    } & Partial<UpdateOptions>,
  ): Promise<UpdateResult>;
  /**
   * Update existing records in the Table
   *
   * An update operation can be used to adjust existing values.  Use the
   * returned builder to specify which columns to update.  The new value
   * can be a literal value (e.g. replacing nulls with some default value)
   * or an expression applied to the old value (e.g. incrementing a value)
   *
   * An optional condition can be specified (e.g. "only update if the old
   * value is 0")
   *
   * Note: if your condition is something like "some_id_column == 7" and
   * you are updating many rows (with different ids) then you will get
   * better performance with a single [`merge_insert`] call instead of
   * repeatedly calling this method.
   * @param {Map<string, string> | Record<string, string>} updates - the
   * columns to update
   * @returns {Promise<UpdateResult>} A promise that resolves to an object
   * containing the number of rows updated and the new version number
   *
   * Keys in the map should specify the name of the column to update.
   * Values in the map provide the new value of the column.  These can
   * be SQL literal strings (e.g. "7" or "'foo'") or they can be expressions
   * based on the row being updated (e.g. "my_col + 1")
   * @param {Partial<UpdateOptions>} options - additional options to control
   * the update behavior
   */
  abstract update(
    updates: Map<string, string> | Record<string, string>,
    options?: Partial<UpdateOptions>,
  ): Promise<UpdateResult>;

  /** Count the total number of rows in the dataset. */
  abstract countRows(filter?: string): Promise<number>;
  /**
   * Delete the rows that satisfy the predicate.
   * @returns {Promise<DeleteResult>} A promise that resolves to an object
   * containing the new version number of the table
   */
  abstract delete(predicate: string): Promise<DeleteResult>;
  /**
   * Create an index to speed up queries.
   *
   * Indices can be created on vector columns or scalar columns.
   * Indices on vector columns will speed up vector searches.
   * Indices on scalar columns will speed up filtering (in both
   * vector and non-vector searches)
   *
   * We currently don't support custom named indexes.
   * The index name will always be `${column}_idx`.
   *
   * On LanceDB Embedded tables the index is built before the returned promise
   * resolves. On LanceDB Enterprise tables it resolves once the build is
   * scheduled. Until the build finishes, a vector index may be missing from
   * {@link Table.listIndices}, a scalar or FTS index may be listed with 0
   * indexed rows, and queries won't use the index. To wait until the index is
   * ready, set {@link IndexOptions.waitTimeoutSeconds}, call
   * {@link Table.waitForIndex}, or use {@link Table.createIndexAsync} and wait
   * on its job when the server returned a job ID.
   *
   * `waitTimeoutSeconds` waits for the submitted build when the server returns
   * a job ID. On older servers that don't return one, it falls back to polling
   * the index by name. {@link Table.waitForIndex} always waits by name. When
   * replacing an existing index, those name-based waits may return before the
   * new build finishes because the old index can still match.
   * `createIndexAsync` can track a specific build only when the server returned
   * a job ID; otherwise it returns an already-complete job and callers must
   * use {@link Table.waitForIndex} for that build.
   *
   * @example
   * // If the column has a vector (fixed size list) data type then
   * // an IvfPq vector index will be created.
   * const table = await conn.openTable("my_table");
   * await table.createIndex("vector");
   * @example
   * // For advanced control over vector index creation you can specify
   * // the index type and options.
   * const table = await conn.openTable("my_table");
   * await table.createIndex("vector", {
   *   config: lancedb.Index.ivfPq({
   *     numPartitions: 128,
   *     numSubVectors: 16,
   *   }),
   * });
   * @example
   * // Or size the partitions by row count instead of choosing how many.
   * await table.createIndex("vector", {
   *   config: lancedb.Index.ivfPq({ targetPartitionSize: 4096 }),
   * });
   * @example
   * // Or create a Scalar index
   * await table.createIndex("my_float_col");
   */
  abstract createIndex(
    column: string,
    options?: Partial<IndexOptions>,
  ): Promise<void>;

  /**
   * Create an index, returning a handle to the indexing job.
   *
   * The job may already be complete when returned; callers must not assume
   * the index exists until {@link Job.wait} resolves.
   *
   * In LanceDB Enterprise, the returned job tracks this submission only if the
   * server returns a job ID. If it does not, the job is already complete and
   * does not wait for the background build; use {@link Table.waitForIndex} and
   * account for the replacement caveat described in {@link Table.createIndex}.
   */
  abstract createIndexAsync(
    column: string,
    options?: Partial<IndexOptions>,
  ): Promise<Job>;

  /**
   * Drop an index from the table.
   *
   * @param name The name of the index.
   *
   * This does not delete the index from disk, it just removes it from the table.
   * To delete the index, run {@link Table#optimize} after dropping the index.
   *
   * Use {@link Table.listIndices} to find the names of the indices.
   */
  abstract dropIndex(name: string): Promise<void>;

  /**
   * Prewarm an index in the table.
   *
   * @param name The name of the index.
   *
   * This will load the index into memory.  This may reduce the cold-start time for
   * future queries.  If the index does not fit in the cache then this call may be
   * wasteful.
   */
  abstract prewarmIndex(name: string): Promise<void>;

  /**
   * Prewarm one or more columns of data in the table.
   *
   * @param columns The columns to prewarm. If undefined, all columns are prewarmed.
   *
   * This will load the column data into the page cache so that future queries that
   * read those columns avoid the initial cold-start latency.  This call initiates
   * prewarming and returns once the request is accepted; the warming itself may
   * continue in the background.  Calling it on already-prewarmed columns is a
   * no-op on the server.
   *
   * Prewarming is generally useful for columns used in filters or projections.
   * Large columns (e.g. high-dimensional vectors or binary data) may not be
   * practical to prewarm.
   *
   * This feature is currently only supported in LanceDB Enterprise.
   */
  abstract prewarmData(columns?: string[]): Promise<void>;

  /**
   * Waits for asynchronous indexing to complete on the table.
   *
   * @param indexNames The name of the indices to wait for
   * @param timeoutSeconds The number of seconds to wait before timing out
   *
   * This will raise an error if the indices are not created and fully indexed within the timeout.
   */
  abstract waitForIndex(
    indexNames: string[],
    timeoutSeconds: number,
  ): Promise<void>;

  /**
   * Create a {@link Query} Builder.
   *
   * Queries allow you to search your existing data.  By default the query will
   * return all the data in the table in no particular order.  The builder
   * returned by this method can be used to control the query using filtering,
   * vector similarity, sorting, and more.
   *
   * Note: By default, all columns are returned.  For best performance, you should
   * only fetch the columns you need.
   *
   * When appropriate, various indices and statistics based pruning will be used to
   * accelerate the query.
   * @example
   * // SQL-style filtering
   * //
   * // This query will return up to 1000 rows whose value in the `id` column
   * // is greater than 5. LanceDb supports a broad set of filtering functions.
   * for await (const batch of table
   *   .query()
   *   .where("id > 1")
   *   .select(["id"])
   *   .limit(20)) {
   *   console.log(batch);
   * }
   * @example
   * // Vector Similarity Search
   * //
   * // This example will find the 10 rows whose value in the "vector" column are
   * // closest to the query vector [1.0, 2.0, 3.0].  If an index has been created
   * // on the "vector" column then this will perform an ANN search.
   * //
   * // The `refineFactor` and `nprobes` methods are used to control the recall /
   * // latency tradeoff of the search.
   * for await (const batch of table
   *   .query()
   *   .where("id > 1")
   *   .select(["id"])
   *   .limit(20)) {
   *   console.log(batch);
   * }
   * @example
   * // Scan the full dataset
   * //
   * // This query will return everything in the table in no particular order.
   * for await (const batch of table.query()) {
   *   console.log(batch);
   * }
   * @returns {Query} A builder that can be used to parameterize the query
   */
  abstract query(): Query;

  /**
   * Create a query that returns a subset of the rows in the table.
   * @param offsets The offsets of the rows to return.
   * @returns A builder that can be used to parameterize the query.
   */
  abstract takeOffsets(offsets: number[]): TakeQuery;

  /**
   * Create a query that returns a subset of the rows in the table.
   * @param rowIds The row ids of the rows to return.
   *
   * Row ids returned by `withRowId()` are `bigint`, so `bigint[]` is supported.
   * For convenience / backwards compatibility, `number[]` is also accepted (for
   * small row ids that fit in a safe integer).
   * @returns A builder that can be used to parameterize the query.
   */
  abstract takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery;

  /**
   * Blob v2 columns addressable by row ID, including struct-nested dotted paths.
   * Blobs inside lists are omitted because a row ID does not identify an element.
   */
  abstract blobColumns(): Promise<string[]>;

  /**
   * Bytes for `column` at row IDs from {@link Query.withRowId}.
   *
   * Reads the table's current checkout. IDs from another version can fail after
   * compaction unless stable row ids are enabled. Results keep input order and
   * duplicates. Null blobs are `null`. Empty blobs are empty buffers.
   * One missing or deleted row ID rejects the entire batch, including on remote
   * tables; no partial results are returned.
   * Blobs inside lists cannot be fetched by row ID.
   *
   * Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes.
   * The client splits requests automatically and reads an individual larger
   * blob through the Range route. This method still materializes all bytes in
   * memory; use {@link Table.fetchBlobFiles} for large values.
   */
  abstract fetchBlobs(
    column: string,
    rowIds: readonly (bigint | number)[],
    options?: BlobReadOptions,
  ): Promise<(Buffer | null)[]>;

  /**
   * Opens lazy blob handles for `column` at the given row IDs using the
   * table's current checkout.
   *
   * Preserves input order, duplicates, and nulls. Use this for large payloads.
   * See {@link Table.fetchBlobs} for row-ID validity across versions.
   * One missing or deleted row ID rejects the entire batch, including on remote
   * tables; no partial handles are returned.
   */
  abstract fetchBlobFiles(
    column: string,
    rowIds: readonly (bigint | number)[],
    options?: BlobReadOptions,
  ): Promise<(BlobFile | null)[]>;

  /**
   * Create a search query to find the nearest neighbors
   * of the given query
   * @param {string | IntoVector} query - the query, a vector or string
   * @param {string} queryType - the type of the query, "vector", "fts", or "auto"
   * @param {string | string[]} ftsColumns - the columns to search in for full text search
   *    for now, only one column can be searched at a time.
   *
   * when "auto" is used, if the query is a string and an embedding function is defined, it will be treated as a vector query
   * if the query is a string and no embedding function is defined, it will be treated as a full text search query
   */
  abstract search(
    query: string | IntoVector | MultiVector | FullTextQuery,
    queryType?: string,
    ftsColumns?: string | string[],
  ): VectorQuery | Query | AutoQuery;
  /**
   * Search the table with a given query vector.
   *
   * This is a convenience method for preparing a vector query and
   * is the same thing as calling `nearestTo` on the builder returned
   * by `query`.  @see {@link Query#nearestTo} for more details.
   */
  abstract vectorSearch(vector: IntoVector | MultiVector): VectorQuery;
  /**
   * Add new columns with defined values.
   *
   * LanceDB Embedded computed-column declaration and refresh require a source
   * build with the `experimental-functions` Cargo feature. Published packages
   * disable it; LanceDB Enterprise operations remain available.
   *
   * The `{ computed }` form stores the expression rather than evaluating it
   * now: the column is committed with no values, and rows get them from
   * {@link Table#refreshColumn}. Declaring one therefore costs the same on a
   * large table as on an empty one.
   *
   * A refresh also recomputes the rows whose inputs changed since they were
   * computed, so a mutated input is reflected by the next refresh. While a
   * declaration reads a column, that column cannot be renamed, retyped or
   * dropped.
   *
   * In LanceDB Enterprise the expression is planned by the
   * server, and the refresh runs as a server job -- see
   * {@link Table#refreshColumnAsync}.
   * @param {AddColumnsSql[] | Field | Field[] | Schema} newColumnTransforms Either:
   *   - An array of objects with column names and SQL expressions to calculate values
   *   - A single Arrow Field defining one column with its data type (column will be initialized with null values)
   *   - An array of Arrow Fields defining columns with their data types (columns will be initialized with null values)
   *   - An Arrow Schema defining columns with their data types (columns will be initialized with null values)
   *   - `{ computed }`, declaring columns defined by a SQL expression whose type and inputs are derived from it
   * @returns {Promise<AddColumnsResult>} A promise that resolves to an object
   * containing the new version number of the table after adding the columns.
   * @example
   * ```ts
   * await table.addColumns({ computed: [{ name: "doubled", valueSql: "x * 2" }] });
   * const { rowsFilled } = await table.refreshColumn("doubled");
   * ```
   */
  abstract addColumns(
    newColumnTransforms:
      | AddColumnsSql[]
      | Field
      | Field[]
      | Schema
      | { computed: AddColumnsSql[] },
  ): Promise<AddColumnsResult>;

  /**
   * Fill the rows of a computed column that hold no value yet.
   * Local refresh requires a source build with `experimental-functions`.
   *
   * Rows appended since the last refresh are filled by the next one, and
   * rows whose inputs changed since they were computed are recomputed;
   * everything else is left as it is. LanceDB Embedded only: in LanceDB
   * Enterprise a refresh runs as a server job, through {@link Table#refreshColumnAsync}.
   * @param {string} column The name of the computed column to fill.
   * @param {RefreshColumnOptions} options Refresh options, all LanceDB
   * Enterprise only; see {@link Table#refreshColumnAsync}.
   * @returns {Promise<RefreshColumnResult>} A promise that resolves to the
   * number of rows filled and the new version number of the table.
   */
  abstract refreshColumn(
    column: string,
    options?: RefreshColumnOptions,
  ): Promise<RefreshColumnResult>;

  /**
   * Like {@link Table#refreshColumn}, but returns a handle to the refresh
   * job instead of blocking until it completes.
   * LanceDB Embedded refresh requires a source build with
   * `experimental-functions`. LanceDB Enterprise jobs do not require this
   * feature.
   *
   * The job may already be complete when returned; callers must not assume
   * the column is filled until {@link Job.wait} resolves. Invalid input --
   * an unknown column, or one that is not computed -- rejects here rather
   * than failing the job. In LanceDB Embedded the job runs in-process; in
   * LanceDB Enterprise it is the server's backfill job.
   * @param {string} column The name of the computed column to fill.
   * @param {RefreshColumnOptions} options Refresh options, mirroring the SQL
   * `REFRESH COLUMN ... WHERE ... WITH (...)` form: `where` narrows the rows,
   * `errorPolicy`, `maxFailedFraction`, `maxFailedRows`, `rowTimeoutSeconds`
   * and `initTimeoutSeconds` govern failing rows. `batchSize`, `gpuClass`,
   * `maxConcurrency`, `resourceBudget` and
   * `maxExecutorProcessesPerWorker` size registered Function refreshes.
   * `maxConcurrency` overrides the registered default for this logical
   * execution only and is a positive integer; it is not an account-wide
   * quota. A `resourceBudget` requires `cpuMillis` and
   * `memoryBytes`; its `gpuCount` defaults to zero for a CPU-only Function.
   * The budget applies only to this explicit refresh and does not configure
   * a later automatic refresh.
   * `gpuCount` at the top level is the shorthand that preserves the
   * deployment defaults for CPU and memory, and cannot be combined with
   * `resourceBudget`. `numWorkers` applies only to expression columns; use
   * `resourceBudget` for registered Functions. All are LanceDB Enterprise
   * only; a LanceDB Embedded refresh rejects every option other than
   * `batchSize`, `gpuClass` and `gpuCount`.
   * @example
   * ```ts
   * const job = await table.refreshColumnAsync("embedding", {
   *   where: "published_at > '2026-01-01'",
   *   resourceBudget: {
   *     cpuMillis: 8000,
   *     memoryBytes: 16 * 1024 ** 3,
   *   },
   *   maxConcurrency: 4,
   *   errorPolicy: "skip",
   *   maxFailedFraction: 0.01,
   *   rowTimeoutSeconds: 30,
   * });
   * await job.wait();
   * console.log(await job.status()); // "finished"
   * ```
   */
  abstract refreshColumnAsync(
    column: string,
    options?: RefreshColumnOptions,
  ): Promise<Job>;

  /**
   * The per-row errors Function refreshes recorded on this table.
   *
   * A refresh running under a skip policy records each row it skipped with
   * the input that failed and the error. This lists those records, newest
   * job first, plus a summary for any fragment whose per-row detail was
   * capped. LanceDB Enterprise only; reading errors needs read
   * access to the table, since a message carries the value that failed.
   * @param {FunctionErrorsOptions} options Optional filters: `jobId`,
   * `column`, and `limit` (server default 10000, cap 100000).
   * @returns {Promise<FunctionErrors>} The records, the capped fragments,
   * and whether the listing stopped at its limit.
   * @example
   * ```ts
   * const { records, truncated } = await table.functionErrors({ column: "embedding" });
   * ```
   */
  abstract functionErrors(
    options?: FunctionErrorsOptions,
  ): Promise<FunctionErrors>;

  /**
   * Return exact freshness status for Function-backed output columns.
   *
   * All results describe one server table version. Omit `column` for every
   * Function output. Rows already failed remain visible and do not prevent
   * `waitFresh` from completing. A handle pinned to an exact version requests
   * that version; a latest handle retains normal read-freshness behavior.
   */
  abstract functionColumnStatuses(
    column?: string,
  ): Promise<FunctionColumnStatus[]>;

  /** Return exact freshness status for one Function-backed output column. */
  abstract functionColumnStatus(column: string): Promise<FunctionColumnStatus>;

  /**
   * Poll until no stale or never-computed rows remain.
   *
   * Only HTTP 503 with the exact finalization-pending error code is retried;
   * every other error propagates. Failed rows do not block completion.
   */
  abstract waitFresh(
    column: string,
    timeoutSeconds?: number,
  ): Promise<FunctionColumnStatus>;

  /**
   * Recompute this table's contents from its materialized-view definition.
   *
   * Plumbing for {@link MaterializedView.refresh}, which is the way to call
   * it: rejects tables that carry no view definition.
   * @ignore
   */
  abstract refreshMaterializedView(
    sourceVersion?: number,
    full?: boolean,
  ): Promise<RefreshMaterializedViewResult>;

  /** @ignore */
  abstract materializedViewDefinition(): Promise<string>;

  /**
   * Alter the name or nullability of columns.
   *
   * Local tables reject renaming source or vector columns referenced by
   * `embedding_functions` schema metadata, preserving the embedding binding.
   * @param {ColumnAlteration[]} columnAlterations One or more alterations to
   * apply to columns.
   * @returns {Promise<AlterColumnsResult>} A promise that resolves to an object
   * containing the new version number of the table after altering the columns.
   */
  abstract alterColumns(
    columnAlterations: ColumnAlteration[],
  ): Promise<AlterColumnsResult>;

  /**
   * Update per-field (column) metadata.
   *
   * The following keys are treated specially, by convention, and should be
   * used when appropriate:
   *
   * - `lancedb:description`: for a human-readable description of a field.
   * - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
   *   names the tag category; e.g. `lancedb:tag:model: "clip"`.
   * - `lancedb:status`: for status options (`production`, `candidate`,
   *   `deprecated`, `archived`) to designate the current life cycle state of
   *   this column.
   * @param {FieldMetadataUpdate[]} updates One or more per-field updates. Each
   * update's metadata is merged into the field's existing metadata by default;
   * a value of `null` deletes that key, and `replace: true` swaps the whole map.
   * @returns {Promise<UpdateFieldMetadataResult>} resolves to the new table version.
   */
  abstract updateFieldMetadata(
    updates: FieldMetadataUpdate[],
  ): Promise<UpdateFieldMetadataResult>;

  /**
   * Drop one or more columns from the dataset
   *
   * This is a metadata-only operation and does not remove the data from the
   * underlying storage. In order to remove the data, you must subsequently
   * call ``compact_files`` to rewrite the data without the removed columns and
   * then call ``cleanup_files`` to remove the old files.
   *
   * Local tables reject dropping source or vector columns referenced by
   * `embedding_functions` schema metadata, preserving the embedding binding.
   * @param {string[]} columnNames The names of the columns to drop. These can
   * be nested column references (e.g. "a.b.c") or top-level column names
   * (e.g. "a").
   * @returns {Promise<DropColumnsResult>} A promise that resolves to an object
   * containing the new version number of the table after dropping the columns.
   */
  abstract dropColumns(columnNames: string[]): Promise<DropColumnsResult>;
  /**
   * Set the unenforced primary key for this table to a single column.
   *
   * "Unenforced" means LanceDB does not check uniqueness on writes; the
   * column is recorded in the schema as the primary key for use by features
   * such as `merge_insert`. Only single-column primary keys are supported,
   * and the key cannot be changed once set.
   * @param {string | string[]} columns The primary key column. A one-element
   * array is also accepted; passing more than one column is rejected.
   * @returns {Promise<void>}
   */
  abstract setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
  /**
   * Install an {@link RTEWriteSpec} on this table, selecting Lance's MemWAL
   * LSM-style write path for future `mergeInsert` calls.
   *
   * `RTEWriteSpec` chooses one of three sharding strategies via `specType`:
   *
   * - `"bucket"` — hash-bucket writes by the single-column unenforced primary
   *   key (`column` and `numBuckets` required).
   * - `"identity"` — shard by the raw value of a scalar `column`.
   * - `"unsharded"` — route every write to a single shard.
   *
   * All variants require the table to have an unenforced primary key
   * ({@link Table#setUnenforcedPrimaryKey}); bucket sharding additionally
   * requires it to be the single column being bucketed.
   *
   * Omitting `maintainedIndexes` maintains every index the table has,
   * including ones created later, and skips a kind the RTE cannot maintain.
   * Naming them pins an exact set, and a still-building index is rejected
   * rather than quietly omitted.
   * @param {RTEWriteSpec} spec The sharding spec to install.
   * @returns {Promise<void>}
   * @example
   * ```ts
   * await table.setUnenforcedPrimaryKey("id");
   * await table.enableRTE({
   *   specType: "bucket",
   *   column: "id",
   *   numBuckets: 16,
   *   maintainedIndexes: ["id_idx"],
   * });
   * ```
   */
  abstract enableRTE(spec: RTEWriteSpec): Promise<void>;
  /**
   * Remove the {@link RTEWriteSpec} from this table, reverting to the standard
   * `mergeInsert` write path.
   *
   * Errors if no spec is currently set.
   * @returns {Promise<void>}
   */
  abstract disableRTE(): Promise<void>;
  /**
   * Read the {@link RTEWriteSpec} currently installed on this table.
   *
   * Resolves to `undefined` when the RTE write path is not enabled (no
   * spec has been set, or it was removed with {@link Table#disableRTE}).
   * The spec is the one installed, including its maintained-index selection:
   * an absent `maintainedIndexes` for every index the table has, an empty
   * array for none.
   * @returns {Promise<RTEWriteSpec | undefined>}
   */
  abstract getRTESpec(): Promise<RTEWriteSpec | undefined>;
  /**
   * Drain and close any cached RTE shard writers held for this table.
   *
   * When an {@link RTEWriteSpec} is installed, `mergeInsert` opens RTE
   * shard writers and caches them for reuse across calls. This closes them,
   * flushing pending data; writers reopen lazily on the next `mergeInsert`.
   * It is a no-op when no writers are cached.
   * @returns {Promise<void>}
   */
  abstract closeRTEWriters(): Promise<void>;
  /**
   * Freeze every table shard's active memtable into a new SSTable.
   *
   * Returns once the freeze is committed. Freezing an empty memtable is a no-op,
   * so this is safe to call repeatedly.
   * @returns {Promise<void>}
   */
  abstract flushRTE(): Promise<void>;
  /**
   * Trigger a background SSTable compaction pass per table shard.
   *
   * Returns once the passes are *dispatched*, not once they finish — watch
   * {@link Table#getRTEStats} for progress, or use
   * {@link Table#checkpointRTE} to wait for convergence.
   * @returns {Promise<void>}
   */
  abstract compactRTE(): Promise<void>;
  /**
   * Converge this table's RTE write path into its base table.
   *
   * Freezes once, then triggers compaction and polls until the SSTables that existed
   * at the start is gone. The target set is fixed at the start, so
   * SSTables created *during* the checkpoint are ignored — that is what
   * lets it terminate under write load, and what makes it best-effort: it
   * converges the fresh tier as of some instant. Idempotent, abandonable at
   * any point, and safe to run on a cadence.
   *
   * There is no liveness bound — the compactor pool is shared across tables,
   * so a checkpoint queued behind unrelated work looks exactly like one that
   * is merging. The caller owns the deadline.
   * @returns {Promise<void>}
   * @example
   * ```ts
   * const before = await table.getRTEStats();
   * await table.checkpointRTE();
   * const after = await table.getRTEStats();
   * ```
   */
  abstract checkpointRTE(): Promise<void>;
  /**
   * Read live per-table-shard RTE state.
   *
   * Answers "how far behind is my fresh tier", "which table shard is hot", and
   * "why is my fresh-tier vector search brute-force". Mutates no table state.
   *
   * Resolves to `undefined` only when the RTE write path is not enabled.
   * @param {boolean} includeSstableRows Also count rows per SSTable.
   *   Off by default because each count opens an uncached Lance dataset.
   * @returns {Promise<RTEStats | undefined>}
   */
  abstract getRTEStats(
    includeSstableRows?: boolean,
  ): Promise<RTEStats | undefined>;
  /** Retrieve the version of the table */

  abstract version(): Promise<number>;
  /**
   * Checks out a specific version of the table _This is an in-place operation._
   *
   * This allows viewing previous versions of the table. If you wish to
   * keep writing to the dataset starting from an old version, then use
   * the `restore` function.
   *
   * Calling this method will set the table into time-travel mode. If you
   * wish to return to standard mode, call `checkoutLatest`.
   * @param {number | string} version The version to checkout, could be version number or tag
   * @example
   * ```typescript
   * import * as lancedb from "@lancedb/lancedb"
   * const db = await lancedb.connect("./.lancedb");
   * const table = await db.createTable("my_table", [
   *   { vector: [1.1, 0.9], type: "vector" },
   * ]);
   *
   * console.log(await table.version()); // 1
   * console.log(table.display());
   * await table.add([{ vector: [0.5, 0.2], type: "vector" }]);
   * await table.checkout(1);
   * console.log(await table.version()); // 2
   * ```
   */
  abstract checkout(version: number | string): Promise<void>;

  /**
   * Checkout the latest version of the table. _This is an in-place operation._
   *
   * The table will be set back into standard mode, and will track the latest
   * version of the table.
   */
  abstract checkoutLatest(): Promise<void>;

  /**
   * List all the versions of the table
   */
  abstract listVersions(): Promise<Version[]>;

  /**
   * Get a tags manager for this table.
   *
   * Tags allow you to label specific versions of a table with a human-readable name.
   * The returned tags manager can be used to list, create, update, or delete tags.
   *
   * @returns {Tags} A tags manager for this table
   * @example
   * ```typescript
   * const tagsManager = await table.tags();
   * await tagsManager.create("v1", 1);
   * const tags = await tagsManager.list();
   * console.log(tags); // { "v1": { version: 1, manifestSize: ... } }
   * ```
   */
  abstract tags(): Promise<Tags>;

  /**
   * Get the branch manager for this table.
   *
   * Branches are isolated, writable lines of history forked from another
   * branch (or version). Writes on a branch do not affect `main`.
   */
  abstract branches(): Promise<Branches>;

  /**
   * The branch this table handle is scoped to, or `null` for the main branch.
   *
   * A handle returned by {@link Branches.create} or {@link Branches.checkout}
   * reports the branch it targets; a handle opened normally reports `null`.
   */
  abstract currentBranch(): string | null;

  /**
   * Restore the table to the currently checked out version
   *
   * This operation will fail if checkout has not been called previously
   *
   * This operation will overwrite the latest version of the table with a
   * previous version.  Any changes made since the checked out version will
   * no longer be visible.
   *
   * Once the operation concludes the table will no longer be in a checked
   * out state and the read_consistency_interval, if any, will apply.
   */
  abstract restore(): Promise<void>;
  /**
   * Optimize the on-disk data and indices for better performance.
   *
   * Modeled after ``VACUUM`` in PostgreSQL.
   *
   *  Optimization covers three operations:
   *
   *  - Compaction: Merges small files into larger ones
   *  - Prune: Removes old versions of the dataset
   *  - Index: Optimizes the indices, adding new data to existing indices
   *
   *
   *  The frequency an application should call optimize is based on the frequency of
   *  data modifications.  If data is frequently added, deleted, or updated then
   *  optimize should be run frequently.  A good rule of thumb is to run optimize if
   *  you have added or modified 100,000 or more records or run more than 20 data
   *  modification operations.
   *
   * On a LanceDB Enterprise (remote) table, only compaction is currently
   * supported. Pruning old versions and index optimization are silently
   * skipped there instead of being attempted, so this call resolves
   * successfully, with zeroed pruning statistics, once the compaction job
   * has been submitted; it does not wait for that job to finish.
   * `compactionOptions` can tune the four per-job settings documented in
   * {@link CompactionOptions}. Older-server errors are propagated without
   * retrying with the settings removed.
   */
  abstract optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
  /**
   * List all indices that have been created with {@link Table.createIndex}
   *
   * On LanceDB Enterprise tables, an index whose build hasn't finished may be
   * missing (vector indices) or listed with 0 indexed rows (scalar and FTS
   * indices), and `numUnindexedRows` may be undefined. Use
   * {@link Table.waitForIndex} to wait until an index is ready.
   */
  abstract listIndices(): Promise<IndexConfig[]>;
  /**
   * Tokenize a full-text search query using the tokenizer configured on an FTS index.
   *
   * Specify exactly one of `column` or `indexName`.
   *
   * Model-backed tokenizers such as `jieba/*` and `lindera/*` are rebuilt in
   * the client process from index metadata. In LanceDB Enterprise, this means the
   * same tokenizer model files must also exist locally (under
   * `LANCE_LANGUAGE_MODEL_HOME`) and must match the files the server uses. A
   * mismatch cannot be detected: the returned tokens may differ from what the
   * server's index produces. If the model files are missing, an error naming
   * the tokenizer is raised.
   */
  abstract tokenize(
    query: string,
    options: TokenizeTableOptions,
  ): Promise<FtsToken[]>;
  /** Return the table as an arrow table */
  abstract toArrow(): Promise<ArrowTable>;

  /**
   * Create a {@link MergeInsertBuilder}, which combines new data with the
   * existing table in a single transaction — inserting, updating and deleting
   * rows depending on how they match.
   *
   * @param on - The column, or columns, to match source rows against target
   * rows on. Typically a key or id column. Several columns match on the
   * composite key: a source row updates a target row only when it agrees on
   * every one of them.
   */
  abstract mergeInsert(on: string | string[]): MergeInsertBuilder;

  /** List all the stats of a specified index
   *
   * @param {string} name The name of the index.
   * @returns {IndexStatistics | undefined} The stats of the index. If the index does not exist, it will return undefined
   *
   * Use {@link Table.listIndices} to find the names of the indices.
   */
  abstract indexStats(name: string): Promise<IndexStatistics | undefined>;

  /** Returns table and fragment statistics
   *
   * @returns {TableStatistics} The table and fragment statistics
   *
   */
  abstract stats(): Promise<TableStatistics>;

  /**
   * Get the initial storage options that were passed in when opening this table.
   *
   * For dynamically refreshed options (e.g., credential vending), use
   * {@link Table.latestStorageOptions}.
   *
   * Warning: This is an internal API and the return value is subject to change.
   *
   * @returns The storage options, or undefined if no storage options were configured.
   */
  abstract initialStorageOptions(): Promise<
    Record<string, string> | null | undefined
  >;

  /**
   * Get the latest storage options, refreshing from provider if configured.
   *
   * This method is useful for credential vending scenarios where storage options
   * may be refreshed dynamically. If no dynamic provider is configured, this
   * returns the initial static options.
   *
   * Warning: This is an internal API and the return value is subject to change.
   *
   * @returns The storage options, or undefined if no storage options were configured.
   */
  abstract latestStorageOptions(): Promise<
    Record<string, string> | null | undefined
  >;
}

export class LocalTable extends Table {
  private readonly inner: _NativeTable;

  constructor(inner: _NativeTable) {
    super();
    this.inner = inner;
  }
  get name(): string {
    return this.inner.name;
  }
  isOpen(): boolean {
    return this.inner.isOpen();
  }

  close(): void {
    this.inner.close();
  }

  display(): string {
    return this.inner.display();
  }

  private async getEmbeddingFunctions(
    inner: _NativeTable = this.inner,
  ): Promise<Map<string, EmbeddingFunctionConfig>> {
    const schemaBuf = await inner.schema();
    const schema = tableFromIPC(schemaBuf).schema;
    const registry = getRegistry();
    return registry.parseFunctions(schema.metadata);
  }

  /** Get the schema of the table. */
  async schema(): Promise<Schema> {
    const schemaBuf = await this.inner.schema();
    const tbl = tableFromIPC(schemaBuf);
    return tbl.schema;
  }

  async add(data: Data, options?: Partial<AddDataOptions>): Promise<AddResult> {
    const mode = options?.mode ?? "append";
    const schema = await this.schema();

    const buffer = await fromDataToBuffer(data, undefined, schema);
    // Wrap the user callback so a thrown error doesn't surface as an
    // unhandled exception (the callback fires from a napi threadsafe
    // function — exceptions there crash the process).
    const userProgress = options?.progress;
    const progress = userProgress
      ? (p: WriteProgress) => {
          try {
            userProgress(p);
          } catch (e) {
            console.warn("Table.add progress callback threw:", e);
          }
        }
      : undefined;
    return await this.inner.add(
      buffer,
      mode,
      progress,
      options?.allowExternalBlobOutsideBases,
    );
  }

  async update(
    optsOrUpdates:
      | (Map<string, string> | Record<string, string>)
      | ({
          values: Map<string, IntoSql> | Record<string, IntoSql>;
        } & Partial<UpdateOptions>)
      | ({
          valuesSql: Map<string, string> | Record<string, string>;
        } & Partial<UpdateOptions>),
    options?: Partial<UpdateOptions>,
  ): Promise<UpdateResult> {
    const isValues =
      "values" in optsOrUpdates && typeof optsOrUpdates.values !== "string";
    const isValuesSql =
      "valuesSql" in optsOrUpdates &&
      typeof optsOrUpdates.valuesSql !== "string";
    const isMap = (obj: unknown): obj is Map<string, string> => {
      return obj instanceof Map;
    };

    let predicate;
    let columns: [string, string][];
    switch (true) {
      case isMap(optsOrUpdates):
        columns = Array.from(optsOrUpdates.entries());
        predicate = options?.where;
        break;
      case isValues && isMap(optsOrUpdates.values):
        columns = Array.from(optsOrUpdates.values.entries()).map(([k, v]) => [
          k,
          toSQL(v),
        ]);
        predicate = optsOrUpdates.where;
        break;
      case isValues && !isMap(optsOrUpdates.values):
        columns = Object.entries(optsOrUpdates.values).map(([k, v]) => [
          k,
          toSQL(v),
        ]);
        predicate = optsOrUpdates.where;
        break;

      case isValuesSql && isMap(optsOrUpdates.valuesSql):
        columns = Array.from(optsOrUpdates.valuesSql.entries());
        predicate = optsOrUpdates.where;
        break;
      case isValuesSql && !isMap(optsOrUpdates.valuesSql):
        columns = Object.entries(optsOrUpdates.valuesSql).map(([k, v]) => [
          k,
          v,
        ]);
        predicate = optsOrUpdates.where;
        break;
      default:
        columns = Object.entries(optsOrUpdates as Record<string, string>);
        predicate = options?.where;
    }
    return await this.inner.update(predicate, columns);
  }

  async countRows(filter?: string): Promise<number> {
    return await this.inner.countRows(filter);
  }

  async delete(predicate: string): Promise<DeleteResult> {
    return await this.inner.delete(predicate);
  }

  async createIndex(column: string, options?: Partial<IndexOptions>) {
    // Bit of a hack to get around the fact that TS has no package-scope.
    // biome-ignore lint/suspicious/noExplicitAny: skip
    const nativeIndex = (options?.config as any)?.inner;
    await this.inner.createIndex(
      nativeIndex,
      column,
      options?.replace,
      options?.waitTimeoutSeconds,
      options?.name,
      options?.train,
    );
  }

  async createIndexAsync(
    column: string,
    options?: Partial<IndexOptions>,
  ): Promise<Job> {
    // biome-ignore lint/suspicious/noExplicitAny: skip
    const nativeIndex = (options?.config as any)?.inner;
    return new Job(
      await this.inner.createIndexAsync(
        nativeIndex,
        column,
        options?.replace,
        options?.waitTimeoutSeconds,
        options?.name,
        options?.train,
      ),
    );
  }

  async dropIndex(name: string): Promise<void> {
    await this.inner.dropIndex(name);
  }

  async prewarmIndex(name: string): Promise<void> {
    await this.inner.prewarmIndex(name);
  }

  async prewarmData(columns?: string[]): Promise<void> {
    await this.inner.prewarmData(columns);
  }

  async waitForIndex(
    indexNames: string[],
    timeoutSeconds: number,
  ): Promise<void> {
    await this.inner.waitForIndex(indexNames, timeoutSeconds);
  }

  takeOffsets(offsets: number[]): TakeQuery {
    return new TakeQuery(this.inner.takeOffsets(offsets), {
      table: this.inner,
      withRowId: false,
    });
  }

  takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery {
    return new TakeQuery(this.inner.takeRowIds(rowIdsToBigInts(rowIds)), {
      table: this.inner,
      withRowId: false,
    });
  }

  blobColumns(): Promise<string[]> {
    return this.inner.blobColumns();
  }

  async fetchBlobs(
    column: string,
    rowIds: readonly (bigint | number)[],
    options?: BlobReadOptions,
  ): Promise<(Buffer | null)[]> {
    const values = await runWithSignal(options?.signal, (signal) =>
      this.inner.fetchBlobs(column, rowIdsToBigInts(rowIds), signal),
    );
    // N-API Option maps missing values to undefined. Collapse those to null.
    return values.map((value) => value ?? null);
  }

  async fetchBlobFiles(
    column: string,
    rowIds: readonly (bigint | number)[],
    options?: BlobReadOptions,
  ): Promise<(BlobFile | null)[]> {
    const files = await runWithSignal(options?.signal, (signal) =>
      this.inner.fetchBlobFiles(column, rowIdsToBigInts(rowIds), signal),
    );
    // N-API Option maps missing values to undefined. Collapse those to null.
    return files.map((file) =>
      file == null ? null : BlobFile.fromNative(file),
    );
  }

  query(): Query {
    return new Query(this.inner);
  }

  search(
    query: string | IntoVector | MultiVector | FullTextQuery,
    queryType: string = "auto",
    ftsColumns?: string | string[],
  ): VectorQuery | Query | AutoQuery {
    if (typeof query !== "string" && !instanceOfFullTextQuery(query)) {
      if (queryType === "fts") {
        throw new Error("Cannot perform full text search on a vector query");
      }
      return this.vectorSearch(query);
    }

    // If the query is a string, we need to determine if it is a vector query or a full text search query
    if (queryType === "fts") {
      return this.query().fullTextSearch(query, {
        columns: ftsColumns,
      });
    }

    if (queryType === "auto") {
      if (instanceOfFullTextQuery(query)) {
        return this.query().fullTextSearch(query, {
          columns: ftsColumns,
        });
      }

      const columns =
        typeof ftsColumns === "string" ? [ftsColumns] : (ftsColumns ?? null);
      return createAutoQuery(this.inner, query, columns, async (metadata) => {
        const functions = await getRegistry().parseFunctions(
          new Map([["embedding_functions", metadata]]),
        );
        // TODO: Support multiple embedding functions
        const embeddingFunc: EmbeddingFunctionConfig | undefined = functions
          .values()
          .next().value;
        // The route only calls this callback when embedding metadata exists.
        // parseFunctions either yields a provider or reports malformed metadata.
        if (!embeddingFunc)
          throw new Error("Invalid embedding function metadata");
        return await embeddingFunc.function.computeQueryEmbeddings(query);
      });
    }

    const queryPromise = this.getEmbeddingFunctions().then(
      async (functions) => {
        // TODO: Support multiple embedding functions
        const embeddingFunc: EmbeddingFunctionConfig | undefined = functions
          .values()
          .next().value;
        if (!embeddingFunc) {
          return Promise.reject(
            new Error("No embedding functions are defined in the table"),
          );
        }
        return await embeddingFunc.function.computeQueryEmbeddings(query);
      },
    );

    return this.query().nearestTo(queryPromise);
  }

  vectorSearch(vector: IntoVector | MultiVector): VectorQuery {
    if (isMultiVector(vector)) {
      const query = this.query().nearestTo(vector[0]);
      for (const v of vector.slice(1)) {
        query.addQueryVector(v);
      }
      return query;
    }

    return this.query().nearestTo(vector);
  }

  // TODO: Support BatchUDF

  async addColumns(
    newColumnTransforms:
      | AddColumnsSql[]
      | Field
      | Field[]
      | Schema
      | { computed: AddColumnsSql[] },
  ): Promise<AddColumnsResult> {
    // Columns defined by an expression are declared, not materialized here.
    if (
      typeof newColumnTransforms === "object" &&
      !Array.isArray(newColumnTransforms) &&
      "computed" in newColumnTransforms
    ) {
      return await this.inner.addComputedColumns(newColumnTransforms.computed);
    }

    // Handle single Field -> convert to array of Fields
    if (newColumnTransforms instanceof Field) {
      newColumnTransforms = [newColumnTransforms];
    }

    // Handle array of Fields -> convert to Schema
    if (
      Array.isArray(newColumnTransforms) &&
      newColumnTransforms.length > 0 &&
      newColumnTransforms[0] instanceof Field
    ) {
      const fields = newColumnTransforms as Field[];
      newColumnTransforms = new Schema(fields);
    }

    // Handle Schema -> use schema-based approach
    if (newColumnTransforms instanceof Schema) {
      const schema = newColumnTransforms;
      // Convert schema to buffer using Arrow IPC format
      const emptyTable = makeEmptyTable(schema);
      const schemaBuf = await fromTableToBuffer(emptyTable);
      return await this.inner.addColumnsWithSchema(schemaBuf);
    }

    // Handle SQL expressions (existing functionality)
    if (Array.isArray(newColumnTransforms)) {
      return await this.inner.addColumns(
        newColumnTransforms as AddColumnsSql[],
      );
    }

    throw new Error("Invalid input type for addColumns");
  }

  async refreshColumn(
    column: string,
    options?: RefreshColumnOptions,
  ): Promise<RefreshColumnResult> {
    return await this.inner.refreshColumn(column, options);
  }

  async refreshColumnAsync(
    column: string,
    options?: RefreshColumnOptions,
  ): Promise<Job> {
    return new Job(await this.inner.refreshColumnAsync(column, options));
  }

  async functionErrors(
    options?: FunctionErrorsOptions,
  ): Promise<FunctionErrors> {
    validateNonNegativeInteger(options?.limit, "limit");
    return await this.inner.functionErrors(options);
  }

  async functionColumnStatuses(
    column?: string,
  ): Promise<FunctionColumnStatus[]> {
    return (await this.inner.functionColumnStatuses(
      column,
    )) as FunctionColumnStatus[];
  }

  async functionColumnStatus(column: string): Promise<FunctionColumnStatus> {
    return (await this.inner.functionColumnStatus(
      column,
    )) as FunctionColumnStatus;
  }

  async waitFresh(
    column: string,
    timeoutSeconds?: number,
  ): Promise<FunctionColumnStatus> {
    validateNonNegativeInteger(timeoutSeconds, "timeoutSeconds");
    return (await this.inner.waitForFunctionColumn(
      column,
      timeoutSeconds,
    )) as FunctionColumnStatus;
  }

  async refreshMaterializedView(
    sourceVersion?: number,
    full?: boolean,
  ): Promise<RefreshMaterializedViewResult> {
    return await this.inner.refreshMaterializedView(sourceVersion, full);
  }

  async materializedViewDefinition(): Promise<string> {
    return await this.inner.materializedViewDefinition();
  }

  async alterColumns(
    columnAlterations: ColumnAlteration[],
  ): Promise<AlterColumnsResult> {
    const processedAlterations = columnAlterations.map((alteration) => {
      if (typeof alteration.dataType === "string") {
        return {
          ...alteration,
          dataType: JSON.stringify({ type: alteration.dataType }),
        };
      } else if (alteration.dataType === undefined) {
        return {
          ...alteration,
          dataType: undefined,
        };
      } else {
        const dataType = sanitizeType(alteration.dataType);
        return {
          ...alteration,
          dataType: JSON.stringify(dataTypeToJson(dataType)),
        };
      }
    });

    return await this.inner.alterColumns(processedAlterations);
  }

  async updateFieldMetadata(
    updates: FieldMetadataUpdate[],
  ): Promise<UpdateFieldMetadataResult> {
    return await this.inner.updateFieldMetadata(updates);
  }

  async dropColumns(columnNames: string[]): Promise<DropColumnsResult> {
    return await this.inner.dropColumns(columnNames);
  }

  async setUnenforcedPrimaryKey(columns: string | string[]): Promise<void> {
    const cols = typeof columns === "string" ? [columns] : columns;
    return await this.inner.setUnenforcedPrimaryKey(cols);
  }

  async enableRTE(spec: RTEWriteSpec): Promise<void> {
    return await this.inner.enableRTE(spec);
  }

  async disableRTE(): Promise<void> {
    return await this.inner.disableRTE();
  }

  async getRTESpec(): Promise<RTEWriteSpec | undefined> {
    // The native binding types `specType` as a plain `string`; narrow it back
    // to the public union. The Rust `From` impl only ever emits one of the
    // three valid values, so the cast is safe.
    return ((await this.inner.getRTESpec()) ?? undefined) as
      | RTEWriteSpec
      | undefined;
  }

  async closeRTEWriters(): Promise<void> {
    return await this.inner.closeRTEWriters();
  }

  async flushRTE(): Promise<void> {
    return await this.inner.flushRTE();
  }

  async compactRTE(): Promise<void> {
    return await this.inner.compactRTE();
  }

  async checkpointRTE(): Promise<void> {
    return await this.inner.checkpointRTE();
  }

  async getRTEStats(
    includeSstableRows: boolean = false,
  ): Promise<RTEStats | undefined> {
    return (await this.inner.getRTEStats(includeSstableRows)) ?? undefined;
  }

  async version(): Promise<number> {
    return await this.inner.version();
  }

  async checkout(version: number | string): Promise<void> {
    if (typeof version === "string") {
      return this.inner.checkoutTag(version);
    }
    return this.inner.checkout(version);
  }

  async checkoutLatest(): Promise<void> {
    await this.inner.checkoutLatest();
  }

  async listVersions(): Promise<Version[]> {
    return (await this.inner.listVersions()).map((version) => ({
      version: version.version,
      timestamp: new Date(version.timestamp / 1000),
      metadata: version.metadata,
    }));
  }

  async restore(): Promise<void> {
    await this.inner.restore();
  }

  async tags(): Promise<Tags> {
    return await this.inner.tags();
  }

  async branches(): Promise<Branches> {
    return new Branches(await this.inner.branches());
  }

  currentBranch(): string | null {
    return this.inner.currentBranch() ?? null;
  }

  async optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats> {
    // JavaScript callers can supply extra keys that TypeScript would reject.
    // Refuse them before the native binding can silently discard a work limit.
    const supported = new Set([
      "targetRowsPerFragment",
      "maxSourceRows",
      "ioBufferSize",
      "batchSize",
    ]);
    for (const key of Object.keys(options?.compactionOptions ?? {})) {
      if (!supported.has(key)) {
        throw new Error(`Unsupported compaction option: ${key}`);
      }
    }
    return await this.inner.optimize(
      options?.cleanupOlderThan?.getTime(),
      options?.deleteUnverified,
      options?.compactionOptions,
    );
  }

  async listIndices(): Promise<IndexConfig[]> {
    return await this.inner.listIndices();
  }

  async tokenize(
    query: string,
    options: TokenizeTableOptions,
  ): Promise<FtsToken[]> {
    return await this.inner.tokenize(
      query,
      options?.column,
      options?.indexName,
    );
  }

  async toArrow(): Promise<ArrowTable> {
    return await this.query().toArrow();
  }

  async indexStats(name: string): Promise<IndexStatistics | undefined> {
    const stats = await this.inner.indexStats(name);
    if (stats === null) {
      return undefined;
    }
    return stats;
  }

  async stats(): Promise<TableStatistics> {
    return await this.inner.stats();
  }

  async initialStorageOptions(): Promise<
    Record<string, string> | null | undefined
  > {
    return await this.inner.initialStorageOptions();
  }

  async latestStorageOptions(): Promise<
    Record<string, string> | null | undefined
  > {
    return await this.inner.latestStorageOptions();
  }

  mergeInsert(on: string | string[]): MergeInsertBuilder {
    on = Array.isArray(on) ? on : [on];
    return new MergeInsertBuilder(this.inner.mergeInsert(on), this.schema());
  }

  /**
   * Check if the table uses the new manifest path scheme.
   *
   * This function will return true if the table uses the V2 manifest
   * path scheme.
   */
  async usesV2ManifestPaths(): Promise<boolean> {
    return await this.inner.usesV2ManifestPaths();
  }

  /**
   * Migrate the table to use the new manifest path scheme.
   *
   * This function will rename all V1 manifests to V2 manifest paths.
   * These paths provide more efficient opening of datasets with many versions
   * on object stores.
   *
   * This function is idempotent, and can be run multiple times without
   * changing the state of the object store.
   *
   * However, it should not be run while other concurrent operations are happening.
   * And it should also run until completion before resuming other operations.
   */
  async migrateManifestPathsV2(): Promise<void> {
    await this.inner.migrateManifestPathsV2();
  }
}

/**
 *  A definition of a column alteration. The alteration changes the column at
 * `path` to have the new name `name`, to be nullable if `nullable` is true,
 * and to have the data type `data_type`. At least one of `rename` or `nullable`
 * must be provided.
 */
export interface ColumnAlteration {
  /**
   * The path to the column to alter. This is a dot-separated path to the column.
   * If it is a top-level column then it is just the name of the column. If it is
   * a nested column then it is the path to the column, e.g. "a.b.c" for a column
   * `c` nested inside a column `b` nested inside a column `a`.
   */
  path: string;
  /**
   * The new name of the column. If not provided then the name will not be changed.
   * This must be distinct from the names of all other columns in the table.
   */
  rename?: string;
  /**
   * A new data type for the column. If not provided then the data type will not be changed.
   * Changing data types is limited to casting to the same general type. For example, these
   * changes are valid:
   * * `int32` -> `int64` (integers)
   * * `double` -> `float` (floats)
   * * `string` -> `large_string` (strings)
   * But these changes are not:
   * * `int32` -> `double` (mix integers and floats)
   * * `string` -> `int32` (mix strings and integers)
   */
  dataType?: string | DataType;
  /** Set the new nullability. Note that a nullable column cannot be made non-nullable. */
  nullable?: boolean;
}

/** A per-field metadata update, addressed by dot-path. */
export interface FieldMetadataUpdate {
  /**
   * Dot-separated path to the field. For a top-level column this is just its
   * name; for a nested field it's the path, e.g. "a.b.c".
   */
  path: string;
  /**
   * Metadata key/value pairs. Merged into the field's existing metadata by
   * default; a value of `null` deletes that key. See
   * {@link Table.updateFieldMetadata} for the conventional `lancedb:*` keys.
   */
  metadata: Record<string, string | null>;
  /** If true, replace the field's entire metadata map instead of merging. */
  replace?: boolean;
}

/** Summary of a column in a branch diff. */
export interface BranchColumnSummary {
  name: string;
  dataType: string;
  nullable: boolean;
}

/** A column whose definition differs between main and the branch. */
export interface BranchColumnChange {
  name: string;
  main: BranchColumnSummary;
  branch: BranchColumnSummary;
}

/** Summary of an index in a branch diff. */
export interface BranchIndexSummary {
  indexName: string;
  columns: string[];
  indexType?: string;
  status: string;
}

/** Row-level comparison between main and the branch. */
export interface BranchRowCountSummary {
  unchanged: number;
  newOnBase: number;
  newOnBranch: number;
  staleRecompute: number;
  inputsChanged: number;
  deltaAvailable: boolean;
}

/** A reason why a cherry-pick cannot currently land. */
export interface CherryPickError {
  code: string;
  message: string;
}

/** Read-only comparison of a branch against main. */
export interface BranchDiff {
  fromBranch: string;
  parentVersion: number;
  mainVersion: number;
  branchVersion: number;
  baseMoved: boolean;
  rowCountMain: number;
  rowCountBranch: number;
  rowSummary: BranchRowCountSummary;
  addedColumns: BranchColumnSummary[];
  removedColumns: BranchColumnSummary[];
  changedColumns: BranchColumnChange[];
  addedIndexes: BranchIndexSummary[];
  removedIndexes: BranchIndexSummary[];
  errors: CherryPickError[];
}

/** Changes that would be, or were, promoted by a cherry-pick. */
export interface CherryPickPreview {
  promotedColumns: string[];
}

/** Result of previewing or attempting a cherry-pick. */
export interface CherryPickResult {
  status: "ready" | "failed" | "notImplemented" | "cherryPicked" | "unknown";
  diff: BranchDiff;
  preview: CherryPickPreview;
  mainVersionAfter?: number;
}

/**
 * Branch manager for a {@link Table}.
 *
 * Unlike tags, `create` and `checkout` return a new {@link Table} handle scoped
 * to the branch; writes on it do not affect `main`.
 */
export class Branches {
  #inner: NativeBranches;

  /**
   * Construct a Branches manager. Internal use only.
   * @hidden
   */
  constructor(inner: NativeBranches) {
    this.#inner = inner;
  }

  /** List all branches, mapping name to branch metadata. */
  async list(): Promise<Record<string, BranchContents>> {
    return await this.#inner.list();
  }

  /**
   * Create a branch and return a handle scoped to it.
   *
   * @param name Name of the new branch.
   * @param fromRef Source branch to fork from. Defaults to `main`.
   * @param fromVersion A specific version on `fromRef`. Defaults to latest.
   */
  async create(
    name: string,
    fromRef?: string,
    fromVersion?: number,
  ): Promise<Table> {
    return new LocalTable(await this.#inner.create(name, fromRef, fromVersion));
  }

  /**
   * Check out an existing branch and return a handle scoped to it.
   *
   * With `version` set, the returned handle is pinned to that version of the
   * branch (a read-only, detached view); otherwise it tracks the branch's
   * latest and stays writable.
   */
  async checkout(name: string, version?: number): Promise<Table> {
    return new LocalTable(await this.#inner.checkout(name, version));
  }

  /** Delete a branch. */
  async delete(name: string): Promise<void> {
    return await this.#inner.delete(name);
  }

  /** Compare a branch against main without modifying either branch. */
  async diff(fromBranch: string): Promise<BranchDiff> {
    return (await this.#inner.diff(fromBranch)) as unknown as BranchDiff;
  }

  /**
   * Cherry-pick a branch onto main.
   *
   * Set `dryRun` to `true` to preview. A failed cherry-pick resolves
   * with `status: "failed"` instead of throwing.
   *
   * @param fromBranch Branch to cherry-pick from.
   * @param dryRun When true, only preview. Defaults to false.
   */
  async cherryPick(
    fromBranch: string,
    dryRun: boolean = false,
  ): Promise<CherryPickResult> {
    return (await this.#inner.cherryPick(
      fromBranch,
      dryRun,
    )) as unknown as CherryPickResult;
  }
}

function rowIdsToBigInts(rowIds: readonly (bigint | number)[]): bigint[] {
  return rowIds.map((id) => {
    if (typeof id === "bigint") {
      return id;
    }
    if (!Number.isInteger(id)) {
      throw new Error("Row id must be an integer (or bigint)");
    }
    if (id < 0) {
      throw new Error("Row id cannot be negative");
    }
    if (!Number.isSafeInteger(id)) {
      throw new Error("Row id is too large for number; use bigint instead");
    }
    return BigInt(id);
  });
}