refactor: rename the rust crate from vectordb to lancedb (#1012)

This also renames the new experimental node package to lancedb. The classic node package remains named vectordb. The goal here is to avoid introducing piecemeal breaking changes to the vectordb crate. Instead, once the new API is stabilized, we will officially release the lancedb crate and deprecate the vectordb crate. The same pattern will eventually happen with the npm package vectordb.
2026-01-07 04:12:59 +00:00 · 2024-02-22 19:56:39 -08:00
parent c5b0934bfb
commit 2163502b31
58 changed files with 140 additions and 137 deletions
--- a/nodejs/lancedb/arrow.ts
+++ b/nodejs/lancedb/arrow.ts
@@ -0,0 +1,188 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {
+  Int64,
+  Field,
+  FixedSizeList,
+  Float,
+  Float32,
+  Schema,
+  Table as ArrowTable,
+  Table,
+  Vector,
+  vectorFromArray,
+  tableToIPC,
+  DataType,
+} from "apache-arrow";
+
+/** Data type accepted by NodeJS SDK */
+export type Data = Record<string, unknown>[] | ArrowTable;
+
+export class VectorColumnOptions {
+  /** Vector column type. */
+  type: Float = new Float32();
+
+  constructor(values?: Partial<VectorColumnOptions>) {
+    Object.assign(this, values);
+  }
+}
+
+/** Options to control the makeArrowTable call. */
+export class MakeArrowTableOptions {
+  /** Provided schema. */
+  schema?: Schema;
+
+  /** Vector columns */
+  vectorColumns: Record<string, VectorColumnOptions> = {
+    vector: new VectorColumnOptions(),
+  };
+
+  constructor(values?: Partial<MakeArrowTableOptions>) {
+    Object.assign(this, values);
+  }
+}
+
+/**
+ * An enhanced version of the {@link makeTable} function from Apache Arrow
+ * that supports nested fields and embeddings columns.
+ *
+ * Note that it currently does not support nulls.
+ *
+ * @param data input data
+ * @param options options to control the makeArrowTable call.
+ *
+ * @example
+ *
+ * ```ts
+ *
+ * import { fromTableToBuffer, makeArrowTable } from "../arrow";
+ * import { Field, FixedSizeList, Float16, Float32, Int32, Schema } from "apache-arrow";
+ *
+ * const schema = new Schema([
+ *   new Field("a", new Int32()),
+ *   new Field("b", new Float32()),
+ *   new Field("c", new FixedSizeList(3, new Field("item", new Float16()))),
+ *  ]);
+ *  const table = makeArrowTable([
+ *    { a: 1, b: 2, c: [1, 2, 3] },
+ *    { a: 4, b: 5, c: [4, 5, 6] },
+ *    { a: 7, b: 8, c: [7, 8, 9] },
+ *  ], { schema });
+ * ```
+ *
+ * It guesses the vector columns if the schema is not provided. For example,
+ * by default it assumes that the column named `vector` is a vector column.
+ *
+ * ```ts
+ *
+ * const schema = new Schema([
+ new Field("a", new Float64()),
+ new Field("b", new Float64()),
+ new Field(
+ "vector",
+ new FixedSizeList(3, new Field("item", new Float32()))
+ ),
+ ]);
+ const table = makeArrowTable([
+ { a: 1, b: 2, vector: [1, 2, 3] },
+ { a: 4, b: 5, vector: [4, 5, 6] },
+ { a: 7, b: 8, vector: [7, 8, 9] },
+ ]);
+ assert.deepEqual(table.schema, schema);
+ * ```
+ *
+ * You can specify the vector column types and names using the options as well
+ *
+ * ```typescript
+ *
+ * const schema = new Schema([
+ new Field('a', new Float64()),
+ new Field('b', new Float64()),
+ new Field('vec1', new FixedSizeList(3, new Field('item', new Float16()))),
+ new Field('vec2', new FixedSizeList(3, new Field('item', new Float16())))
+ ]);
+ * const table = makeArrowTable([
+ { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
+ { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
+ { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] }
+ ], {
+ vectorColumns: {
+ vec1: { type: new Float16() },
+ vec2: { type: new Float16() }
+ }
+ }
+ * assert.deepEqual(table.schema, schema)
+ * ```
+ */
+export function makeArrowTable(
+  data: Record<string, any>[],
+  options?: Partial<MakeArrowTableOptions>
+): Table {
+  if (data.length === 0) {
+    throw new Error("At least one record needs to be provided");
+  }
+  const opt = new MakeArrowTableOptions(options ?? {});
+  const columns: Record<string, Vector> = {};
+  // TODO: sample dataset to find missing columns
+  const columnNames = Object.keys(data[0]);
+  for (const colName of columnNames) {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+    let values = data.map((datum) => datum[colName]);
+    let vector: Vector;
+
+    if (opt.schema !== undefined) {
+      // Explicit schema is provided, highest priority
+      const fieldType: DataType | undefined = opt.schema.fields.filter((f) => f.name === colName)[0]?.type as DataType;
+      if (fieldType instanceof Int64) {
+        // wrap in BigInt to avoid bug: https://github.com/apache/arrow/issues/40051
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+        values = values.map((v) => BigInt(v));
+      }
+      vector = vectorFromArray(values, fieldType);
+    } else {
+      const vectorColumnOptions = opt.vectorColumns[colName];
+      if (vectorColumnOptions !== undefined) {
+        const fslType = new FixedSizeList(
+          (values[0] as any[]).length,
+          new Field("item", vectorColumnOptions.type, false)
+        );
+        vector = vectorFromArray(values, fslType);
+      } else {
+        // Normal case
+        vector = vectorFromArray(values);
+      }
+    }
+    columns[colName] = vector;
+  }
+
+  return new Table(columns);
+}
+
+/**
+ * Convert an Arrow Table to a Buffer.
+ *
+ * @param data Arrow Table
+ * @param schema Arrow Schema, optional
+ * @returns Buffer node
+ */
+export function toBuffer(data: Data, schema?: Schema): Buffer {
+  let tbl: Table;
+  if (data instanceof Table) {
+    tbl = data;
+  } else {
+    tbl = makeArrowTable(data, { schema });
+  }
+  return Buffer.from(tableToIPC(tbl));
+}
--- a/nodejs/lancedb/connection.ts
+++ b/nodejs/lancedb/connection.ts
@@ -0,0 +1,70 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { toBuffer } from "./arrow";
+import { Connection as _NativeConnection } from "./native";
+import { Table } from "./table";
+import { Table as ArrowTable } from "apache-arrow";
+
+/**
+ * A LanceDB Connection that allows you to open tables and create new ones.
+ *
+ * Connection could be local against filesystem or remote against a server.
+ */
+export class Connection {
+  readonly inner: _NativeConnection;
+
+  constructor(inner: _NativeConnection) {
+    this.inner = inner;
+  }
+
+  /** List all the table names in this database. */
+  async tableNames(): Promise<string[]> {
+    return this.inner.tableNames();
+  }
+
+  /**
+   * Open a table in the database.
+   *
+   * @param name The name of the table.
+   * @param embeddings An embedding function to use on this table
+   */
+  async openTable(name: string): Promise<Table> {
+    const innerTable = await this.inner.openTable(name);
+    return new Table(innerTable);
+  }
+
+  /**
+   * Creates a new Table and initialize it with new data.
+   *
+   * @param {string} name - The name of the table.
+   * @param data - Non-empty Array of Records to be inserted into the table
+   */
+  async createTable(
+    name: string,
+    data: Record<string, unknown>[] | ArrowTable
+  ): Promise<Table> {
+    const buf = toBuffer(data);
+    const innerTable = await this.inner.createTable(name, buf);
+    return new Table(innerTable);
+  }
+
+  /**
+   * Drop an existing table.
+   * @param name The name of the table to drop.
+   */
+  async dropTable(name: string): Promise<void> {
+    return this.inner.dropTable(name);
+  }
+}
--- a/nodejs/lancedb/index.ts
+++ b/nodejs/lancedb/index.ts
@@ -0,0 +1,64 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { Connection } from "./connection";
+import { Connection as NativeConnection, ConnectionOptions } from "./native.js";
+
+export {
+  ConnectionOptions,
+  WriteOptions,
+  Query,
+  MetricType,
+} from "./native.js";
+export { Connection } from "./connection";
+export { Table } from "./table";
+export { Data } from "./arrow";
+export { IvfPQOptions, IndexBuilder } from "./indexer";
+
+/**
+ * Connect to a LanceDB instance at the given URI.
+ *
+ * Accpeted formats:
+ *
+ * - `/path/to/database` - local database
+ * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+ * - `db://host:port` - remote database (LanceDB cloud)
+ *
+ * @param uri The uri of the database. If the database uri starts with `db://` then it connects to a remote database.
+ *
+ * @see {@link ConnectionOptions} for more details on the URI format.
+ */
+export async function connect(uri: string): Promise<Connection>;
+export async function connect(
+  opts: Partial<ConnectionOptions>
+): Promise<Connection>;
+export async function connect(
+  args: string | Partial<ConnectionOptions>
+): Promise<Connection> {
+  let opts: ConnectionOptions;
+  if (typeof args === "string") {
+    opts = { uri: args };
+  } else {
+    opts = Object.assign(
+      {
+        uri: "",
+        apiKey: undefined,
+        hostOverride: undefined,
+      },
+      args
+    );
+  }
+  const nativeConn = await NativeConnection.new(opts);
+  return new Connection(nativeConn);
+}
--- a/nodejs/lancedb/indexer.ts
+++ b/nodejs/lancedb/indexer.ts
@@ -0,0 +1,102 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {
+  MetricType,
+  IndexBuilder as NativeBuilder,
+  Table as NativeTable,
+} from "./native";
+
+/** Options to create `IVF_PQ` index */
+export interface IvfPQOptions {
+  /** Number of IVF partitions. */
+  num_partitions?: number;
+
+  /** Number of sub-vectors in PQ coding. */
+  num_sub_vectors?: number;
+
+  /** Number of bits used for each PQ code.
+   */
+  num_bits?: number;
+
+  /** Metric type to calculate the distance between vectors.
+   *
+   * Supported metrics: `L2`, `Cosine` and `Dot`.
+   */
+  metric_type?: MetricType;
+
+  /** Number of iterations to train K-means.
+   *
+   * Default is 50. The more iterations it usually yield better results,
+   * but it takes longer to train.
+   */
+  max_iterations?: number;
+
+  sample_rate?: number;
+}
+
+/**
+ * Building an index on LanceDB {@link Table}
+ *
+ * @see {@link Table.createIndex} for detailed usage.
+ */
+export class IndexBuilder {
+  private inner: NativeBuilder;
+
+  constructor(tbl: NativeTable) {
+    this.inner = tbl.createIndex();
+  }
+
+  /** Instruct the builder to build an `IVF_PQ` index */
+  ivf_pq(options?: IvfPQOptions): IndexBuilder {
+    this.inner.ivfPq(
+      options?.metric_type,
+      options?.num_partitions,
+      options?.num_sub_vectors,
+      options?.num_bits,
+      options?.max_iterations,
+      options?.sample_rate
+    );
+    return this;
+  }
+
+  /** Instruct the builder to build a Scalar index. */
+  scalar(): IndexBuilder {
+    this.scalar();
+    return this;
+  }
+
+  /** Set the column(s) to create index on top of. */
+  column(col: string): IndexBuilder {
+    this.inner.column(col);
+    return this;
+  }
+
+  /** Set to true to replace existing index. */
+  replace(val: boolean): IndexBuilder {
+    this.inner.replace(val);
+    return this;
+  }
+
+  /** Specify the name of the index. Optional */
+  name(n: string): IndexBuilder {
+    this.inner.name(n);
+    return this;
+  }
+
+  /** Building the index. */
+  async build() {
+    await this.inner.build();
+  }
+}
--- a/nodejs/lancedb/native.d.ts
+++ b/nodejs/lancedb/native.d.ts
@@ -0,0 +1,92 @@
+/* tslint:disable */
+/* eslint-disable */
+
+/* auto-generated by NAPI-RS */
+
+export const enum IndexType {
+  Scalar = 0,
+  IvfPq = 1
+}
+export const enum MetricType {
+  L2 = 0,
+  Cosine = 1,
+  Dot = 2
+}
+export interface ConnectionOptions {
+  uri: string
+  apiKey?: string
+  hostOverride?: string
+  /**
+   * (For LanceDB OSS only): The interval, in seconds, at which to check for
+   * updates to the table from other processes. If None, then consistency is not
+   * checked. For performance reasons, this is the default. For strong
+   * consistency, set this to zero seconds. Then every read will check for
+   * updates from other processes. As a compromise, you can set this to a
+   * non-zero value for eventual consistency. If more than that interval
+   * has passed since the last check, then the table will be checked for updates.
+   * Note: this consistency only applies to read operations. Write operations are
+   * always consistent.
+   */
+  readConsistencyInterval?: number
+}
+/** Write mode for writing a table. */
+export const enum WriteMode {
+  Create = 'Create',
+  Append = 'Append',
+  Overwrite = 'Overwrite'
+}
+/** Write options when creating a Table. */
+export interface WriteOptions {
+  mode?: WriteMode
+}
+export function connect(options: ConnectionOptions): Promise<Connection>
+export class Connection {
+  /** Create a new Connection instance from the given URI. */
+  static new(options: ConnectionOptions): Promise<Connection>
+  /** List all tables in the dataset. */
+  tableNames(): Promise<Array<string>>
+  /**
+   * Create table from a Apache Arrow IPC (file) buffer.
+   *
+   * Parameters:
+   * - name: The name of the table.
+   * - buf: The buffer containing the IPC file.
+   *
+   */
+  createTable(name: string, buf: Buffer): Promise<Table>
+  openTable(name: string): Promise<Table>
+  /** Drop table with the name. Or raise an error if the table does not exist. */
+  dropTable(name: string): Promise<void>
+}
+export class IndexBuilder {
+  replace(v: boolean): void
+  column(c: string): void
+  name(name: string): void
+  ivfPq(metricType?: MetricType | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, numBits?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): void
+  scalar(): void
+  build(): Promise<void>
+}
+/** Typescript-style Async Iterator over RecordBatches  */
+export class RecordBatchIterator {
+  next(): Promise<Buffer | null>
+}
+export class Query {
+  column(column: string): void
+  filter(filter: string): void
+  select(columns: Array<string>): void
+  limit(limit: number): void
+  prefilter(prefilter: boolean): void
+  nearestTo(vector: Float32Array): void
+  refineFactor(refineFactor: number): void
+  nprobes(nprobe: number): void
+  executeStream(): Promise<RecordBatchIterator>
+}
+export class Table {
+  /** Return Schema as empty Arrow IPC file. */
+  schema(): Promise<Buffer>
+  add(buf: Buffer): Promise<void>
+  countRows(filter?: string | undefined | null): Promise<bigint>
+  delete(predicate: string): Promise<void>
+  createIndex(): IndexBuilder
+  query(): Query
+}
--- a/nodejs/lancedb/native.js
+++ b/nodejs/lancedb/native.js
@@ -0,0 +1,308 @@
+/* tslint:disable */
+/* eslint-disable */
+/* prettier-ignore */
+
+/* auto-generated by NAPI-RS */
+
+const { existsSync, readFileSync } = require('fs')
+const { join } = require('path')
+
+const { platform, arch } = process
+
+let nativeBinding = null
+let localFileExisted = false
+let loadError = null
+
+function isMusl() {
+  // For Node 10
+  if (!process.report || typeof process.report.getReport !== 'function') {
+    try {
+      const lddPath = require('child_process').execSync('which ldd').toString().trim()
+      return readFileSync(lddPath, 'utf8').includes('musl')
+    } catch (e) {
+      return true
+    }
+  } else {
+    const { glibcVersionRuntime } = process.report.getReport().header
+    return !glibcVersionRuntime
+  }
+}
+
+switch (platform) {
+  case 'android':
+    switch (arch) {
+      case 'arm64':
+        localFileExisted = existsSync(join(__dirname, 'lancedb-nodejs.android-arm64.node'))
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./lancedb-nodejs.android-arm64.node')
+          } else {
+            nativeBinding = require('lancedb-android-arm64')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      case 'arm':
+        localFileExisted = existsSync(join(__dirname, 'lancedb-nodejs.android-arm-eabi.node'))
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./lancedb-nodejs.android-arm-eabi.node')
+          } else {
+            nativeBinding = require('lancedb-android-arm-eabi')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      default:
+        throw new Error(`Unsupported architecture on Android ${arch}`)
+    }
+    break
+  case 'win32':
+    switch (arch) {
+      case 'x64':
+        localFileExisted = existsSync(
+          join(__dirname, 'lancedb-nodejs.win32-x64-msvc.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./lancedb-nodejs.win32-x64-msvc.node')
+          } else {
+            nativeBinding = require('lancedb-win32-x64-msvc')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      case 'ia32':
+        localFileExisted = existsSync(
+          join(__dirname, 'lancedb-nodejs.win32-ia32-msvc.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./lancedb-nodejs.win32-ia32-msvc.node')
+          } else {
+            nativeBinding = require('lancedb-win32-ia32-msvc')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      case 'arm64':
+        localFileExisted = existsSync(
+          join(__dirname, 'lancedb-nodejs.win32-arm64-msvc.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./lancedb-nodejs.win32-arm64-msvc.node')
+          } else {
+            nativeBinding = require('lancedb-win32-arm64-msvc')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      default:
+        throw new Error(`Unsupported architecture on Windows: ${arch}`)
+    }
+    break
+  case 'darwin':
+    localFileExisted = existsSync(join(__dirname, 'lancedb-nodejs.darwin-universal.node'))
+    try {
+      if (localFileExisted) {
+        nativeBinding = require('./lancedb-nodejs.darwin-universal.node')
+      } else {
+        nativeBinding = require('lancedb-darwin-universal')
+      }
+      break
+    } catch {}
+    switch (arch) {
+      case 'x64':
+        localFileExisted = existsSync(join(__dirname, 'lancedb-nodejs.darwin-x64.node'))
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./lancedb-nodejs.darwin-x64.node')
+          } else {
+            nativeBinding = require('lancedb-darwin-x64')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      case 'arm64':
+        localFileExisted = existsSync(
+          join(__dirname, 'lancedb-nodejs.darwin-arm64.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./lancedb-nodejs.darwin-arm64.node')
+          } else {
+            nativeBinding = require('lancedb-darwin-arm64')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      default:
+        throw new Error(`Unsupported architecture on macOS: ${arch}`)
+    }
+    break
+  case 'freebsd':
+    if (arch !== 'x64') {
+      throw new Error(`Unsupported architecture on FreeBSD: ${arch}`)
+    }
+    localFileExisted = existsSync(join(__dirname, 'lancedb-nodejs.freebsd-x64.node'))
+    try {
+      if (localFileExisted) {
+        nativeBinding = require('./lancedb-nodejs.freebsd-x64.node')
+      } else {
+        nativeBinding = require('lancedb-freebsd-x64')
+      }
+    } catch (e) {
+      loadError = e
+    }
+    break
+  case 'linux':
+    switch (arch) {
+      case 'x64':
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'lancedb-nodejs.linux-x64-musl.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./lancedb-nodejs.linux-x64-musl.node')
+            } else {
+              nativeBinding = require('lancedb-linux-x64-musl')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'lancedb-nodejs.linux-x64-gnu.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./lancedb-nodejs.linux-x64-gnu.node')
+            } else {
+              nativeBinding = require('lancedb-linux-x64-gnu')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        }
+        break
+      case 'arm64':
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'lancedb-nodejs.linux-arm64-musl.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./lancedb-nodejs.linux-arm64-musl.node')
+            } else {
+              nativeBinding = require('lancedb-linux-arm64-musl')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'lancedb-nodejs.linux-arm64-gnu.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./lancedb-nodejs.linux-arm64-gnu.node')
+            } else {
+              nativeBinding = require('lancedb-linux-arm64-gnu')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        }
+        break
+      case 'arm':
+        localFileExisted = existsSync(
+          join(__dirname, 'lancedb-nodejs.linux-arm-gnueabihf.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./lancedb-nodejs.linux-arm-gnueabihf.node')
+          } else {
+            nativeBinding = require('lancedb-linux-arm-gnueabihf')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      case 'riscv64':
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'lancedb-nodejs.linux-riscv64-musl.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./lancedb-nodejs.linux-riscv64-musl.node')
+            } else {
+              nativeBinding = require('lancedb-linux-riscv64-musl')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'lancedb-nodejs.linux-riscv64-gnu.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./lancedb-nodejs.linux-riscv64-gnu.node')
+            } else {
+              nativeBinding = require('lancedb-linux-riscv64-gnu')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        }
+        break
+      case 's390x':
+        localFileExisted = existsSync(
+          join(__dirname, 'lancedb-nodejs.linux-s390x-gnu.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./lancedb-nodejs.linux-s390x-gnu.node')
+          } else {
+            nativeBinding = require('lancedb-linux-s390x-gnu')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      default:
+        throw new Error(`Unsupported architecture on Linux: ${arch}`)
+    }
+    break
+  default:
+    throw new Error(`Unsupported OS: ${platform}, architecture: ${arch}`)
+}
+
+if (!nativeBinding) {
+  if (loadError) {
+    throw loadError
+  }
+  throw new Error(`Failed to load native binding`)
+}
+
+const { Connection, IndexType, MetricType, IndexBuilder, RecordBatchIterator, Query, Table, WriteMode, connect } = nativeBinding
+
+module.exports.Connection = Connection
+module.exports.IndexType = IndexType
+module.exports.MetricType = MetricType
+module.exports.IndexBuilder = IndexBuilder
+module.exports.RecordBatchIterator = RecordBatchIterator
+module.exports.Query = Query
+module.exports.Table = Table
+module.exports.WriteMode = WriteMode
+module.exports.connect = connect
--- a/nodejs/lancedb/query.ts
+++ b/nodejs/lancedb/query.ts
@@ -0,0 +1,152 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { RecordBatch, tableFromIPC, Table as ArrowTable } from "apache-arrow";
+import {
+  RecordBatchIterator as NativeBatchIterator,
+  Query as NativeQuery,
+  Table as NativeTable,
+} from "./native";
+
+class RecordBatchIterator implements AsyncIterator<RecordBatch> {
+  private promised_inner?: Promise<NativeBatchIterator>;
+  private inner?: NativeBatchIterator;
+
+  constructor(
+    inner?: NativeBatchIterator,
+    promise?: Promise<NativeBatchIterator>
+  ) {
+    // TODO: check promise reliably so we dont need to pass two arguments.
+    this.inner = inner;
+    this.promised_inner = promise;
+  }
+
+  async next(): Promise<IteratorResult<RecordBatch<any>, any>> {
+    if (this.inner === undefined) {
+      this.inner = await this.promised_inner;
+    }
+    if (this.inner === undefined) {
+      throw new Error("Invalid iterator state state");
+    }
+    const n = await this.inner.next();
+    if (n == null) {
+      return Promise.resolve({ done: true, value: null });
+    }
+    const tbl = tableFromIPC(n);
+    if (tbl.batches.length != 1) {
+      throw new Error("Expected only one batch");
+    }
+    return Promise.resolve({ done: false, value: tbl.batches[0] });
+  }
+}
+/* eslint-enable */
+
+/** Query executor */
+export class Query implements AsyncIterable<RecordBatch> {
+  private readonly inner: NativeQuery;
+
+  constructor(tbl: NativeTable) {
+    this.inner = tbl.query();
+  }
+
+  /** Set the column to run query. */
+  column(column: string): Query {
+    this.inner.column(column);
+    return this;
+  }
+
+  /** Set the filter predicate, only returns the results that satisfy the filter.
+   *
+   */
+  filter(predicate: string): Query {
+    this.inner.filter(predicate);
+    return this;
+  }
+
+  /**
+   * Select the columns to return. If not set, all columns are returned.
+   */
+  select(columns: string[]): Query {
+    this.inner.select(columns);
+    return this;
+  }
+
+  /**
+   * Set the limit of rows to return.
+   */
+  limit(limit: number): Query {
+    this.inner.limit(limit);
+    return this;
+  }
+
+  prefilter(prefilter: boolean): Query {
+    this.inner.prefilter(prefilter);
+    return this;
+  }
+
+  /**
+   * Set the query vector.
+   */
+  nearestTo(vector: number[]): Query {
+    this.inner.nearestTo(Float32Array.from(vector));
+    return this;
+  }
+
+  /**
+   * Set the number of IVF partitions to use for the query.
+   */
+  nprobes(nprobes: number): Query {
+    this.inner.nprobes(nprobes);
+    return this;
+  }
+
+  /**
+   * Set the refine factor for the query.
+   */
+  refineFactor(refine_factor: number): Query {
+    this.inner.refineFactor(refine_factor);
+    return this;
+  }
+
+  /**
+   * Execute the query and return the results as an AsyncIterator.
+   */
+  async executeStream(): Promise<RecordBatchIterator> {
+    const inner = await this.inner.executeStream();
+    return new RecordBatchIterator(inner);
+  }
+
+  /** Collect the results as an Arrow Table. */
+  async toArrow(): Promise<ArrowTable> {
+    const batches = [];
+    for await (const batch of this) {
+      batches.push(batch);
+    }
+    return new ArrowTable(batches);
+  }
+
+  /** Returns a JSON Array of All results.
+   *
+   */
+  async toArray(): Promise<any[]> {
+    const tbl = await this.toArrow();
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+    return tbl.toArray();
+  }
+
+  [Symbol.asyncIterator](): AsyncIterator<RecordBatch<any>> {
+    const promise = this.inner.executeStream();
+    return new RecordBatchIterator(undefined, promise);
+  }
+}
--- a/nodejs/lancedb/table.ts
+++ b/nodejs/lancedb/table.ts
@@ -0,0 +1,153 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { Schema, tableFromIPC } from "apache-arrow";
+import { Table as _NativeTable } from "./native";
+import { toBuffer, Data } from "./arrow";
+import { Query } from "./query";
+import { IndexBuilder } from "./indexer";
+
+/**
+ * A LanceDB Table is the collection of Records.
+ *
+ * Each Record has one or more vector fields.
+ */
+export class Table {
+  private readonly inner: _NativeTable;
+
+  /** Construct a Table. Internal use only. */
+  constructor(inner: _NativeTable) {
+    this.inner = inner;
+  }
+
+  /** Get the schema of the table. */
+  async schema(): Promise<Schema> {
+    const schemaBuf = await this.inner.schema();
+    const tbl = tableFromIPC(schemaBuf);
+    return tbl.schema;
+  }
+
+  /**
+   * Insert records into this Table.
+   *
+   * @param {Data} data Records to be inserted into the Table
+   * @return The number of rows added to the table
+   */
+  async add(data: Data): Promise<void> {
+    const buffer = toBuffer(data);
+    await this.inner.add(buffer);
+  }
+
+  /** Count the total number of rows in the dataset. */
+  async countRows(filter?: string): Promise<bigint> {
+    return await this.inner.countRows(filter);
+  }
+
+  /** Delete the rows that satisfy the predicate. */
+  async delete(predicate: string): Promise<void> {
+    await this.inner.delete(predicate);
+  }
+
+  /** Create an index over the columns.
+   *
+   * @param {string} column The column to create the index on. If not specified,
+   *                        it will create an index on vector field.
+   *
+   * @example
+   *
+   * By default, it creates vector idnex on one vector column.
+   *
+   * ```typescript
+   * const table = await conn.openTable("my_table");
+   * await table.createIndex().build();
+   * ```
+   *
+   * You can specify `IVF_PQ` parameters via `ivf_pq({})` call.
+   * ```typescript
+   * const table = await conn.openTable("my_table");
+   * await table.createIndex("my_vec_col")
+   *   .ivf_pq({ num_partitions: 128, num_sub_vectors: 16 })
+   *   .build();
+   * ```
+   *
+   * Or create a Scalar index
+   *
+   * ```typescript
+   * await table.createIndex("my_float_col").build();
+   * ```
+   */
+  createIndex(column?: string): IndexBuilder {
+    let builder = new IndexBuilder(this.inner);
+    if (column !== undefined) {
+      builder = builder.column(column);
+    }
+    return builder;
+  }
+
+  /**
+   * Create a generic {@link Query} Builder.
+   *
+   * When appropriate, various indices and statistics based pruning will be used to
+   * accelerate the query.
+   *
+   * @example
+   *
+   * ### Run a SQL-style query
+   * ```typescript
+   * for await (const batch of table.query()
+   *                          .filter("id > 1").select(["id"]).limit(20)) {
+   *  console.log(batch);
+   * }
+   * ```
+   *
+   * ### Run Top-10 vector similarity search
+   * ```typescript
+   * for await (const batch of table.query()
+   *                    .nearestTo([1, 2, 3])
+   *                    .refineFactor(5).nprobe(10)
+   *                    .limit(10)) {
+   *  console.log(batch);
+   * }
+   *```
+   *
+   * ### Scan the full dataset
+   * ```typescript
+   * for await (const batch of table.query()) {
+   *   console.log(batch);
+   * }
+   *
+   * ### Return the full dataset as Arrow Table
+   * ```typescript
+   * let arrowTbl = await table.query().nearestTo([1.0, 2.0, 0.5, 6.7]).toArrow();
+   * ```
+   *
+   * @returns {@link Query}
+   */
+  query(): Query {
+    return new Query(this.inner);
+  }
+
+  /** Search the table with a given query vector.
+   *
+   * This is a convenience method for preparing an ANN {@link Query}.
+   */
+  search(vector: number[], column?: string): Query {
+    const q = this.query();
+    q.nearestTo(vector);
+    if (column !== undefined) {
+      q.column(column);
+    }
+    return q;
+  }
+}