feat: rework NodeJS SDK using napi (#847)

Use Napi to write a Node.js SDK that follows Polars for better
maintainability, while keeping most of the logic in Rust.

.github/workflows/nodejs.yml

@@ -0,0 +1,114 @@
+name: NodeJS (NAPI)
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - nodejs/**
+      - .github/workflows/nodejs.yml
+      - docker-compose.yml
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  # Disable full debug symbol generation to speed up CI build and keep memory down
+  # "1" means line tables only, which is useful for panic tracebacks.
+  RUSTFLAGS: "-C debuginfo=1"
+  RUST_BACKTRACE: "1"
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+        working-directory: nodejs
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 20
+        cache: 'npm'
+        cache-dependency-path: nodejs/package-lock.json
+    - uses: Swatinem/rust-cache@v2
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y protobuf-compiler libssl-dev
+    - name: Lint
+      run: |
+        cargo fmt --all -- --check
+        cargo clippy --all --all-features -- -D warnings
+        npm ci
+        npm run lint
+  linux:
+    name: Linux (NodeJS ${{ matrix.node-version }})
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        node-version: [ "18", "20" ]
+    runs-on: "ubuntu-22.04"
+    defaults:
+      run:
+        shell: bash
+        working-directory: nodejs
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+        cache: 'npm'
+        cache-dependency-path: node/package-lock.json
+    - uses: Swatinem/rust-cache@v2
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y protobuf-compiler libssl-dev
+        npm install -g @napi-rs/cli
+    - name: Build
+      run: |
+        npm ci
+        npm run build
+    - name: Test
+      run: npm run test
+  macos:
+    timeout-minutes: 30
+    runs-on: "macos-13"
+    defaults:
+      run:
+        shell: bash
+        working-directory: nodejs
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 20
+        cache: 'npm'
+        cache-dependency-path: node/package-lock.json
+    - uses: Swatinem/rust-cache@v2
+    - name: Install dependencies
+      run: |
+        brew install protobuf
+        npm install -g @napi-rs/cli
+    - name: Build
+      run: |
+        npm ci
+        npm run build
+    - name: Test
+      run: |
+        npm run test
+

.gitignore

@@ -29,8 +29,9 @@ python/dist
 node/dist
 node/examples/**/package-lock.json
 node/examples/**/dist
+dist
 
 ## Rust
 target
 
-Cargo.lock
+Cargo.lock

Cargo.toml

@@ -1,9 +1,15 @@
 [workspace]
-members = ["rust/ffi/node", "rust/vectordb"]
+members = ["rust/ffi/node", "rust/vectordb", "nodejs"]
 # Python package needs to be built by maturin.
 exclude = ["python"]
 resolver = "2"
 
+[workspace.package]
+edition = "2021"
+authors = ["Lance Devs <dev@lancedb.com>"]
+license = "Apache-2.0"
+repository = "https://github.com/lancedb/lancedb"
+
 [workspace.dependencies]
 lance = { "version" = "=0.9.7", "features" = ["dynamodb"] }
 lance-index = { "version" = "=0.9.7" }

nodejs/.eslintrc.js

@@ -0,0 +1,22 @@
+module.exports = {
+  env: {
+    browser: true,
+    es2021: true,
+  },
+  extends: [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/recommended-type-checked",
+    "plugin:@typescript-eslint/stylistic-type-checked",
+  ],
+  overrides: [],
+  parserOptions: {
+    project: "./tsconfig.json",
+    ecmaVersion: "latest",
+    sourceType: "module",
+  },
+  rules: {
+    "@typescript-eslint/method-signature-style": "off",
+    "@typescript-eslint/no-explicit-any": "off",
+  },
+  ignorePatterns: ["node_modules/", "dist/", "build/"],
+};

nodejs/.npmignore

@@ -0,0 +1,15 @@
+target
+Cargo.lock
+.cargo
+.github
+npm
+.eslintrc
+.prettierignore
+rustfmt.toml
+yarn.lock
+*.node
+.yarn
+__test__
+renovate.json
+.idea
+src

nodejs/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "vectordb-nodejs"
+edition = "2021"
+version = "0.0.0"
+license.workspace = true
+repository.workspace = true
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+arrow-ipc.workspace = true
+napi = { version = "2.14", default-features = false, features = [
+    "napi7",
+    "async"
+] }
+napi-derive = "2.14"
+vectordb = { path = "../rust/vectordb" }
+lance.workspace = true
+
+[build-dependencies]
+napi-build = "2.1"
+
+[profile.release]
+lto = true
+strip = "symbols"

nodejs/README.md

@@ -0,0 +1,3 @@
+# (New) LanceDB NodeJS SDK
+
+It will replace the NodeJS SDK when it is ready.

nodejs/__test__/arrow.test.ts

@@ -0,0 +1,106 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { makeArrowTable, toBuffer } from "../vectordb/arrow";
+import {
+  Field,
+  FixedSizeList,
+  Float16,
+  Float32,
+  Int32,
+  tableFromIPC,
+  Schema,
+  Float64,
+} from "apache-arrow";
+
+test("customized schema", function () {
+  const schema = new Schema([
+    new Field("a", new Int32(), true),
+    new Field("b", new Float32(), true),
+    new Field(
+      "c",
+      new FixedSizeList(3, new Field("item", new Float16())),
+      true
+    ),
+  ]);
+  const table = makeArrowTable(
+    [
+      { a: 1, b: 2, c: [1, 2, 3] },
+      { a: 4, b: 5, c: [4, 5, 6] },
+      { a: 7, b: 8, c: [7, 8, 9] },
+    ],
+    { schema }
+  );
+
+  expect(table.schema.toString()).toEqual(schema.toString());
+
+  const buf = toBuffer(table);
+  expect(buf.byteLength).toBeGreaterThan(0);
+
+  const actual = tableFromIPC(buf);
+  expect(actual.numRows).toBe(3);
+  const actualSchema = actual.schema;
+  expect(actualSchema.toString()).toStrictEqual(schema.toString());
+});
+
+test("default vector column", function () {
+  const schema = new Schema([
+    new Field("a", new Float64(), true),
+    new Field("b", new Float64(), true),
+    new Field("vector", new FixedSizeList(3, new Field("item", new Float32()))),
+  ]);
+  const table = makeArrowTable([
+    { a: 1, b: 2, vector: [1, 2, 3] },
+    { a: 4, b: 5, vector: [4, 5, 6] },
+    { a: 7, b: 8, vector: [7, 8, 9] },
+  ]);
+
+  const buf = toBuffer(table);
+  expect(buf.byteLength).toBeGreaterThan(0);
+
+  const actual = tableFromIPC(buf);
+  expect(actual.numRows).toBe(3);
+  const actualSchema = actual.schema;
+  expect(actualSchema.toString()).toEqual(actualSchema.toString());
+});
+
+test("2 vector columns", function () {
+  const schema = new Schema([
+    new Field("a", new Float64()),
+    new Field("b", new Float64()),
+    new Field("vec1", new FixedSizeList(3, new Field("item", new Float16()))),
+    new Field("vec2", new FixedSizeList(3, new Field("item", new Float16()))),
+  ]);
+  const table = makeArrowTable(
+    [
+      { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
+      { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
+      { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] },
+    ],
+    {
+      vectorColumns: {
+        vec1: { type: new Float16() },
+        vec2: { type: new Float16() },
+      },
+    }
+  );
+
+  const buf = toBuffer(table);
+  expect(buf.byteLength).toBeGreaterThan(0);
+
+  const actual = tableFromIPC(buf);
+  expect(actual.numRows).toBe(3);
+  const actualSchema = actual.schema;
+  expect(actualSchema.toString()).toEqual(schema.toString());
+});

nodejs/__test__/index.test.ts

@@ -0,0 +1,34 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import * as os from "os";
+import * as path from "path";
+import * as fs from "fs";
+
+import { Schema, Field, Float64 } from "apache-arrow";
+import { connect } from "../dist/index.js";
+
+test("open database", async () => {
+  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "test-open"));
+
+  const db = await connect(tmpDir);
+  let tableNames = await db.tableNames();
+  expect(tableNames).toStrictEqual([]);
+
+  const tbl = await db.createTable("test", [{ id: 1 }, { id: 2 }]);
+  expect(await db.tableNames()).toStrictEqual(["test"]);
+
+  const schema = tbl.schema;
+  expect(schema).toEqual(new Schema([new Field("id", new Float64(), true)]));
+});

nodejs/build.rs

@@ -0,0 +1,5 @@
+extern crate napi_build;
+
+fn main() {
+    napi_build::setup();
+}

nodejs/jest.config.js

@@ -0,0 +1,5 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+};

nodejs/npm/darwin-arm64/README.md

@@ -0,0 +1,3 @@
+# `vectordb-darwin-arm64`
+
+This is the **aarch64-apple-darwin** binary for `vectordb`

nodejs/npm/darwin-arm64/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "vectordb-darwin-arm64",
+  "version": "0.4.3",
+  "os": [
+    "darwin"
+  ],
+  "cpu": [
+    "arm64"
+  ],
+  "main": "vectordb.darwin-arm64.node",
+  "files": [
+    "vectordb.darwin-arm64.node"
+  ],
+  "license": "MIT",
+  "engines": {
+    "node": ">= 18"
+  }
+}

nodejs/npm/darwin-x64/README.md

@@ -0,0 +1,3 @@
+# `vectordb-darwin-x64`
+
+This is the **x86_64-apple-darwin** binary for `vectordb`

nodejs/npm/darwin-x64/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "vectordb-darwin-x64",
+  "version": "0.4.3",
+  "os": [
+    "darwin"
+  ],
+  "cpu": [
+    "x64"
+  ],
+  "main": "vectordb.darwin-x64.node",
+  "files": [
+    "vectordb.darwin-x64.node"
+  ],
+  "license": "MIT",
+  "engines": {
+    "node": ">= 18"
+  }
+}

nodejs/npm/linux-arm64-gnu/README.md

@@ -0,0 +1,3 @@
+# `vectordb-linux-arm64-gnu`
+
+This is the **aarch64-unknown-linux-gnu** binary for `vectordb`

nodejs/npm/linux-arm64-gnu/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "vectordb-linux-arm64-gnu",
+  "version": "0.4.3",
+  "os": [
+    "linux"
+  ],
+  "cpu": [
+    "arm64"
+  ],
+  "main": "vectordb.linux-arm64-gnu.node",
+  "files": [
+    "vectordb.linux-arm64-gnu.node"
+  ],
+  "license": "MIT",
+  "engines": {
+    "node": ">= 10"
+  },
+  "libc": [
+    "glibc"
+  ]
+}

nodejs/npm/linux-x64-gnu/README.md

@@ -0,0 +1,3 @@
+# `vectordb-linux-x64-gnu`
+
+This is the **x86_64-unknown-linux-gnu** binary for `vectordb`

nodejs/npm/linux-x64-gnu/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "vectordb-linux-x64-gnu",
+  "version": "0.4.3",
+  "os": [
+    "linux"
+  ],
+  "cpu": [
+    "x64"
+  ],
+  "main": "vectordb.linux-x64-gnu.node",
+  "files": [
+    "vectordb.linux-x64-gnu.node"
+  ],
+  "license": "MIT",
+  "engines": {
+    "node": ">= 10"
+  },
+  "libc": [
+    "glibc"
+  ]
+}

nodejs/package.json

@@ -0,0 +1,67 @@
+{
+  "name": "vectordb",
+  "version": "0.4.3",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "napi": {
+    "name": "vectordb-nodejs",
+    "triples": {
+      "defaults": false,
+      "additional": [
+        "aarch64-apple-darwin",
+        "aarch64-unknown-linux-gnu",
+        "x86_64-apple-darwin",
+        "x86_64-unknown-linux-gnu"
+      ]
+    }
+  },
+  "license": "Apache 2.0",
+  "devDependencies": {
+    "@napi-rs/cli": "^2.18.0",
+    "@types/jest": "^29.5.11",
+    "@typescript-eslint/eslint-plugin": "^6.19.0",
+    "@typescript-eslint/parser": "^6.19.0",
+    "eslint": "^8.56.0",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.1.2",
+    "typedoc": "^0.25.7",
+    "typedoc-plugin-markdown": "^3.17.1",
+    "typescript": "^5.3.3"
+  },
+  "ava": {
+    "timeout": "3m"
+  },
+  "engines": {
+    "node": ">= 18"
+  },
+  "cpu": [
+    "x64",
+    "arm64"
+  ],
+  "os": [
+    "darwin",
+    "linux",
+    "windows"
+  ],
+  "scripts": {
+    "artifacts": "napi artifacts",
+    "build:native": "napi build --platform --release --js vectordb/native.js --dts vectordb/native.d.ts dist/",
+    "build:debug": "napi build --platform --dts ../vectordb/native.d.ts --js ../vectordb/native.js dist/",
+    "build": "npm run build:debug && tsc -b",
+    "docs": "typedoc --plugin typedoc-plugin-markdown vectordb/index.ts",
+    "lint": "eslint vectordb --ext .js,.ts",
+    "prepublishOnly": "napi prepublish -t npm",
+    "test": "npm run build && jest",
+    "universal": "napi universal",
+    "version": "napi version"
+  },
+  "optionalDependencies": {
+    "vectordb-darwin-arm64": "0.4.3",
+    "vectordb-darwin-x64": "0.4.3",
+    "vectordb-linux-arm64-gnu": "0.4.3",
+    "vectordb-linux-x64-gnu": "0.4.3"
+  },
+  "dependencies": {
+    "apache-arrow": "^15.0.0"
+  }
+}

nodejs/src/connection.rs

@@ -0,0 +1,86 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use napi::bindgen_prelude::*;
+use napi_derive::*;
+
+use crate::table::Table;
+use vectordb::connection::{Connection as LanceDBConnection, Database};
+use vectordb::ipc::ipc_file_to_batches;
+
+#[napi]
+pub struct Connection {
+    conn: Arc<dyn LanceDBConnection>,
+}
+
+#[napi]
+impl Connection {
+    /// Create a new Connection instance from the given URI.
+    #[napi(factory)]
+    pub async fn new(uri: String) -> napi::Result<Self> {
+        Ok(Self {
+            conn: Arc::new(Database::connect(&uri).await.map_err(|e| {
+                napi::Error::from_reason(format!("Failed to connect to database: {}", e))
+            })?),
+        })
+    }
+
+    /// List all tables in the dataset.
+    #[napi]
+    pub async fn table_names(&self) -> napi::Result<Vec<String>> {
+        self.conn
+            .table_names()
+            .await
+            .map_err(|e| napi::Error::from_reason(format!("{}", e)))
+    }
+
+    /// Create table from a Apache Arrow IPC (file) buffer.
+    ///
+    /// Parameters:
+    /// - name: The name of the table.
+    /// - buf: The buffer containing the IPC file.
+    ///
+    #[napi]
+    pub async fn create_table(&self, name: String, buf: Buffer) -> napi::Result<Table> {
+        let batches = ipc_file_to_batches(buf.to_vec())
+            .map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
+        let tbl = self
+            .conn
+            .create_table(&name, Box::new(batches), None)
+            .await
+            .map_err(|e| napi::Error::from_reason(format!("{}", e)))?;
+        Ok(Table::new(tbl))
+    }
+
+    #[napi]
+    pub async fn open_table(&self, name: String) -> napi::Result<Table> {
+        let tbl = self
+            .conn
+            .open_table(&name)
+            .await
+            .map_err(|e| napi::Error::from_reason(format!("{}", e)))?;
+        Ok(Table::new(tbl))
+    }
+
+    /// Drop table with the name. Or raise an error if the table does not exist.
+    #[napi]
+    pub async fn drop_table(&self, name: String) -> napi::Result<()> {
+        self.conn
+            .drop_table(&name)
+            .await
+            .map_err(|e| napi::Error::from_reason(format!("{}", e)))
+    }
+}

nodejs/src/lib.rs

@@ -0,0 +1,46 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use connection::Connection;
+use napi_derive::*;
+
+mod connection;
+mod query;
+mod table;
+
+#[napi(object)]
+pub struct ConnectionOptions {
+    pub uri: String,
+    pub api_key: Option<String>,
+    pub host_override: Option<String>,
+}
+
+/// Write mode for writing a table.
+#[napi(string_enum)]
+pub enum WriteMode {
+    Create,
+    Append,
+    Overwrite,
+}
+
+/// Write options when creating a Table.
+#[napi(object)]
+pub struct WriteOptions {
+    pub mode: Option<WriteMode>,
+}
+
+#[napi]
+pub async fn connect(options: ConnectionOptions) -> napi::Result<Connection> {
+    Connection::new(options.uri.clone()).await
+}

nodejs/src/query.rs

@@ -0,0 +1,48 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use napi::bindgen_prelude::*;
+use napi_derive::napi;
+use vectordb::query::Query as LanceDBQuery;
+
+use crate::table::Table;
+
+#[napi]
+pub struct Query {
+    inner: LanceDBQuery,
+}
+
+#[napi]
+impl Query {
+    pub fn new(table: &Table) -> Self {
+        Self {
+            inner: table.table.query(),
+        }
+    }
+
+    #[napi]
+    pub fn vector(&mut self, vector: Float32Array) {
+        let inn = self.inner.clone().query_vector(&vector);
+        self.inner = inn;
+    }
+
+    #[napi]
+    pub fn to_arrow(&self) -> napi::Result<()> {
+        // let buf = self.inner.to_arrow().map_err(|e| {
+        //     napi::Error::from_reason(format!("Failed to convert query to arrow: {}", e))
+        // })?;
+        // Ok(buf)
+        todo!()
+    }
+}

nodejs/src/table.rs

@@ -0,0 +1,81 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::query::Query;
+use arrow_ipc::writer::FileWriter;
+use napi::bindgen_prelude::*;
+use napi_derive::napi;
+use vectordb::{ipc::ipc_file_to_batches, table::Table as LanceDBTable};
+
+#[napi]
+pub struct Table {
+    pub(crate) table: LanceDBTable,
+}
+
+#[napi]
+impl Table {
+    pub(crate) fn new(table: LanceDBTable) -> Self {
+        Self { table }
+    }
+
+    /// Return Schema as empty Arrow IPC file.
+    #[napi]
+    pub fn schema(&self) -> napi::Result<Buffer> {
+        let mut writer = FileWriter::try_new(vec![], &self.table.schema())
+            .map_err(|e| napi::Error::from_reason(format!("Failed to create IPC file: {}", e)))?;
+        writer
+            .finish()
+            .map_err(|e| napi::Error::from_reason(format!("Failed to finish IPC file: {}", e)))?;
+        Ok(Buffer::from(writer.into_inner().map_err(|e| {
+            napi::Error::from_reason(format!("Failed to get IPC file: {}", e))
+        })?))
+    }
+
+    #[napi]
+    pub async unsafe fn add(&mut self, buf: Buffer) -> napi::Result<()> {
+        let batches = ipc_file_to_batches(buf.to_vec())
+            .map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
+        self.table.add(batches, None).await.map_err(|e| {
+            napi::Error::from_reason(format!(
+                "Failed to add batches to table {}: {}",
+                self.table, e
+            ))
+        })
+    }
+
+    #[napi]
+    pub async fn count_rows(&self) -> napi::Result<usize> {
+        self.table.count_rows().await.map_err(|e| {
+            napi::Error::from_reason(format!(
+                "Failed to count rows in table {}: {}",
+                self.table, e
+            ))
+        })
+    }
+
+    #[napi]
+    pub async unsafe fn delete(&mut self, predicate: String) -> napi::Result<()> {
+        self.table.delete(&predicate).await.map_err(|e| {
+            napi::Error::from_reason(format!(
+                "Failed to delete rows in table {}: predicate={}",
+                self.table, e
+            ))
+        })
+    }
+
+    #[napi]
+    pub fn query(&self) -> Query {
+        Query::new(self)
+    }
+}

nodejs/tsconfig.json

@@ -0,0 +1,31 @@
+{
+  "include": [
+    "vectordb/*.ts",
+    "vectordb/**/*.ts",
+    "vectordb/*.js",
+  ],
+  "compilerOptions": {
+    "target": "es2022",
+    "module": "commonjs",
+    "declaration": true,
+    "outDir": "./dist",
+    "strict": true,
+    "allowJs": true,
+    "resolveJsonModule": true,
+  },
+  "exclude": [
+    "./dist/*",
+  ],
+  "typedocOptions": {
+    "entryPoints": [
+      "vectordb/index.ts"
+    ],
+    "out": "../docs/src/javascript/",
+    "visibilityFilters": {
+      "protected": false,
+      "private": false,
+      "inherited": true,
+      "external": false,
+    }
+  }
+}

nodejs/vectordb/arrow.ts

@@ -0,0 +1,183 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {
+  Field,
+  FixedSizeList,
+  Float,
+  Float32,
+  Schema,
+  Table as ArrowTable,
+  Table,
+  Vector,
+  vectorFromArray,
+  tableToIPC,
+} from "apache-arrow";
+
+/** Data type accepted by NodeJS SDK */
+export type Data = Record<string, unknown>[] | ArrowTable;
+
+export class VectorColumnOptions {
+  /** Vector column type. */
+  type: Float = new Float32();
+
+  constructor(values?: Partial<VectorColumnOptions>) {
+    Object.assign(this, values);
+  }
+}
+
+/** Options to control the makeArrowTable call. */
+export class MakeArrowTableOptions {
+  /** Provided schema. */
+  schema?: Schema;
+
+  /** Vector columns */
+  vectorColumns: Record<string, VectorColumnOptions> = {
+    vector: new VectorColumnOptions(),
+  };
+
+  constructor(values?: Partial<MakeArrowTableOptions>) {
+    Object.assign(this, values);
+  }
+}
+
+/**
+ * An enhanced version of the {@link makeTable} function from Apache Arrow
+ * that supports nested fields and embeddings columns.
+ *
+ * Note that it currently does not support nulls.
+ *
+ * @param data input data
+ * @param options options to control the makeArrowTable call.
+ *
+ * @example
+ *
+ * ```ts
+ *
+ * import { fromTableToBuffer, makeArrowTable } from "../arrow";
+ * import { Field, FixedSizeList, Float16, Float32, Int32, Schema } from "apache-arrow";
+ *
+ * const schema = new Schema([
+ *   new Field("a", new Int32()),
+ *   new Field("b", new Float32()),
+ *   new Field("c", new FixedSizeList(3, new Field("item", new Float16()))),
+ *  ]);
+ *  const table = makeArrowTable([
+ *    { a: 1, b: 2, c: [1, 2, 3] },
+ *    { a: 4, b: 5, c: [4, 5, 6] },
+ *    { a: 7, b: 8, c: [7, 8, 9] },
+ *  ], { schema });
+ * ```
+ *
+ * It guesses the vector columns if the schema is not provided. For example,
+ * by default it assumes that the column named `vector` is a vector column.
+ *
+ * ```ts
+ *
+ * const schema = new Schema([
+ new Field("a", new Float64()),
+ new Field("b", new Float64()),
+ new Field(
+ "vector",
+ new FixedSizeList(3, new Field("item", new Float32()))
+ ),
+ ]);
+ const table = makeArrowTable([
+ { a: 1, b: 2, vector: [1, 2, 3] },
+ { a: 4, b: 5, vector: [4, 5, 6] },
+ { a: 7, b: 8, vector: [7, 8, 9] },
+ ]);
+ assert.deepEqual(table.schema, schema);
+ * ```
+ *
+ * You can specify the vector column types and names using the options as well
+ *
+ * ```typescript
+ *
+ * const schema = new Schema([
+ new Field('a', new Float64()),
+ new Field('b', new Float64()),
+ new Field('vec1', new FixedSizeList(3, new Field('item', new Float16()))),
+ new Field('vec2', new FixedSizeList(3, new Field('item', new Float16())))
+ ]);
+ * const table = makeArrowTable([
+ { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
+ { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
+ { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] }
+ ], {
+ vectorColumns: {
+ vec1: { type: new Float16() },
+ vec2: { type: new Float16() }
+ }
+ }
+ * assert.deepEqual(table.schema, schema)
+ * ```
+ */
+export function makeArrowTable(
+  data: Record<string, any>[],
+  options?: Partial<MakeArrowTableOptions>
+): Table {
+  if (data.length === 0) {
+    throw new Error("At least one record needs to be provided");
+  }
+  const opt = new MakeArrowTableOptions(options ?? {});
+  const columns: Record<string, Vector> = {};
+  // TODO: sample dataset to find missing columns
+  const columnNames = Object.keys(data[0]);
+  for (const colName of columnNames) {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+    const values = data.map((datum) => datum[colName]);
+    let vector: Vector;
+
+    if (opt.schema !== undefined) {
+      // Explicit schema is provided, highest priority
+      vector = vectorFromArray(
+        values,
+        opt.schema?.fields.filter((f) => f.name === colName)[0]?.type
+      );
+    } else {
+      const vectorColumnOptions = opt.vectorColumns[colName];
+      if (vectorColumnOptions !== undefined) {
+        const fslType = new FixedSizeList(
+          (values[0] as any[]).length,
+          new Field("item", vectorColumnOptions.type, false)
+        );
+        vector = vectorFromArray(values, fslType);
+      } else {
+        // Normal case
+        vector = vectorFromArray(values);
+      }
+    }
+    columns[colName] = vector;
+  }
+
+  return new Table(columns);
+}
+
+/**
+ * Convert an Arrow Table to a Buffer.
+ *
+ * @param data Arrow Table
+ * @param schema Arrow Schema, optional
+ * @returns Buffer node
+ */
+export function toBuffer(data: Data, schema?: Schema): Buffer {
+  let tbl: Table;
+  if (data instanceof Table) {
+    tbl = data;
+  } else {
+    tbl = makeArrowTable(data, { schema });
+  }
+  return Buffer.from(tableToIPC(tbl, "file"));
+}

nodejs/vectordb/connection.ts

@@ -0,0 +1,70 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { toBuffer } from "./arrow";
+import { Connection as _NativeConnection } from "./native";
+import { Table } from "./table";
+import { Table as ArrowTable } from "apache-arrow";
+
+/**
+ * A LanceDB Connection that allows you to open tables and create new ones.
+ *
+ * Connection could be local against filesystem or remote against a server.
+ */
+export class Connection {
+  readonly inner: _NativeConnection;
+
+  constructor(inner: _NativeConnection) {
+    this.inner = inner;
+  }
+
+  /** List all the table names in this database. */
+  async tableNames(): Promise<string[]> {
+    return this.inner.tableNames();
+  }
+
+  /**
+   * Open a table in the database.
+   *
+   * @param name The name of the table.
+   * @param embeddings An embedding function to use on this table
+   */
+  async openTable(name: string): Promise<Table> {
+    const innerTable = await this.inner.openTable(name);
+    return new Table(innerTable);
+  }
+
+  /**
+   * Creates a new Table and initialize it with new data.
+   *
+   * @param {string} name - The name of the table.
+   * @param data - Non-empty Array of Records to be inserted into the table
+   */
+  async createTable(
+    name: string,
+    data: Record<string, unknown>[] | ArrowTable
+  ): Promise<Table> {
+    const buf = toBuffer(data);
+    const innerTable = await this.inner.createTable(name, buf);
+    return new Table(innerTable);
+  }
+
+  /**
+   * Drop an existing table.
+   * @param name The name of the table to drop.
+   */
+  async dropTable(name: string): Promise<void> {
+    return this.inner.dropTable(name);
+  }
+}

nodejs/vectordb/index.ts

@@ -0,0 +1,58 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { Connection } from "./connection";
+import { Connection as NativeConnection, ConnectionOptions } from "./native.js";
+
+export { ConnectionOptions, WriteOptions, Query } from "./native.js";
+export { Connection } from "./connection";
+export { Table } from "./table";
+export { Data } from "./arrow";
+
+/**
+ * Connect to a LanceDB instance at the given URI.
+ *
+ * Accpeted formats:
+ *
+ * - `/path/to/database` - local database
+ * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+ * - `db://host:port` - remote database (LanceDB cloud)
+ *
+ * @param uri The uri of the database. If the database uri starts with `db://` then it connects to a remote database.
+ *
+ * @see {@link ConnectionOptions} for more details on the URI format.
+ */
+export async function connect(uri: string): Promise<Connection>;
+export async function connect(
+  opts: Partial<ConnectionOptions>
+): Promise<Connection>;
+export async function connect(
+  args: string | Partial<ConnectionOptions>
+): Promise<Connection> {
+  let opts: ConnectionOptions;
+  if (typeof args === "string") {
+    opts = { uri: args };
+  } else {
+    opts = Object.assign(
+      {
+        uri: "",
+        apiKey: "",
+        hostOverride: "",
+      },
+      args
+    );
+  }
+  const nativeConn = await NativeConnection.new(opts.uri);
+  return new Connection(nativeConn);
+}

nodejs/vectordb/native.d.ts

@@ -0,0 +1,50 @@
+/* eslint-disable */
+
+/* auto-generated by NAPI-RS */
+
+export interface ConnectionOptions {
+  uri: string
+  apiKey?: string
+  hostOverride?: string
+}
+/** Write mode for writing a table. */
+export const enum WriteMode {
+  Create = 'Create',
+  Append = 'Append',
+  Overwrite = 'Overwrite'
+}
+/** Write options when creating a Table. */
+export interface WriteOptions {
+  mode?: WriteMode
+}
+export function connect(options: ConnectionOptions): Promise<Connection>
+export class Connection {
+  /** Create a new Connection instance from the given URI. */
+  static new(uri: string): Promise<Connection>
+  /** List all tables in the dataset. */
+  tableNames(): Promise<Array<string>>
+  /**
+   * Create table from a Apache Arrow IPC (file) buffer.
+   *
+   * Parameters:
+   * - name: The name of the table.
+   * - buf: The buffer containing the IPC file.
+   *
+   */
+  createTable(name: string, buf: Buffer): Promise<Table>
+  openTable(name: string): Promise<Table>
+  /** Drop table with the name. Or raise an error if the table does not exist. */
+  dropTable(name: string): Promise<void>
+}
+export class Query {
+  vector(vector: Float32Array): void
+  toArrow(): void
+}
+export class Table {
+  /** Return Schema as empty Arrow IPC file. */
+  schema(): Buffer
+  add(buf: Buffer): Promise<void>
+  countRows(): Promise<bigint>
+  delete(predicate: string): Promise<void>
+  query(): Query
+}

nodejs/vectordb/native.js

@@ -0,0 +1,303 @@
+/* eslint-disable */
+/* prettier-ignore */
+
+/* auto-generated by NAPI-RS */
+
+const { existsSync, readFileSync } = require('fs')
+const { join } = require('path')
+
+const { platform, arch } = process
+
+let nativeBinding = null
+let localFileExisted = false
+let loadError = null
+
+function isMusl() {
+  // For Node 10
+  if (!process.report || typeof process.report.getReport !== 'function') {
+    try {
+      const lddPath = require('child_process').execSync('which ldd').toString().trim()
+      return readFileSync(lddPath, 'utf8').includes('musl')
+    } catch (e) {
+      return true
+    }
+  } else {
+    const { glibcVersionRuntime } = process.report.getReport().header
+    return !glibcVersionRuntime
+  }
+}
+
+switch (platform) {
+  case 'android':
+    switch (arch) {
+      case 'arm64':
+        localFileExisted = existsSync(join(__dirname, 'vectordb-nodejs.android-arm64.node'))
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./vectordb-nodejs.android-arm64.node')
+          } else {
+            nativeBinding = require('vectordb-android-arm64')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      case 'arm':
+        localFileExisted = existsSync(join(__dirname, 'vectordb-nodejs.android-arm-eabi.node'))
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./vectordb-nodejs.android-arm-eabi.node')
+          } else {
+            nativeBinding = require('vectordb-android-arm-eabi')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      default:
+        throw new Error(`Unsupported architecture on Android ${arch}`)
+    }
+    break
+  case 'win32':
+    switch (arch) {
+      case 'x64':
+        localFileExisted = existsSync(
+          join(__dirname, 'vectordb-nodejs.win32-x64-msvc.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./vectordb-nodejs.win32-x64-msvc.node')
+          } else {
+            nativeBinding = require('vectordb-win32-x64-msvc')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      case 'ia32':
+        localFileExisted = existsSync(
+          join(__dirname, 'vectordb-nodejs.win32-ia32-msvc.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./vectordb-nodejs.win32-ia32-msvc.node')
+          } else {
+            nativeBinding = require('vectordb-win32-ia32-msvc')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      case 'arm64':
+        localFileExisted = existsSync(
+          join(__dirname, 'vectordb-nodejs.win32-arm64-msvc.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./vectordb-nodejs.win32-arm64-msvc.node')
+          } else {
+            nativeBinding = require('vectordb-win32-arm64-msvc')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      default:
+        throw new Error(`Unsupported architecture on Windows: ${arch}`)
+    }
+    break
+  case 'darwin':
+    localFileExisted = existsSync(join(__dirname, 'vectordb-nodejs.darwin-universal.node'))
+    try {
+      if (localFileExisted) {
+        nativeBinding = require('./vectordb-nodejs.darwin-universal.node')
+      } else {
+        nativeBinding = require('vectordb-darwin-universal')
+      }
+      break
+    } catch {}
+    switch (arch) {
+      case 'x64':
+        localFileExisted = existsSync(join(__dirname, 'vectordb-nodejs.darwin-x64.node'))
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./vectordb-nodejs.darwin-x64.node')
+          } else {
+            nativeBinding = require('vectordb-darwin-x64')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      case 'arm64':
+        localFileExisted = existsSync(
+          join(__dirname, 'vectordb-nodejs.darwin-arm64.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./vectordb-nodejs.darwin-arm64.node')
+          } else {
+            nativeBinding = require('vectordb-darwin-arm64')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      default:
+        throw new Error(`Unsupported architecture on macOS: ${arch}`)
+    }
+    break
+  case 'freebsd':
+    if (arch !== 'x64') {
+      throw new Error(`Unsupported architecture on FreeBSD: ${arch}`)
+    }
+    localFileExisted = existsSync(join(__dirname, 'vectordb-nodejs.freebsd-x64.node'))
+    try {
+      if (localFileExisted) {
+        nativeBinding = require('./vectordb-nodejs.freebsd-x64.node')
+      } else {
+        nativeBinding = require('vectordb-freebsd-x64')
+      }
+    } catch (e) {
+      loadError = e
+    }
+    break
+  case 'linux':
+    switch (arch) {
+      case 'x64':
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'vectordb-nodejs.linux-x64-musl.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./vectordb-nodejs.linux-x64-musl.node')
+            } else {
+              nativeBinding = require('vectordb-linux-x64-musl')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'vectordb-nodejs.linux-x64-gnu.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./vectordb-nodejs.linux-x64-gnu.node')
+            } else {
+              nativeBinding = require('vectordb-linux-x64-gnu')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        }
+        break
+      case 'arm64':
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'vectordb-nodejs.linux-arm64-musl.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./vectordb-nodejs.linux-arm64-musl.node')
+            } else {
+              nativeBinding = require('vectordb-linux-arm64-musl')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'vectordb-nodejs.linux-arm64-gnu.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./vectordb-nodejs.linux-arm64-gnu.node')
+            } else {
+              nativeBinding = require('vectordb-linux-arm64-gnu')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        }
+        break
+      case 'arm':
+        localFileExisted = existsSync(
+          join(__dirname, 'vectordb-nodejs.linux-arm-gnueabihf.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./vectordb-nodejs.linux-arm-gnueabihf.node')
+          } else {
+            nativeBinding = require('vectordb-linux-arm-gnueabihf')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      case 'riscv64':
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'vectordb-nodejs.linux-riscv64-musl.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./vectordb-nodejs.linux-riscv64-musl.node')
+            } else {
+              nativeBinding = require('vectordb-linux-riscv64-musl')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'vectordb-nodejs.linux-riscv64-gnu.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./vectordb-nodejs.linux-riscv64-gnu.node')
+            } else {
+              nativeBinding = require('vectordb-linux-riscv64-gnu')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        }
+        break
+      case 's390x':
+        localFileExisted = existsSync(
+          join(__dirname, 'vectordb-nodejs.linux-s390x-gnu.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./vectordb-nodejs.linux-s390x-gnu.node')
+          } else {
+            nativeBinding = require('vectordb-linux-s390x-gnu')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      default:
+        throw new Error(`Unsupported architecture on Linux: ${arch}`)
+    }
+    break
+  default:
+    throw new Error(`Unsupported OS: ${platform}, architecture: ${arch}`)
+}
+
+if (!nativeBinding) {
+  if (loadError) {
+    throw loadError
+  }
+  throw new Error(`Failed to load native binding`)
+}
+
+const { Connection, Query, Table, WriteMode, connect } = nativeBinding
+
+module.exports.Connection = Connection
+module.exports.Query = Query
+module.exports.Table = Table
+module.exports.WriteMode = WriteMode
+module.exports.connect = connect

nodejs/vectordb/query.ts

@@ -0,0 +1,93 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { RecordBatch } from "apache-arrow";
+import { Table } from "./table";
+
+// TODO: re-eanble eslint once we have a real implementation
+/* eslint-disable */
+class RecordBatchIterator implements AsyncIterator<RecordBatch> {
+  next(
+    ...args: [] | [undefined]
+  ): Promise<IteratorResult<RecordBatch<any>, any>> {
+    throw new Error("Method not implemented.");
+  }
+  return?(value?: any): Promise<IteratorResult<RecordBatch<any>, any>> {
+    throw new Error("Method not implemented.");
+  }
+  throw?(e?: any): Promise<IteratorResult<RecordBatch<any>, any>> {
+    throw new Error("Method not implemented.");
+  }
+}
+/* eslint-enable */
+
+/** Query executor */
+export class Query implements AsyncIterable<RecordBatch> {
+  private readonly tbl: Table;
+  private _filter?: string;
+  private _limit?: number;
+
+  // Vector search
+  private _vector?: Float32Array;
+  private _nprobes?: number;
+  private _refine_factor?: number = 1;
+
+  constructor(tbl: Table) {
+    this.tbl = tbl;
+  }
+
+  /** Set the filter predicate, only returns the results that satisfy the filter.
+   *
+   */
+  filter(predicate: string): Query {
+    this._filter = predicate;
+    return this;
+  }
+
+  /**
+   * Set the limit of rows to return.
+   */
+  limit(limit: number): Query {
+    this._limit = limit;
+    return this;
+  }
+
+  /**
+   * Set the query vector.
+   */
+  vector(vector: number[]): Query {
+    this._vector = Float32Array.from(vector);
+    return this;
+  }
+
+  /**
+   * Set the number of probes to use for the query.
+   */
+  nprobes(nprobes: number): Query {
+    this._nprobes = nprobes;
+    return this;
+  }
+
+  /**
+   * Set the refine factor for the query.
+   */
+  refine_factor(refine_factor: number): Query {
+    this._refine_factor = refine_factor;
+    return this;
+  }
+
+  [Symbol.asyncIterator](): AsyncIterator<RecordBatch<any>, any, undefined> {
+    throw new RecordBatchIterator();
+  }
+}

nodejs/vectordb/table.ts

@@ -0,0 +1,68 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { Schema, tableFromIPC } from "apache-arrow";
+import { Table as _NativeTable } from "./native";
+import { toBuffer, Data } from "./arrow";
+import { Query } from "./query";
+
+/**
+ * A LanceDB Table is the collection of Records.
+ *
+ * Each Record has one or more vector fields.
+ */
+export class Table {
+  private readonly inner: _NativeTable;
+
+  /** Construct a Table. Internal use only. */
+  constructor(inner: _NativeTable) {
+    this.inner = inner;
+  }
+
+  /** Get the schema of the table. */
+  get schema(): Schema {
+    const schemaBuf = this.inner.schema();
+    const tbl = tableFromIPC(schemaBuf);
+    return tbl.schema;
+  }
+
+  /**
+   * Insert records into this Table.
+   *
+   * @param {Data} data Records to be inserted into the Table
+   * @return The number of rows added to the table
+   */
+  async add(data: Data): Promise<void> {
+    const buffer = toBuffer(data);
+    await this.inner.add(buffer);
+  }
+
+  /** Count the total number of rows in the dataset. */
+  async countRows(): Promise<bigint> {
+    return await this.inner.countRows();
+  }
+
+  /** Delete the rows that satisfy the predicate. */
+  async delete(predicate: string): Promise<void> {
+    await this.inner.delete(predicate);
+  }
+
+  search(vector?: number[]): Query {
+    const q = new Query(this);
+    if (vector !== undefined) {
+      q.vector(vector);
+    }
+    return q;
+  }
+}

rust/vectordb/Cargo.toml

@@ -16,6 +16,7 @@ arrow-data = { workspace = true }
 arrow-schema = { workspace = true }
 arrow-ord = { workspace = true }
 arrow-cast = { workspace = true }
+arrow-ipc.workspace = true
 chrono = { workspace = true }
 object_store = { workspace = true }
 snafu = { workspace = true }
@@ -37,4 +38,4 @@ serde_json = { version = "1" }
 [dev-dependencies]
 tempfile = "3.5.0"
 rand = { version = "0.8.3", features = ["small_rng"] }
-walkdir = "2"
+walkdir = "2"

rust/vectordb/src/ipc.rs

@@ -0,0 +1,79 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! IPC support
+
+use std::io::Cursor;
+
+use arrow_array::RecordBatchReader;
+use arrow_ipc::reader::FileReader;
+
+use crate::Result;
+
+/// Convert a Arrow IPC file to a batch reader
+pub fn ipc_file_to_batches(buf: Vec<u8>) -> Result<impl RecordBatchReader> {
+    let buf_reader = Cursor::new(buf);
+    let reader = FileReader::try_new(buf_reader, None)?;
+    Ok(reader)
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+    use arrow_array::{Float32Array, Int64Array, RecordBatch};
+    use arrow_ipc::writer::FileWriter;
+    use arrow_schema::{DataType, Field, Schema};
+    use std::sync::Arc;
+
+    fn create_record_batch() -> Result<RecordBatch> {
+        let schema = Schema::new(vec![
+            Field::new("a", DataType::Int64, false),
+            Field::new("b", DataType::Float32, false),
+        ]);
+
+        let a = Int64Array::from(vec![1, 2, 3, 4, 5]);
+        let b = Float32Array::from(vec![1.1, 2.2, 3.3, 4.4, 5.5]);
+
+        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)])?;
+
+        Ok(batch)
+    }
+
+    #[test]
+    fn test_ipc_file_to_batches() -> Result<()> {
+        let batch = create_record_batch()?;
+
+        let mut writer = FileWriter::try_new(vec![], &batch.schema())?;
+        writer.write(&batch)?;
+        writer.finish()?;
+
+        let buf = writer.into_inner().unwrap();
+        let mut reader = ipc_file_to_batches(buf).unwrap();
+        let read_batch = reader.next().unwrap()?;
+
+        assert_eq!(batch.num_columns(), read_batch.num_columns());
+        assert_eq!(batch.num_rows(), read_batch.num_rows());
+
+        for i in 0..batch.num_columns() {
+            let batch_column = batch.column(i);
+            let read_batch_column = read_batch.column(i);
+
+            assert_eq!(batch_column.data_type(), read_batch_column.data_type());
+            assert_eq!(batch_column.len(), read_batch_column.len());
+        }
+
+        Ok(())
+    }
+}

rust/vectordb/src/lib.rs

@@ -136,11 +136,13 @@ pub mod data;
 pub mod error;
 pub mod index;
 pub mod io;
+pub mod ipc;
 pub mod query;
 pub mod table;
 pub mod utils;
 
 pub use connection::Connection;
+pub use error::{Error, Result};
 pub use table::Table;
 
 pub use lance::dataset::WriteMode;

rust/vectordb/src/query.rs

@@ -22,6 +22,7 @@ use lance_linalg::distance::MetricType;
 use crate::error::Result;
 
 /// A builder for nearest neighbor queries for LanceDB.
+#[derive(Clone)]
 pub struct Query {
     pub dataset: Arc<Dataset>,
     pub query_vector: Option<Float32Array>,
@@ -114,8 +115,8 @@ impl Query {
     /// # Arguments
     ///
     /// * `vector` - The vector that will be used for search.
-    pub fn query_vector(mut self, query_vector: Float32Array) -> Query {
+    pub fn query_vector(mut self, vector: &[f32]) -> Query {
-        self.query_vector = Some(query_vector);
+        self.query_vector = Some(Float32Array::from(vector.to_vec()));
         self
     }
 
@@ -211,7 +212,7 @@ mod tests {
         let new_vector = Float32Array::from_iter_values([9.8, 8.7]);
 
         let query = query
-            .query_vector(new_vector.clone())
+            .query_vector(&[9.8, 8.7])
             .limit(100)
             .nprobes(1000)
             .use_index(true)

rust/vectordb/src/table.rs

@@ -204,7 +204,7 @@ impl Table {
     /// # Returns
     ///
     /// * A [Table] object.
-    pub async fn create(
+    pub(crate) async fn create(
         uri: &str,
         name: &str,
         batches: impl RecordBatchReader + Send + 'static,
@@ -310,6 +310,10 @@ impl Table {
         Ok(())
     }
 
+    pub fn query(&self) -> Query {
+        Query::new(self.dataset.clone(), None)
+    }
+
     /// Creates a new Query object that can be executed.
     ///
     /// # Arguments