refactor: pull node binaries into separate packages (#88)

Changes: * Refactors the Node module to load the shared library from a separate package. When a user does `npm install vectordb`, the correct optional dependency is automatically downloaded by npm. * Brings Rust and Node versions in alignment at 0.1.2. * Add scripts and instructions to build Linux and MacOS node artifacts locally. * Add instructions for publishing the npm module and crates.
2026-01-05 19:32:56 +00:00 · 2023-06-01 09:17:19 -07:00
parent 6d8cf52e01
commit e50b642d80
16 changed files with 707 additions and 37 deletions
--- a/node/.npmignore
+++ b/node/.npmignore
@@ -0,0 +1,4 @@
+gen_test_data.py
+index.node
+dist/lancedb*.tgz
+vectordb*.tgz
--- a/node/README.md
+++ b/node/README.md
@@ -8,6 +8,10 @@ A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb)
 npm install vectordb
 ```

+This will download the appropriate native library for your platform. We currently
+support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
+yet support Windows or musl-based Linux (such as Alpine Linux).
+
 ## Usage

 ### Basic Example
@@ -24,17 +28,33 @@ The [examples](./examples) folder contains complete examples.

 ## Development

-The LanceDB javascript is built with npm:
+To build everything fresh:
+
+```bash
+npm install
+npm run tsc
+npm run build
+```
+
+Then you should be able to run the tests with:
+
+```bash
+npm test
+```
+
+### Rebuilding Rust library
+
+```bash
+npm run build
+```
+
+### Rebuilding Typescript

 ```bash
 npm run tsc
 ```

-Run the tests with
-
-```bash
-npm test
-```
+### Fix lints

 To run the linter and have it automatically fix all errors

--- a/node/native.js
+++ b/node/native.js
@@ -12,29 +12,26 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+const { currentTarget } = require('@neon-rs/load');
+
 let nativeLib;

-function getPlatformLibrary() {
-    if (process.platform === "darwin" && process.arch == "arm64") {
-        return require('./aarch64-apple-darwin.node');
-    } else if (process.platform === "darwin" && process.arch == "x64") {
-        return require('./x86_64-apple-darwin.node');
-    } else if (process.platform === "linux" && process.arch == "x64") {
-        return require('./x86_64-unknown-linux-gnu.node');
-    } else {
-        throw new Error(`vectordb: unsupported platform ${process.platform}_${process.arch}. Please file a bug report at https://github.com/lancedb/lancedb/issues`)
-    }
-}
-
 try {
-    nativeLib = require('./index.node')
+    nativeLib = require(`@lancedb/vectordb-${currentTarget()}`);
 } catch (e) {
-    if (e.code === "MODULE_NOT_FOUND") {
-        nativeLib = getPlatformLibrary();
-    } else {
-        throw new Error('vectordb: failed to load native library. Please file a bug report at https://github.com/lancedb/lancedb/issues');
-    }
+    try {
+        // Might be developing locally, so try that. But don't expose that error
+        // to the user.
+        nativeLib = require("./index.node");
+    } catch {
+        throw new Error(`vectordb: failed to load native library.
+  You may need to run \`npm install @lancedb/vectordb-${currentTarget()}\`.
+      
+  If that does not work, please file a bug report at https://github.com/lancedb/lancedb/issues
+      
+  Source error: ${e}`);
+    }   
 }

-module.exports = nativeLib
-
+// Dynamic require for runtime.
+module.exports = nativeLib;
--- a/node/package-lock.json
+++ b/node/package-lock.json
@@ -1,18 +1,28 @@
 {
  "name": "vectordb",
-  "version": "0.1.1",
+  "version": "0.1.2",
  "lockfileVersion": 2,
  "requires": true,
  "packages": {
    "": {
      "name": "vectordb",
-      "version": "0.1.1",
+      "version": "0.1.2",
+      "cpu": [
+        "x64",
+        "arm64"
+      ],
      "license": "Apache-2.0",
+      "os": [
+        "darwin",
+        "linux"
+      ],
      "dependencies": {
        "@apache-arrow/ts": "^12.0.0",
+        "@neon-rs/load": "^0.0.74",
        "apache-arrow": "^12.0.0"
      },
      "devDependencies": {
+        "@neon-rs/cli": "^0.0.74",
        "@types/chai": "^4.3.4",
        "@types/mocha": "^10.0.1",
        "@types/node": "^18.16.2",
@@ -30,6 +40,12 @@
        "ts-node": "^10.9.1",
        "ts-node-dev": "^2.0.0",
        "typescript": "*"
+      },
+      "optionalDependencies": {
+        "@lancedb/vectordb-darwin-arm64": "0.1.2",
+        "@lancedb/vectordb-darwin-x64": "0.1.2",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.1.2",
+        "@lancedb/vectordb-linux-x64-gnu": "0.1.2"
      }
    },
    "node_modules/@apache-arrow/ts": {
@@ -197,6 +213,46 @@
        "@jridgewell/sourcemap-codec": "^1.4.10"
      }
    },
+    "node_modules/@lancedb/vectordb-darwin-arm64": {
+      "version": "0.1.2",
+      "resolved": "https://npm.pkg.github.com/download/@lancedb/vectordb-darwin-arm64/0.1.2/84d71331e03e8aaeb9fb12cdacc759dc82cfd3b0",
+      "integrity": "sha512-DU6tHmmn/coSj5r5FGwTMXMQfsSSxQN1ozOl9mFUXr0aVtlx5nlA8ZY5BAF/V371yL5QzNPKtaNpogP6iw51NA==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
+      "version": "0.1.2",
+      "resolved": "https://npm.pkg.github.com/download/@lancedb/vectordb-linux-arm64-gnu/0.1.2/d5a9d66c3969494cf3546195fb5511f9f49aa295",
+      "integrity": "sha512-LZZ4KgoGqD5AzKX/utBrsxrwXq6whpUNa02tWxl/ND/601ruNi9ZUaXCTb1rSVUWJkgMR2wASk15kssyaPRSjw==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@neon-rs/cli": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
+      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
+      "dev": true,
+      "bin": {
+        "neon": "index.js"
+      }
+    },
+    "node_modules/@neon-rs/load": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
+      "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
+    },
    "node_modules/@nodelib/fs.scandir": {
      "version": "2.1.5",
      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -4191,6 +4247,29 @@
        "@jridgewell/sourcemap-codec": "^1.4.10"
      }
    },
+    "@lancedb/vectordb-darwin-arm64": {
+      "version": "0.1.2",
+      "resolved": "https://npm.pkg.github.com/download/@lancedb/vectordb-darwin-arm64/0.1.2/84d71331e03e8aaeb9fb12cdacc759dc82cfd3b0",
+      "integrity": "sha512-DU6tHmmn/coSj5r5FGwTMXMQfsSSxQN1ozOl9mFUXr0aVtlx5nlA8ZY5BAF/V371yL5QzNPKtaNpogP6iw51NA==",
+      "optional": true
+    },
+    "@lancedb/vectordb-linux-arm64-gnu": {
+      "version": "0.1.2",
+      "resolved": "https://npm.pkg.github.com/download/@lancedb/vectordb-linux-arm64-gnu/0.1.2/d5a9d66c3969494cf3546195fb5511f9f49aa295",
+      "integrity": "sha512-LZZ4KgoGqD5AzKX/utBrsxrwXq6whpUNa02tWxl/ND/601ruNi9ZUaXCTb1rSVUWJkgMR2wASk15kssyaPRSjw==",
+      "optional": true
+    },
+    "@neon-rs/cli": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
+      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
+      "dev": true
+    },
+    "@neon-rs/load": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
+      "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
+    },
    "@nodelib/fs.scandir": {
      "version": "2.1.5",
      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
--- a/node/package.json
+++ b/node/package.json
@@ -1,15 +1,18 @@
 {
  "name": "vectordb",
-  "version": "0.1.1",
+  "version": "0.1.2",
  "description": " Serverless, low-latency vector database for AI applications",
  "main": "dist/index.js",
  "types": "dist/index.d.ts",
  "scripts": {
    "tsc": "tsc -b",
-    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json-render-diagnostics",
+    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json",
    "build-release": "npm run build -- --release",
+    "cross-release": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cross build --message-format=json --release -p vectordb-node",
    "test": "mocha -recursive dist/test",
-    "lint": "eslint src --ext .js,.ts"
+    "lint": "eslint src --ext .js,.ts",
+    "pack-build": "neon pack-build",
+    "check-npm": "printenv && which node && which npm && npm --version"
  },
  "repository": {
    "type": "git",
@@ -24,6 +27,7 @@
  "author": "Lance Devs",
  "license": "Apache-2.0",
  "devDependencies": {
+    "@neon-rs/cli": "^0.0.74",
    "@types/chai": "^4.3.4",
    "@types/mocha": "^10.0.1",
    "@types/node": "^18.16.2",
@@ -44,6 +48,29 @@
  },
  "dependencies": {
    "@apache-arrow/ts": "^12.0.0",
+    "@neon-rs/load": "^0.0.74",
    "apache-arrow": "^12.0.0"
+  },
+  "os": [
+    "darwin",
+    "linux"
+  ],
+  "cpu": [
+    "x64",
+    "arm64"
+  ],
+  "neon": {
+    "targets": {
+      "x86_64-apple-darwin": "@lancedb/vectordb-darwin-x64",
+      "aarch64-apple-darwin": "@lancedb/vectordb-darwin-arm64",
+      "x86_64-unknown-linux-gnu": "@lancedb/vectordb-linux-x64-gnu",
+      "aarch64-unknown-linux-gnu": "@lancedb/vectordb-linux-arm64-gnu"
+    }
+  },
+  "optionalDependencies": {
+    "@lancedb/vectordb-darwin-arm64": "0.1.2",
+    "@lancedb/vectordb-darwin-x64": "0.1.2",
+    "@lancedb/vectordb-linux-x64-gnu": "0.1.2",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.1.2"
  }
 }