feat(node): pull node binaries into separate packages (2) (#197)

* Refactors the Node module to load the shared library from a separate package. When a user does `npm install vectordb`, the correct optional dependency is automatically downloaded by npm. * Add scripts and instructions to build Linux and MacOS node artifacts locally. * Add instructions for publishing the npm module and crates. Co-authored-by: Will Jones <willjones127@gmail.com>
2026-01-07 04:12:59 +00:00 · 2023-06-15 16:15:42 -07:00
parent cbb56e25ab
commit 0724d41c4b
11 changed files with 392 additions and 32 deletions
--- a/node/.npmignore
+++ b/node/.npmignore
@@ -0,0 +1,4 @@
+gen_test_data.py
+index.node
+dist/lancedb*.tgz
+vectordb*.tgz
--- a/node/README.md
+++ b/node/README.md
@@ -8,6 +8,10 @@ A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb)
 npm install vectordb
 ```

+This will download the appropriate native library for your platform. We currently
+support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
+yet support Windows or musl-based Linux (such as Alpine Linux).
+
 ## Usage

 ### Basic Example
@@ -24,17 +28,33 @@ The [examples](./examples) folder contains complete examples.

 ## Development

-The LanceDB javascript is built with npm:
+To build everything fresh:
+
+```bash
+npm install
+npm run tsc
+npm run build
+```
+
+Then you should be able to run the tests with:
+
+```bash
+npm test
+```
+
+### Rebuilding Rust library
+
+```bash
+npm run build
+```
+
+### Rebuilding Typescript

 ```bash
 npm run tsc
 ```

-Run the tests with
-
-```bash
-npm test
-```
+### Fix lints

 To run the linter and have it automatically fix all errors

--- a/node/native.js
+++ b/node/native.js
@@ -12,29 +12,26 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+const { currentTarget } = require('@neon-rs/load');
+
 let nativeLib;

-function getPlatformLibrary() {
-    if (process.platform === "darwin" && process.arch == "arm64") {
-        return require('./aarch64-apple-darwin.node');
-    } else if (process.platform === "darwin" && process.arch == "x64") {
-        return require('./x86_64-apple-darwin.node');
-    } else if (process.platform === "linux" && process.arch == "x64") {
-        return require('./x86_64-unknown-linux-gnu.node');
-    } else {
-        throw new Error(`vectordb: unsupported platform ${process.platform}_${process.arch}. Please file a bug report at https://github.com/lancedb/lancedb/issues`)
-    }
-}
-
 try {
-    nativeLib = require('./index.node')
+    nativeLib = require(`vectordb-${currentTarget()}`);
 } catch (e) {
-    if (e.code === "MODULE_NOT_FOUND") {
-        nativeLib = getPlatformLibrary();
-    } else {
-        throw new Error('vectordb: failed to load native library. Please file a bug report at https://github.com/lancedb/lancedb/issues');
-    }
+    try {
+        // Might be developing locally, so try that. But don't expose that error
+        // to the user.
+        nativeLib = require("./index.node");
+    } catch {
+        throw new Error(`vectordb: failed to load native library.
+  You may need to run \`npm install vectordb-${currentTarget()}\`.
+      
+  If that does not work, please file a bug report at https://github.com/lancedb/lancedb/issues
+      
+  Source error: ${e}`);
+    }   
 }

-module.exports = nativeLib
-
+// Dynamic require for runtime.
+module.exports = nativeLib;
--- a/node/package-lock.json
+++ b/node/package-lock.json
@@ -7,12 +7,22 @@
    "": {
      "name": "vectordb",
      "version": "0.1.5",
+      "cpu": [
+        "x64",
+        "arm64"
+      ],
      "license": "Apache-2.0",
+      "os": [
+        "darwin",
+        "linux"
+      ],
      "dependencies": {
        "@apache-arrow/ts": "^12.0.0",
+        "@neon-rs/load": "^0.0.74",
        "apache-arrow": "^12.0.0"
      },
      "devDependencies": {
+        "@neon-rs/cli": "^0.0.74",
        "@types/chai": "^4.3.4",
        "@types/mocha": "^10.0.1",
        "@types/node": "^18.16.2",
@@ -35,6 +45,12 @@
        "typedoc": "^0.24.7",
        "typedoc-plugin-markdown": "^3.15.3",
        "typescript": "*"
+      },
+      "optionalDependencies": {
+        "vectordb-darwin-arm64": "0.1.2",
+        "vectordb-darwin-x64": "0.1.2",
+        "vectordb-linux-arm64-gnu": "0.1.2",
+        "vectordb-linux-x64-gnu": "0.1.2"
      }
    },
    "node_modules/@apache-arrow/ts": {
@@ -202,6 +218,20 @@
        "@jridgewell/sourcemap-codec": "^1.4.10"
      }
    },
+    "node_modules/@neon-rs/cli": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
+      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
+      "dev": true,
+      "bin": {
+        "neon": "index.js"
+      }
+    },
+    "node_modules/@neon-rs/load": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
+      "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
+    },
    "node_modules/@nodelib/fs.scandir": {
      "version": "2.1.5",
      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -4601,6 +4631,17 @@
        "@jridgewell/sourcemap-codec": "^1.4.10"
      }
    },
+    "@neon-rs/cli": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
+      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
+      "dev": true
+    },
+    "@neon-rs/load": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
+      "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
+    },
    "@nodelib/fs.scandir": {
      "version": "2.1.5",
      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
--- a/node/package.json
+++ b/node/package.json
@@ -6,11 +6,14 @@
  "types": "dist/index.d.ts",
  "scripts": {
    "tsc": "tsc -b",
-    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json-render-diagnostics",
+    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json",
    "build-release": "npm run build -- --release",
+    "cross-release": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cross build --message-format=json --release -p vectordb-node",
    "test": "mocha -recursive dist/test",
    "lint": "eslint src --ext .js,.ts",
-    "clean": "rm -rf node_modules *.node dist/"
+    "clean": "rm -rf node_modules *.node dist/",
+    "pack-build": "neon pack-build",
+    "check-npm": "printenv && which node && which npm && npm --version"
  },
  "repository": {
    "type": "git",
@@ -25,6 +28,7 @@
  "author": "Lance Devs",
  "license": "Apache-2.0",
  "devDependencies": {
+    "@neon-rs/cli": "^0.0.74",
    "@types/chai": "^4.3.4",
    "@types/mocha": "^10.0.1",
    "@types/node": "^18.16.2",
@@ -50,6 +54,29 @@
  },
  "dependencies": {
    "@apache-arrow/ts": "^12.0.0",
+    "@neon-rs/load": "^0.0.74",
    "apache-arrow": "^12.0.0"
+  },
+  "os": [
+    "darwin",
+    "linux"
+  ],
+  "cpu": [
+    "x64",
+    "arm64"
+  ],
+  "neon": {
+    "targets": {
+      "x86_64-apple-darwin": "vectordb-darwin-x64",
+      "aarch64-apple-darwin": "vectordb-darwin-arm64",
+      "x86_64-unknown-linux-gnu": "vectordb-linux-x64-gnu",
+      "aarch64-unknown-linux-gnu": "vectordb-linux-arm64-gnu"
+    }
+  },
+  "optionalDependencies": {
+    "vectordb-darwin-arm64": "0.1.6",
+    "vectordb-darwin-x64": "0.1.6",
+    "vectordb-linux-x64-gnu": "0.1.6",
+    "vectordb-linux-arm64-gnu": "0.1.6"
  }
 }