more progress on release workflow

2026-01-04 19:02:58 +00:00 · 2023-05-23 16:40:02 -07:00
parent 5e098f4fe5
commit 6c03662c68
15 changed files with 406 additions and 144 deletions
--- a/.github/workflows/build_node_module/action.yml
+++ b/.github/workflows/build_node_module/action.yml
--- a/.github/workflows/node.yml
+++ b/.github/workflows/node.yml
@@ -67,8 +67,10 @@ jobs:
    - name: Build
      run: |
        npm ci
-        npm run build
        npm run tsc
+        npm run build
+        npm run pack-build
+        npm install --no-save ./dist/vectordb-*.tgz
    - name: Test
      run: npm run test
  macos:
@@ -94,8 +96,10 @@ jobs:
    - name: Build
      run: |
        npm ci
-        npm run build
        npm run tsc
+        npm run build
+        npm run pack-build
+        npm install --no-save ./dist/vectordb-*.tgz
    - name: Test
      run: |
        npm run test
--- a/.github/workflows/node_release.yml
+++ b/.github/workflows/node_release.yml
@@ -1,118 +0,0 @@
-name: Node Release
-
-on:
-  pull_request:
-    paths:
-      - node/**
-      - .github/workflows/node_release.yml
-
-jobs:
-  macos:
-    runs-on: macos-12
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Install system dependencies
-      run: brew install protobuf
-    - name: Install rustup target
-      run: rustup target add aarch64-apple-darwin
-    - name: Build x86_64
-      run: cargo build --lib -p vectordb-node --release --target x86_64-apple-darwin
-    - name: Upload x86_64 dylib
-      uses: actions/upload-artifact@v3
-      with:
-        path: target/x86_64-apple-darwin/release/libvectordb_node.dylib
-        name: x86_64-apple-darwin.node
-        retention-days: 5
-    - name: Build aarch64
-      run: cargo build --lib -p vectordb-node --release --target aarch64-apple-darwin
-    - name: Upload aarch64 dylib
-      uses: actions/upload-artifact@v3
-      with:
-        path: target/aarch64-apple-darwin/release/libvectordb_node.dylib
-        name: aarch64-apple-darwin.node
-        retention-days: 5
-
-  linux-x64:
-    runs-on: ubuntu-latest
-    container: quay.io/pypa/manylinux2014_x86_64
-    steps:
-    - uses: actions/checkout@v3
-    - name: Install system dependencies
-      run: |
-        yum install -y openssl-devel
-        # protobuf is too old, so we directly download binaries
-        PB_REL="https://github.com/protocolbuffers/protobuf/releases"
-        PB_VERSION=23.1
-        curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip
-        unzip protoc-$PB_VERSION-linux-x86_64.zip -d $HOME/.local
-        echo "${HOME}/.local/bin" >> $GITHUB_PATH
-    # TODO: replace this with something more modern
-    - uses: actions-rs/toolchain@v1
-      with:
-        profile: minimal
-        toolchain: stable
-        override: true
-    - name: Build GNU x86_64
-      run: cargo build --lib -p vectordb-node --release --verbose
-    - name: Upload GNU x86_64 dylib
-      uses: actions/upload-artifact@v3
-      with:
-        path: target/x86_64-unknown-linux-gnu/release/libvectordb_node.so
-        name: x86_64-unknown-linux-gnu.node
-        retention-days: 5
-  
-  draft-release:
-    runs-on: ubuntu-latest
-    needs:
-      - linux-x64
-      - macos
-    steps:
-    - uses: actions/checkout@v3
-    - uses: actions/download-artifact@v3
-    - name: Move libraries
-      run: mv *.node node/
-    # TODO: make sure we aren't rebuilding the library here.
-    - name: Build module
-      run: |
-        npm install
-        npm run build
-      working-directory: node
-    - name: Test module
-      run: |
-        npm run tsc
-        npm test
-      working-directory: node
-    - name: Zip module
-      run: |
-        rm -rf node/node_modules
-        zip -r vectordb-node.zip node
-    - name: Upload
-      uses: actions/upload-artifact@v3
-      with:
-        path: vectordb-node.zip
-        retention-days: 5
-    # - name: Create draft release
-    #   uses: softprops/action-gh-release@v1
-    #   if: startsWith(github.ref, 'refs/tags/')
-    #   with:
-    #     draft: true
-    #     generate_release_notes: true
-    #     files:
-    #       vectordb-node-$GITHUB_REF-node-v3-linux-x64.zip # x86_64-unknown-linux-gnu.node
-    #       vectordb-node-$GITHUB_REF-node-v3-darwin-x64.zip # x86_64-apple-darwin.node
-    #       vectordb-node-$GITHUB_REF-node-v3-linux-x64.zip # aarch64-apple-darwin.node
-
-    # TODO: create draft release
-    # use: https://github.com/softprops/action-gh-release
-    # format like: https://github.com/tungv/cypher.js/releases/tag/v1.0.0-beta.6
-    # 
-
-    # test-linux-x64:
-    #   runs-on: ubuntu-latest
-    #   needs:
-    #     - draft-release
-    #   steps:
-    #   - 
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -0,0 +1,123 @@
+name: Prepare Release
+
+# Based on https://github.com/dherman/neon-prebuild-example/blob/eaa4d33d682e5eb7abbc3da7aed153a1b1acb1b3/.github/workflows/publish.yml
+
+on:
+  push:
+    tags:
+      - v*
+
+# TODO:
+# Create draft relase
+# Build and upload artifacts
+#  * Build wheel and sdist
+#  * Build npm module
+#  * Build native node modules
+# Test release
+# Publish release
+
+env:
+  NODE_VERSION: 18.x
+  NPM_REGISTRY: 'https://registry.npmjs.org'
+
+jobs:
+  draft-release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          prerelease: true # hardcoded on for now
+          generate_release_notes: true
+  
+  python:
+    runs-on: ubuntu-latest
+    needs: draft-release
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.10
+      - name: Build wheel
+        run: python setup.py sdist bdist_wheel
+      - uses: softprops/action-gh-release@v1
+        with:
+          files: |
+            python/dist/lancedb-*.tar.gz 
+            python/dist/lancedb-*.whl
+
+  node:
+    runs-on: ubuntu-latest
+    needs: draft-release
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          cache: 'npm'
+          cache-dependency-path: node/package-lock.json
+      - uses: Swatinem/rust-cache@v2
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Build
+        run: |
+          npm ci
+          npm run tsc
+      - uses: softprops/action-gh-release@v1
+        with:
+          files: node/vectordb-*.tgz
+
+  node-macos:
+    runs-on: macos-12
+    needs: draft-release
+    strategy:
+      matrix:
+        target: [x86_64-apple-darwin, aarch64-apple-darwin]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Install system dependencies
+      run: brew install protobuf
+    - name: Build MacOS native node modules
+      run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
+    - uses: softprops/action-gh-release@v1
+      with:
+        files: node/dist/vectordb-darwin*.tgz
+
+  # node-linux:
+  #   runs-on: ubuntu-latest
+  #   needs: draft-release
+  #   strategy:
+  #     matrix:
+  #       target:
+  #         - x86_64-unknown-linux-gnu:centos
+  #         - aarch64-unknown-linux-gnu:centos
+  #         - aarch64-unknown-linux-musl
+  #         - x86_64-unknown-linux-musl
+  #   steps:
+  #   - name: Install system dependencies
+  #     run: |
+  #       yum install -y openssl-devel
+  #       # protobuf is too old, so we directly download binaries
+  #       PB_REL="https://github.com/protocolbuffers/protobuf/releases"
+  #       PB_VERSION=23.1
+  #       curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip
+  #       unzip protoc-$PB_VERSION-linux-x86_64.zip -d $HOME/.local
+  #       echo "${HOME}/.local/bin" >> $GITHUB_PATH
+
+  
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,8 @@
 **/__pycache__
 .DS_Store

+.vscode
+
 rust/target
 rust/Cargo.lock

--- a/Cross.toml
+++ b/Cross.toml
@@ -0,0 +1,14 @@
+[target.x86_64-unknown-linux-gnu]
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH protobuf-compiler",
+]
+image = "ghcr.io/cross-rs/x86_64-unknown-linux-gnu:main-centos"
+
+[target.aarch64-unknown-linux-gnu]
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH protobuf-compiler",
+]
+image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:centos"
+# docker pull ghcr.io/cross-rs/aarch64-unknown-linux-gnu:edge-centos@sha256:8ca737af41df2a8078eb03a0a19e38405854ad05891a7381f7f671727a9ff736
--- a/ci/build_linux_artifacts.sh
+++ b/ci/build_linux_artifacts.sh
@@ -0,0 +1,40 @@
+# Builds the Linux artifacts (node binaries).
+# Usage: ./build_linux_artifacts.sh [target]
+# Targets supported: 
+# - x86_64-unknown-linux-gnu:centos
+# - aarch64-unknown-linux-gnu:centos
+# - aarch64-unknown-linux-musl
+# - x86_64-unknown-linux-musl
+
+# On MacOS, need to run in a linux container:
+# docker run -v $(pwd):/io -w /io 
+
+# Must run rustup toolchain install stable-x86_64-unknown-linux-gnu --force-non-host
+
+set -e
+
+build_node_binaries() {
+    pushd node
+    
+    for target in $1
+    do
+        echo "Building node library for $target"
+        # cross doesn't yet pass this down to Docker, so we do it ourselves.
+        if [[ $target == x86_64* ]]; then
+            export CROSS_CONTAINER_OPTS="--platform linux/amd64"
+        else
+            export CROSS_CONTAINER_OPTS="--platform linux/arm64/v8"
+        fi
+        npm run cross-release -- --target $target
+        npm run pack-build -- --target $target
+    done
+    popd
+}
+
+if [ -n "$1" ]; then
+    targets=$1
+else
+    # targets="x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu aarch64-unknown-linux-musl x86_64-unknown-linux-musl"
+    targets="aarch64-unknown-linux-gnu"
+fi
+build_node_binaries $targets
--- a/ci/build_macos_artifacts.sh
+++ b/ci/build_macos_artifacts.sh
@@ -0,0 +1,22 @@
+# Builds the macOS artifacts (node binaries).
+# Usage: ./build_macos_artifacts.sh [target]
+# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
+
+build_node_binaries() {
+    pushd node
+    
+    for target in $1
+    do
+        echo "Building node library for $target"
+        npm run build-release -- --target $target
+        npm run pack-build -- --target $target
+    done
+    popd
+}
+
+if [ -n "$1" ]; then
+    targets=$1
+else
+    targets="x86_64-apple-darwin aarch64-apple-darwin"
+fi
+build_node_binaries $targets
--- a/ci/ubuntu_build.dockerfile
+++ b/ci/ubuntu_build.dockerfile
@@ -0,0 +1,31 @@
+# On MacOS, need to run in a linux container:
+# cat ci/ubuntu_build.dockerfile | docker build -t lancedb-node-build -
+# docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd):/io -w /io lancedb-node-build bash ci/build_linux_artifacts.sh
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/Moscow
+
+RUN apt update && apt install -y protobuf-compiler libssl-dev build-essential curl \
+    software-properties-common npm docker.io
+
+# Install rust
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install cross
+# https://github.com/cross-rs/cross/issues/1257#issuecomment-1544553706
+RUN cargo install cross --git https://github.com/cross-rs/cross
+
+# Install additional build targets
+RUN rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu aarch64-unknown-linux-musl x86_64-unknown-linux-musl
+
+# Install node
+RUN npm install npm@latest -g && \
+    npm install n -g && \
+    n latest
+
+# set CROSS_CONTAINER_IN_CONTAINER to inform `cross` that it is executed from within a container
+ENV CROSS_CONTAINER_IN_CONTAINER=true
+ENV CROSS_CONTAINER_ENGINE_NO_BUILDKIT=1
--- a/node/.npmignore
+++ b/node/.npmignore
@@ -0,0 +1,2 @@
+gen_test_data.py
+index.node
--- a/node/README.md
+++ b/node/README.md
@@ -27,10 +27,12 @@ The [examples](./examples) folder contains complete examples.

 ## Development

-Build the rust library with:
+Build and install the rust library with:

 ```bash
 npm run build
+npm run pack-build
+npm install --no-save ./dist/vectordb-*.tgz
 ```

 The LanceDB javascript is built with npm:
--- a/node/native.js
+++ b/node/native.js
@@ -12,29 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+const { currentTarget } = require('@neon-rs/load');
+
 let nativeLib;

-function getPlatformLibrary() {
-    if (process.platform === "darwin" && process.arch == "arm64") {
-        return require('./aarch64-apple-darwin.node');
-    } else if (process.platform === "darwin" && process.arch == "x64") {
-        return require('./x86_64-apple-darwin.node');
-    } else if (process.platform === "linux" && process.arch == "x64") {
-        return require('./x86_64-unknown-linux-gnu.node');
-    } else {
-        throw new Error(`vectordb: unsupported platform ${process.platform}_${process.arch}. Please file a bug report at https://github.com/lancedb/lancedb/issues`)
-    }
-}
-
 try {
-    nativeLib = require('./index.node')
-} catch (e) {
-    if (e.code === "MODULE_NOT_FOUND") {
-        nativeLib = getPlatformLibrary();
-    } else {
-        throw new Error('vectordb: failed to load native library. Please file a bug report at https://github.com/lancedb/lancedb/issues');
-    }
+    nativeLib = require(`@vectordb/${currentTarget()}`);
+} catch {
+    throw new Error('vectordb: failed to load native library. Please file a bug report at https://github.com/lancedb/lancedb/issues');
 }

-module.exports = nativeLib
-
+// Dynamic require for runtime.
+module.exports = nativeLib;
--- a/node/package-lock.json
+++ b/node/package-lock.json
@@ -7,12 +7,26 @@
    "": {
      "name": "vectordb",
      "version": "0.1.1",
+      "cpu": [
+        "x64",
+        "arm64"
+      ],
      "license": "Apache-2.0",
+      "os": [
+        "darwin",
+        "linux"
+      ],
      "dependencies": {
        "@apache-arrow/ts": "^12.0.0",
+        "@neon-rs/load": "^0.0.74",
+        "@vectordb/darwin-arm64": "0.1.1",
+        "@vectordb/darwin-x64": "0.1.1",
+        "@vectordb/linux-x64-gnu": "0.1.1",
+        "@vectordb/linux-x64-musl": "0.1.1",
        "apache-arrow": "^12.0.0"
      },
      "devDependencies": {
+        "@neon-rs/cli": "^0.0.74",
        "@types/chai": "^4.3.4",
        "@types/mocha": "^10.0.1",
        "@types/node": "^18.16.2",
@@ -30,6 +44,12 @@
        "ts-node": "^10.9.1",
        "ts-node-dev": "^2.0.0",
        "typescript": "*"
+      },
+      "optionalDependencies": {
+        "@vectordb/darwin-arm64": "0.1.1",
+        "@vectordb/darwin-x64": "0.1.1",
+        "@vectordb/linux-x64-gnu": "0.1.1",
+        "@vectordb/linux-x64-musl": "0.1.1"
      }
    },
    "node_modules/@apache-arrow/ts": {
@@ -197,6 +217,20 @@
        "@jridgewell/sourcemap-codec": "^1.4.10"
      }
    },
+    "node_modules/@neon-rs/cli": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
+      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
+      "dev": true,
+      "bin": {
+        "neon": "index.js"
+      }
+    },
+    "node_modules/@neon-rs/load": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
+      "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
+    },
    "node_modules/@nodelib/fs.scandir": {
      "version": "2.1.5",
      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -4191,6 +4225,17 @@
        "@jridgewell/sourcemap-codec": "^1.4.10"
      }
    },
+    "@neon-rs/cli": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
+      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
+      "dev": true
+    },
+    "@neon-rs/load": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
+      "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
+    },
    "@nodelib/fs.scandir": {
      "version": "2.1.5",
      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
--- a/node/package.json
+++ b/node/package.json
@@ -6,10 +6,12 @@
  "types": "dist/index.d.ts",
  "scripts": {
    "tsc": "tsc -b",
-    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json-render-diagnostics",
+    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json",
    "build-release": "npm run build -- --release",
+    "cross-release": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cross build --message-format=json --release -p vectordb-node",
    "test": "mocha -recursive dist/test",
-    "lint": "eslint src --ext .js,.ts"
+    "lint": "eslint src --ext .js,.ts",
+    "pack-build": "neon pack-build"
  },
  "repository": {
    "type": "git",
@@ -24,6 +26,7 @@
  "author": "Lance Devs",
  "license": "Apache-2.0",
  "devDependencies": {
+    "@neon-rs/cli": "^0.0.74",
    "@types/chai": "^4.3.4",
    "@types/mocha": "^10.0.1",
    "@types/node": "^18.16.2",
@@ -44,6 +47,29 @@
  },
  "dependencies": {
    "@apache-arrow/ts": "^12.0.0",
+    "@neon-rs/load": "^0.0.74",
    "apache-arrow": "^12.0.0"
+  },
+  "os": [
+    "darwin",
+    "linux"
+  ],
+  "cpu": [
+    "x64",
+    "arm64"
+  ],
+  "neon": {
+    "targets": {
+      "x86_64-apple-darwin": "@vectordb/darwin-x64",
+      "aarch64-apple-darwin": "@vectordb/darwin-arm64",
+      "x86_64-unknown-linux-gnu": "@vectordb/linux-x64-gnu",
+      "x86_64-unknown-linux-musl": "@vectordb/linux-x64-musl"
+    }
+  },
+  "optionalDependencies": {
+    "@vectordb/darwin-arm64": "0.1.1",
+    "@vectordb/darwin-x64": "0.1.1",
+    "@vectordb/linux-x64-gnu": "0.1.1",
+    "@vectordb/linux-x64-musl": "0.1.1"
  }
 }
--- a/rust/ffi/node/release_process.md
+++ b/rust/ffi/node/release_process.md
@@ -0,0 +1,83 @@
+
+How to release the node module
+
+### 1. Bump the versions
+
+```shell
+pushd rust/vectordb
+cargo bump minor
+popd
+
+pushd rust/ffi/node
+cargo bump minor
+popd
+
+pushd python
+cargo bump minor
+popd
+
+pushd node
+npm version minor
+popd
+
+git add -u
+git commit -m "Bump versions"
+git push
+```
+
+### 2. Push a new tag
+
+```shell
+git tag vX.X.X
+git push --tag vX.X.X
+```
+
+When the tag is pushed, GitHub actions will start building the libraries and
+will upload them to a draft release. Wait for those jobs to complete.
+
+### 3. Publish the release
+
+Once the jobs are complete, you can edit the 
+
+2. Push a tag, such as vX.X.X. Once the tag is pushrf, GitHub actions will start
+   building the native libraries and uploading them to a draft release. Wait for
+   those jobs to complete.
+3. If the libraries are successful, edit the changelog and then publish the
+   release. Once you publish, a new action will start and upload all the 
+   release artifacts to npm.
+
+## Manual process
+
+You can build the artifacts locally on a MacOS machine.
+
+### Build the MacOS release libraries
+
+One-time setup:
+
+```shell
+rustup target add x86_64-apple-darwin aarch64-apple-darwin
+```
+
+To build:
+
+```shell
+bash ci/build_macos_artifacts.sh
+```
+
+### Build the Linux release libraries
+
+One-time setup, building the Docker container
+
+```shell
+cat ci/ubuntu_build.dockerfile | docker build -t lancedb-node-build -
+```
+
+To build:
+
+```shell
+docker run \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -v $(pwd):/io -w /io \
+    lancedb-node-build \
+    bash ci/build_linux_artifacts.sh
+```