diff --git a/.github/workflows/build_node_module/action.yml b/.github/workflows/build_node_module/action.yml new file mode 100644 index 00000000..e69de29b diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index b1a85421..5a6c81e8 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -67,8 +67,10 @@ jobs: - name: Build run: | npm ci - npm run build npm run tsc + npm run build + npm run pack-build + npm install --no-save ./dist/vectordb-*.tgz - name: Test run: npm run test macos: @@ -94,8 +96,10 @@ jobs: - name: Build run: | npm ci - npm run build npm run tsc + npm run build + npm run pack-build + npm install --no-save ./dist/vectordb-*.tgz - name: Test run: | npm run test diff --git a/.github/workflows/node_release.yml b/.github/workflows/node_release.yml deleted file mode 100644 index d0c62329..00000000 --- a/.github/workflows/node_release.yml +++ /dev/null @@ -1,118 +0,0 @@ -name: Node Release - -on: - pull_request: - paths: - - node/** - - .github/workflows/node_release.yml - -jobs: - macos: - runs-on: macos-12 - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - lfs: true - - name: Install system dependencies - run: brew install protobuf - - name: Install rustup target - run: rustup target add aarch64-apple-darwin - - name: Build x86_64 - run: cargo build --lib -p vectordb-node --release --target x86_64-apple-darwin - - name: Upload x86_64 dylib - uses: actions/upload-artifact@v3 - with: - path: target/x86_64-apple-darwin/release/libvectordb_node.dylib - name: x86_64-apple-darwin.node - retention-days: 5 - - name: Build aarch64 - run: cargo build --lib -p vectordb-node --release --target aarch64-apple-darwin - - name: Upload aarch64 dylib - uses: actions/upload-artifact@v3 - with: - path: target/aarch64-apple-darwin/release/libvectordb_node.dylib - name: aarch64-apple-darwin.node - retention-days: 5 - - linux-x64: - runs-on: ubuntu-latest - container: quay.io/pypa/manylinux2014_x86_64 - steps: - - uses: actions/checkout@v3 - - name: Install system dependencies - run: | - yum install -y openssl-devel - # protobuf is too old, so we directly download binaries - PB_REL="https://github.com/protocolbuffers/protobuf/releases" - PB_VERSION=23.1 - curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip - unzip protoc-$PB_VERSION-linux-x86_64.zip -d $HOME/.local - echo "${HOME}/.local/bin" >> $GITHUB_PATH - # TODO: replace this with something more modern - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - - name: Build GNU x86_64 - run: cargo build --lib -p vectordb-node --release --verbose - - name: Upload GNU x86_64 dylib - uses: actions/upload-artifact@v3 - with: - path: target/x86_64-unknown-linux-gnu/release/libvectordb_node.so - name: x86_64-unknown-linux-gnu.node - retention-days: 5 - - draft-release: - runs-on: ubuntu-latest - needs: - - linux-x64 - - macos - steps: - - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 - - name: Move libraries - run: mv *.node node/ - # TODO: make sure we aren't rebuilding the library here. - - name: Build module - run: | - npm install - npm run build - working-directory: node - - name: Test module - run: | - npm run tsc - npm test - working-directory: node - - name: Zip module - run: | - rm -rf node/node_modules - zip -r vectordb-node.zip node - - name: Upload - uses: actions/upload-artifact@v3 - with: - path: vectordb-node.zip - retention-days: 5 - # - name: Create draft release - # uses: softprops/action-gh-release@v1 - # if: startsWith(github.ref, 'refs/tags/') - # with: - # draft: true - # generate_release_notes: true - # files: - # vectordb-node-$GITHUB_REF-node-v3-linux-x64.zip # x86_64-unknown-linux-gnu.node - # vectordb-node-$GITHUB_REF-node-v3-darwin-x64.zip # x86_64-apple-darwin.node - # vectordb-node-$GITHUB_REF-node-v3-linux-x64.zip # aarch64-apple-darwin.node - - # TODO: create draft release - # use: https://github.com/softprops/action-gh-release - # format like: https://github.com/tungv/cypher.js/releases/tag/v1.0.0-beta.6 - # - - # test-linux-x64: - # runs-on: ubuntu-latest - # needs: - # - draft-release - # steps: - # - \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..bbcef4c6 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,123 @@ +name: Prepare Release + +# Based on https://github.com/dherman/neon-prebuild-example/blob/eaa4d33d682e5eb7abbc3da7aed153a1b1acb1b3/.github/workflows/publish.yml + +on: + push: + tags: + - v* + +# TODO: +# Create draft relase +# Build and upload artifacts +# * Build wheel and sdist +# * Build npm module +# * Build native node modules +# Test release +# Publish release + +env: + NODE_VERSION: 18.x + NPM_REGISTRY: 'https://registry.npmjs.org' + +jobs: + draft-release: + runs-on: ubuntu-latest + steps: + - uses: softprops/action-gh-release@v1 + with: + draft: true + prerelease: true # hardcoded on for now + generate_release_notes: true + + python: + runs-on: ubuntu-latest + needs: draft-release + defaults: + run: + shell: bash + working-directory: python + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + lfs: true + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.10 + - name: Build wheel + run: python setup.py sdist bdist_wheel + - uses: softprops/action-gh-release@v1 + with: + files: | + python/dist/lancedb-*.tar.gz + python/dist/lancedb-*.whl + + node: + runs-on: ubuntu-latest + needs: draft-release + defaults: + run: + shell: bash + working-directory: node + steps: + - name: Checkout + uses: actions/checkout@v2 + - uses: actions/setup-node@v3 + with: + node-version: 20 + cache: 'npm' + cache-dependency-path: node/package-lock.json + - uses: Swatinem/rust-cache@v2 + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y protobuf-compiler libssl-dev + - name: Build + run: | + npm ci + npm run tsc + - uses: softprops/action-gh-release@v1 + with: + files: node/vectordb-*.tgz + + node-macos: + runs-on: macos-12 + needs: draft-release + strategy: + matrix: + target: [x86_64-apple-darwin, aarch64-apple-darwin] + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Install system dependencies + run: brew install protobuf + - name: Build MacOS native node modules + run: bash ci/build_macos_artifacts.sh ${{ matrix.target }} + - uses: softprops/action-gh-release@v1 + with: + files: node/dist/vectordb-darwin*.tgz + + # node-linux: + # runs-on: ubuntu-latest + # needs: draft-release + # strategy: + # matrix: + # target: + # - x86_64-unknown-linux-gnu:centos + # - aarch64-unknown-linux-gnu:centos + # - aarch64-unknown-linux-musl + # - x86_64-unknown-linux-musl + # steps: + # - name: Install system dependencies + # run: | + # yum install -y openssl-devel + # # protobuf is too old, so we directly download binaries + # PB_REL="https://github.com/protocolbuffers/protobuf/releases" + # PB_VERSION=23.1 + # curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip + # unzip protoc-$PB_VERSION-linux-x86_64.zip -d $HOME/.local + # echo "${HOME}/.local/bin" >> $GITHUB_PATH + + \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6f8e1514..56d590ef 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ **/__pycache__ .DS_Store +.vscode + rust/target rust/Cargo.lock diff --git a/Cross.toml b/Cross.toml new file mode 100644 index 00000000..c6080a8c --- /dev/null +++ b/Cross.toml @@ -0,0 +1,14 @@ +[target.x86_64-unknown-linux-gnu] +pre-build = [ + "dpkg --add-architecture $CROSS_DEB_ARCH", + "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH protobuf-compiler", +] +image = "ghcr.io/cross-rs/x86_64-unknown-linux-gnu:main-centos" + +[target.aarch64-unknown-linux-gnu] +pre-build = [ + "dpkg --add-architecture $CROSS_DEB_ARCH", + "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH protobuf-compiler", +] +image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:centos" +# docker pull ghcr.io/cross-rs/aarch64-unknown-linux-gnu:edge-centos@sha256:8ca737af41df2a8078eb03a0a19e38405854ad05891a7381f7f671727a9ff736 diff --git a/ci/build_linux_artifacts.sh b/ci/build_linux_artifacts.sh new file mode 100644 index 00000000..200f73ea --- /dev/null +++ b/ci/build_linux_artifacts.sh @@ -0,0 +1,40 @@ +# Builds the Linux artifacts (node binaries). +# Usage: ./build_linux_artifacts.sh [target] +# Targets supported: +# - x86_64-unknown-linux-gnu:centos +# - aarch64-unknown-linux-gnu:centos +# - aarch64-unknown-linux-musl +# - x86_64-unknown-linux-musl + +# On MacOS, need to run in a linux container: +# docker run -v $(pwd):/io -w /io + +# Must run rustup toolchain install stable-x86_64-unknown-linux-gnu --force-non-host + +set -e + +build_node_binaries() { + pushd node + + for target in $1 + do + echo "Building node library for $target" + # cross doesn't yet pass this down to Docker, so we do it ourselves. + if [[ $target == x86_64* ]]; then + export CROSS_CONTAINER_OPTS="--platform linux/amd64" + else + export CROSS_CONTAINER_OPTS="--platform linux/arm64/v8" + fi + npm run cross-release -- --target $target + npm run pack-build -- --target $target + done + popd +} + +if [ -n "$1" ]; then + targets=$1 +else + # targets="x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu aarch64-unknown-linux-musl x86_64-unknown-linux-musl" + targets="aarch64-unknown-linux-gnu" +fi +build_node_binaries $targets \ No newline at end of file diff --git a/ci/build_macos_artifacts.sh b/ci/build_macos_artifacts.sh new file mode 100644 index 00000000..f940fd84 --- /dev/null +++ b/ci/build_macos_artifacts.sh @@ -0,0 +1,22 @@ +# Builds the macOS artifacts (node binaries). +# Usage: ./build_macos_artifacts.sh [target] +# Targets supported: x86_64-apple-darwin aarch64-apple-darwin + +build_node_binaries() { + pushd node + + for target in $1 + do + echo "Building node library for $target" + npm run build-release -- --target $target + npm run pack-build -- --target $target + done + popd +} + +if [ -n "$1" ]; then + targets=$1 +else + targets="x86_64-apple-darwin aarch64-apple-darwin" +fi +build_node_binaries $targets \ No newline at end of file diff --git a/ci/ubuntu_build.dockerfile b/ci/ubuntu_build.dockerfile new file mode 100644 index 00000000..fd6c3020 --- /dev/null +++ b/ci/ubuntu_build.dockerfile @@ -0,0 +1,31 @@ +# On MacOS, need to run in a linux container: +# cat ci/ubuntu_build.dockerfile | docker build -t lancedb-node-build - +# docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd):/io -w /io lancedb-node-build bash ci/build_linux_artifacts.sh +FROM ubuntu:20.04 + +ARG DEBIAN_FRONTEND=noninteractive +ENV TZ=Europe/Moscow + +RUN apt update && apt install -y protobuf-compiler libssl-dev build-essential curl \ + software-properties-common npm docker.io + +# Install rust +RUN curl https://sh.rustup.rs -sSf | bash -s -- -y + +ENV PATH="/root/.cargo/bin:${PATH}" + +# Install cross +# https://github.com/cross-rs/cross/issues/1257#issuecomment-1544553706 +RUN cargo install cross --git https://github.com/cross-rs/cross + +# Install additional build targets +RUN rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu aarch64-unknown-linux-musl x86_64-unknown-linux-musl + +# Install node +RUN npm install npm@latest -g && \ + npm install n -g && \ + n latest + +# set CROSS_CONTAINER_IN_CONTAINER to inform `cross` that it is executed from within a container +ENV CROSS_CONTAINER_IN_CONTAINER=true +ENV CROSS_CONTAINER_ENGINE_NO_BUILDKIT=1 diff --git a/node/.npmignore b/node/.npmignore new file mode 100644 index 00000000..8da45a2a --- /dev/null +++ b/node/.npmignore @@ -0,0 +1,2 @@ +gen_test_data.py +index.node diff --git a/node/README.md b/node/README.md index 22ae0028..9e4093a9 100644 --- a/node/README.md +++ b/node/README.md @@ -27,10 +27,12 @@ The [examples](./examples) folder contains complete examples. ## Development -Build the rust library with: +Build and install the rust library with: ```bash npm run build +npm run pack-build +npm install --no-save ./dist/vectordb-*.tgz ``` The LanceDB javascript is built with npm: diff --git a/node/native.js b/node/native.js index 0d4172a0..881bf58e 100644 --- a/node/native.js +++ b/node/native.js @@ -12,29 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +const { currentTarget } = require('@neon-rs/load'); + let nativeLib; -function getPlatformLibrary() { - if (process.platform === "darwin" && process.arch == "arm64") { - return require('./aarch64-apple-darwin.node'); - } else if (process.platform === "darwin" && process.arch == "x64") { - return require('./x86_64-apple-darwin.node'); - } else if (process.platform === "linux" && process.arch == "x64") { - return require('./x86_64-unknown-linux-gnu.node'); - } else { - throw new Error(`vectordb: unsupported platform ${process.platform}_${process.arch}. Please file a bug report at https://github.com/lancedb/lancedb/issues`) - } -} - try { - nativeLib = require('./index.node') -} catch (e) { - if (e.code === "MODULE_NOT_FOUND") { - nativeLib = getPlatformLibrary(); - } else { - throw new Error('vectordb: failed to load native library. Please file a bug report at https://github.com/lancedb/lancedb/issues'); - } + nativeLib = require(`@vectordb/${currentTarget()}`); +} catch { + throw new Error('vectordb: failed to load native library. Please file a bug report at https://github.com/lancedb/lancedb/issues'); } -module.exports = nativeLib - +// Dynamic require for runtime. +module.exports = nativeLib; diff --git a/node/package-lock.json b/node/package-lock.json index de82fc04..07e1d897 100644 --- a/node/package-lock.json +++ b/node/package-lock.json @@ -7,12 +7,26 @@ "": { "name": "vectordb", "version": "0.1.1", + "cpu": [ + "x64", + "arm64" + ], "license": "Apache-2.0", + "os": [ + "darwin", + "linux" + ], "dependencies": { "@apache-arrow/ts": "^12.0.0", + "@neon-rs/load": "^0.0.74", + "@vectordb/darwin-arm64": "0.1.1", + "@vectordb/darwin-x64": "0.1.1", + "@vectordb/linux-x64-gnu": "0.1.1", + "@vectordb/linux-x64-musl": "0.1.1", "apache-arrow": "^12.0.0" }, "devDependencies": { + "@neon-rs/cli": "^0.0.74", "@types/chai": "^4.3.4", "@types/mocha": "^10.0.1", "@types/node": "^18.16.2", @@ -30,6 +44,12 @@ "ts-node": "^10.9.1", "ts-node-dev": "^2.0.0", "typescript": "*" + }, + "optionalDependencies": { + "@vectordb/darwin-arm64": "0.1.1", + "@vectordb/darwin-x64": "0.1.1", + "@vectordb/linux-x64-gnu": "0.1.1", + "@vectordb/linux-x64-musl": "0.1.1" } }, "node_modules/@apache-arrow/ts": { @@ -197,6 +217,20 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@neon-rs/cli": { + "version": "0.0.74", + "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz", + "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==", + "dev": true, + "bin": { + "neon": "index.js" + } + }, + "node_modules/@neon-rs/load": { + "version": "0.0.74", + "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz", + "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg==" + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -4191,6 +4225,17 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "@neon-rs/cli": { + "version": "0.0.74", + "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz", + "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==", + "dev": true + }, + "@neon-rs/load": { + "version": "0.0.74", + "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz", + "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg==" + }, "@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", diff --git a/node/package.json b/node/package.json index 2b41f498..52af69e1 100644 --- a/node/package.json +++ b/node/package.json @@ -6,10 +6,12 @@ "types": "dist/index.d.ts", "scripts": { "tsc": "tsc -b", - "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json-render-diagnostics", + "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json", "build-release": "npm run build -- --release", + "cross-release": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cross build --message-format=json --release -p vectordb-node", "test": "mocha -recursive dist/test", - "lint": "eslint src --ext .js,.ts" + "lint": "eslint src --ext .js,.ts", + "pack-build": "neon pack-build" }, "repository": { "type": "git", @@ -24,6 +26,7 @@ "author": "Lance Devs", "license": "Apache-2.0", "devDependencies": { + "@neon-rs/cli": "^0.0.74", "@types/chai": "^4.3.4", "@types/mocha": "^10.0.1", "@types/node": "^18.16.2", @@ -44,6 +47,29 @@ }, "dependencies": { "@apache-arrow/ts": "^12.0.0", + "@neon-rs/load": "^0.0.74", "apache-arrow": "^12.0.0" + }, + "os": [ + "darwin", + "linux" + ], + "cpu": [ + "x64", + "arm64" + ], + "neon": { + "targets": { + "x86_64-apple-darwin": "@vectordb/darwin-x64", + "aarch64-apple-darwin": "@vectordb/darwin-arm64", + "x86_64-unknown-linux-gnu": "@vectordb/linux-x64-gnu", + "x86_64-unknown-linux-musl": "@vectordb/linux-x64-musl" + } + }, + "optionalDependencies": { + "@vectordb/darwin-arm64": "0.1.1", + "@vectordb/darwin-x64": "0.1.1", + "@vectordb/linux-x64-gnu": "0.1.1", + "@vectordb/linux-x64-musl": "0.1.1" } } diff --git a/rust/ffi/node/release_process.md b/rust/ffi/node/release_process.md new file mode 100644 index 00000000..80c4416b --- /dev/null +++ b/rust/ffi/node/release_process.md @@ -0,0 +1,83 @@ + +How to release the node module + +### 1. Bump the versions + +```shell +pushd rust/vectordb +cargo bump minor +popd + +pushd rust/ffi/node +cargo bump minor +popd + +pushd python +cargo bump minor +popd + +pushd node +npm version minor +popd + +git add -u +git commit -m "Bump versions" +git push +``` + +### 2. Push a new tag + +```shell +git tag vX.X.X +git push --tag vX.X.X +``` + +When the tag is pushed, GitHub actions will start building the libraries and +will upload them to a draft release. Wait for those jobs to complete. + +### 3. Publish the release + +Once the jobs are complete, you can edit the + +2. Push a tag, such as vX.X.X. Once the tag is pushrf, GitHub actions will start + building the native libraries and uploading them to a draft release. Wait for + those jobs to complete. +3. If the libraries are successful, edit the changelog and then publish the + release. Once you publish, a new action will start and upload all the + release artifacts to npm. + +## Manual process + +You can build the artifacts locally on a MacOS machine. + +### Build the MacOS release libraries + +One-time setup: + +```shell +rustup target add x86_64-apple-darwin aarch64-apple-darwin +``` + +To build: + +```shell +bash ci/build_macos_artifacts.sh +``` + +### Build the Linux release libraries + +One-time setup, building the Docker container + +```shell +cat ci/ubuntu_build.dockerfile | docker build -t lancedb-node-build - +``` + +To build: + +```shell +docker run \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v $(pwd):/io -w /io \ + lancedb-node-build \ + bash ci/build_linux_artifacts.sh +```