Compare commits

...

12 Commits

Author SHA1 Message Date
Will Jones
a9dcfe7535 make node available to all users 2023-05-25 17:50:07 -07:00
Will Jones
0028b95fd8 mac debug info 2023-05-25 13:33:54 -07:00
Will Jones
102f1d7404 add dbg prints 2023-05-25 09:44:02 -07:00
Will Jones
500aa7b002 give up on musl for now 2023-05-25 09:21:40 -07:00
Will Jones
8aa0f6b4ba use manylinux containers locally 2023-05-25 09:21:40 -07:00
Will Jones
140aa32e08 try manylinux again 2023-05-25 09:21:40 -07:00
Will Jones
a067c3dc85 fixes for action 2023-05-25 09:21:40 -07:00
Will Jones
e762a4db4b cleanup 2023-05-25 09:21:40 -07:00
Will Jones
5e0ff01879 match versions 2023-05-25 09:21:40 -07:00
Will Jones
84356220dd fill out rest of release script 2023-05-25 09:21:40 -07:00
Will Jones
6c03662c68 more progress on release workflow 2023-05-25 09:21:40 -07:00
Will Jones
5e098f4fe5 wip: see if we can build the lib in ci 2023-05-25 09:21:40 -07:00
15 changed files with 584 additions and 29 deletions

View File

@@ -0,0 +1,70 @@
name: Create release commit
on:
workflow_dispatch:
inputs:
dry_run:
description: 'Just create the local commit/tags but do not push it'
required: true
default: "false"
type: choice
options:
- "true"
- "false"
part:
description: 'What kind of release is this?'
required: true
default: 'patch'
type: choice
options:
- patch
- minor
- major
jobs:
bump-version:
runs-on: ubuntu-latest
steps:
- name: Check out main
uses: actions/checkout@v3
with:
ref: main
persist-credentials: false
fetch-depth: 0
lfs: true
- name: Install cargo utils
run: cargo install cargo-bump cargo-get
- name: Bump vectordb
working-directory: rust/vectordb
run: |
cargo bump ${{ inputs.part }}
echo "CRATE_VERSION=$(cargo get version)" >> $GITHUB_ENV
- name: Bump rust/ffi/node
working-directory: rust/ffi/node
run: |
cargo bump ${{ inputs.part }}
echo "FFI_CRATE_VERSION=$(cargo get version)" >> $GITHUB_ENV
- name: Bump node
working-directory: node
run: |
npm version ${{ inputs.part }}
echo "NPM_PACKAGE_VERSION=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV
- name: Create tag
run: |
if [ "$CRATE_VERSION" != "$FFI_CRATE_VERSION" ]; then
echo "Version mismatch between rust/vectordb and rust/ffi/node"
exit 1
fi
if [ "$CRATE_VERSION" != "$NPM_PACKAGE_VERSION" ]; then
echo "Version mismatch between rust/vectordb and node"
exit 1
fi
export TAG="v$CRATE_VERSION'"
git tag $TAG
- name: Push new version and tag
if: ${{ inputs.dry_run }} == "false"
uses: ad-m/github-push-action@master
with:
github_token: ${{ secrets.RELEASE_TOKEN }}
branch: main
tags: true

View File

@@ -67,8 +67,10 @@ jobs:
- name: Build
run: |
npm ci
npm run build
npm run tsc
npm run build
npm run pack-build
npm install --no-save ./dist/vectordb-*.tgz
- name: Test
run: npm run test
macos:
@@ -94,8 +96,10 @@ jobs:
- name: Build
run: |
npm ci
npm run build
npm run tsc
npm run build
npm run pack-build
npm install --no-save ./dist/vectordb-*.tgz
- name: Test
run: |
npm run test

174
.github/workflows/release.yml vendored Normal file
View File

@@ -0,0 +1,174 @@
name: Prepare Release
# TODO: bump versions in CI
# NOTE: Python is a separate release for now.
on:
push:
tags:
- v*
jobs:
draft-release:
runs-on: ubuntu-latest
steps:
- uses: softprops/action-gh-release@v1
with:
draft: true
prerelease: true # hardcoded on for now
generate_release_notes: true
rust:
runs-on: ubuntu-latest
needs: draft-release
defaults:
run:
shell: bash
working-directory: rust/vectordb
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
lfs: true
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Package Rust
run: cargo package --all-features
- uses: softprops/action-gh-release@v1
with:
draft: true
files: target/package/vectordb-*.crate
fail_on_unmatched_files: true
node:
runs-on: ubuntu-latest
needs: draft-release
defaults:
run:
shell: bash
working-directory: node
steps:
- name: Checkout
uses: actions/checkout@v2
- uses: actions/setup-node@v3
with:
node-version: 20
cache: 'npm'
cache-dependency-path: node/package-lock.json
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Build
run: |
npm ci
npm run tsc
npm pack
- uses: softprops/action-gh-release@v1
with:
draft: true
files: node/vectordb-*.tgz
fail_on_unmatched_files: true
node-macos:
runs-on: macos-12
needs: draft-release
strategy:
fail-fast: false
matrix:
target: [x86_64-apple-darwin, aarch64-apple-darwin]
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Install system dependencies
run: brew install protobuf
- name: Install npm dependencies
run: |
cd node
npm ci
- name: Install rustup target
if: ${{ matrix.target == 'aarch64-apple-darwin' }}
run: rustup target add aarch64-apple-darwin
- name: Build MacOS native node modules
run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
- uses: softprops/action-gh-release@v1
with:
draft: true
files: node/dist/vectordb-darwin*.tgz
fail_on_unmatched_files: true
node-linux:
name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }})
runs-on: ubuntu-latest
needs: draft-release
strategy:
fail-fast: false
matrix:
libc:
- gnu
# TODO: re-enable musl once we have refactored to pre-built containers
# Right now we have to build node from source which is too expensive.
# - musl
arch:
- x86_64
- aarch64
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Change owner to root (for npm)
# The docker container is run as root, so we need the files to be owned by root
# Otherwise npm is a nightmare: https://github.com/npm/cli/issues/3773
run: sudo chown -R root:root .
- name: Set up QEMU
if: ${{ matrix.arch == 'aarch64' }}
uses: docker/setup-qemu-action@v2
with:
platforms: arm64
- name: Build Linux GNU native node modules
if: ${{ matrix.libc == 'gnu' }}
run: |
docker run \
-v $(pwd):/io -w /io \
quay.io/pypa/manylinux2014_${{ matrix.arch }} \
bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu
- name: Build musl Linux native node modules
if: ${{ matrix.libc == 'musl' }}
run: |
docker run --platform linux/arm64/v8 \
-v $(pwd):/io -w /io \
quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
- uses: softprops/action-gh-release@v1
with:
draft: true
files: node/dist/vectordb-linux*.tgz
fail_on_unmatched_files: true
release:
needs: [rust, node, node-macos, node-linux]
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v3
- name: Publish to PyPI
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
run: |
python -m twine upload --non-interactive \
--skip-existing \
--repository testpypi python/dist/*
- name: Publish to NPM
run: |
for filename in node/dist/*.tgz; do
npm publish --dry-run $filename
done
- name: Publish to crates.io
env:
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
run: |
cargo publish --dry-run --no-verify rust/target/vectordb-*.crate
# - uses: softprops/action-gh-release@v1
# with:
# draft: false

2
.gitignore vendored
View File

@@ -4,6 +4,8 @@
**/__pycache__
.DS_Store
.vscode
rust/target
rust/Cargo.lock

4
Cargo.lock generated
View File

@@ -3356,7 +3356,7 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "vectordb"
version = "0.0.1"
version = "0.1.2"
dependencies = [
"arrow-array",
"arrow-data",
@@ -3369,7 +3369,7 @@ dependencies = [
[[package]]
name = "vectordb-node"
version = "0.1.0"
version = "0.1.2"
dependencies = [
"arrow-array",
"arrow-ipc",

View File

@@ -0,0 +1,91 @@
#!/bin/bash
# Builds the Linux artifacts (node binaries).
# Usage: ./build_linux_artifacts.sh [target]
# Targets supported:
# - x86_64-unknown-linux-gnu:centos
# - aarch64-unknown-linux-gnu:centos
# - aarch64-unknown-linux-musl
# - x86_64-unknown-linux-musl
# TODO: refactor this into a Docker container we can pull
set -e
setup_dependencies() {
echo "Installing system dependencies..."
if [[ $1 == *musl ]]; then
# musllinux
apk add openssl-dev
else
# manylinux2014
yum install -y openssl-devel unzip
fi
if [[ $1 == x86_64* ]]; then
ARCH=x86_64
else
# gnu target
ARCH=aarch_64
fi
# Install new enough protobuf (yum-provided is old)
PB_REL=https://github.com/protocolbuffers/protobuf/releases
PB_VERSION=23.1
curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local
}
install_node() {
echo "Installing node..."
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
source "$HOME"/.bashrc
if [[ $1 == *musl ]]; then
# This node version is 15, we need 16 or higher:
# apk add nodejs-current npm
# So instead we install from source (nvm doesn't provide binaries for musl):
nvm install -s --no-progress 17
else
nvm install --no-progress 17 # latest that supports glibc 2.17
fi
}
install_rust() {
echo "Installing rust..."
curl https://sh.rustup.rs -sSf | bash -s -- -y
export PATH="$PATH:/root/.cargo/bin"
}
build_node_binary() {
echo "Building node library for $1..."
pushd node
npm ci
if [[ $1 == *musl ]]; then
# This is needed for cargo to allow build cdylibs with musl
export RUSTFLAGS="-C target-feature=-crt-static"
fi
# Cargo can run out of memory while pulling dependencies, espcially when running
# in QEMU. This is a workaround for that.
export CARGO_NET_GIT_FETCH_WITH_CLI=true
# We don't pass in target, since the native target here already matches
# and openblas-src doesn't do well with cross-compilation.
npm run build-release
npm run pack-build
popd
}
TARGET=${1:-x86_64-unknown-linux-gnu}
# Others:
# aarch64-unknown-linux-gnu
# x86_64-unknown-linux-musl
# aarch64-unknown-linux-musl
setup_dependencies $TARGET
install_node $TARGET
install_rust
build_node_binary $TARGET

View File

@@ -0,0 +1,38 @@
# Builds the macOS artifacts (node binaries).
# Usage: ./build_macos_artifacts.sh [target]
# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
prebuild_rust() {
# Building here for the sake of easier debugging.
pushd rust/ffi/node
for target in $1
do
echo "Building rust library for $target"
export RUST_BACKTRACE=1
cargo build --release --target $target
done
popd
}
build_node_binaries() {
pushd node
for target in $1
do
echo "Building node library for $target"
npm run build-release -- --target $target
npm run pack-build -- --target $target
done
popd
}
if [ -n "$1" ]; then
targets=$1
else
targets="x86_64-apple-darwin aarch64-apple-darwin"
fi
prebuild_rust $targets
build_node_binaries $targets

90
ci/release_process.md Normal file
View File

@@ -0,0 +1,90 @@
# How to release
This is for the Rust crate and Node module. For now, the Python module is
released separately.
The release is started by bumping the versions and pushing a new tag. To do this
automatically, use the `make_release_commit` GitHub action.
When the tag is pushed, GitHub actions will start building the libraries and
will upload them to a draft release.
While those jobs are running, edit the release notes as needed. For example,
bring relevant new features and bugfixes to the top of the notes and the testing
and CI changes to the bottom.
Once the jobs have finished, the release will be marked as not draft and the
artifacts will be released to crates.io, NPM, and PyPI.
## Manual process
You can also build the artifacts locally on a MacOS machine.
### Build the MacOS release libraries
One-time setup:
```shell
rustup target add x86_64-apple-darwin aarch64-apple-darwin
```
To build:
```shell
bash ci/build_macos_artifacts.sh
```
### Build the Linux release libraries
To build a Linux library, we need to use docker with a different build script:
```shell
ARCH=aarch64
docker run \
-v $(pwd):/io -w /io \
quay.io/pypa/manylinux2014_$ARCH \
bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-gnu
```
You can change `ARCH` to `x86_64`.
Similar script for musl binaries (not yet working):
```shell
ARCH=aarch64
docker run \
--user $(id -u) \
-v $(pwd):/io -w /io \
quay.io/pypa/musllinux_1_1_$ARCH \
bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-musl
```
<!--
For debugging, use these snippets:
```shell
ARCH=aarch64
docker run -it \
-v $(pwd):/io -w /io \
quay.io/pypa/manylinux2014_$ARCH \
bash
```
```shell
ARCH=aarch64
docker run -it \
-v $(pwd):/io -w /io \
quay.io/pypa/musllinux_1_1_$ARCH \
bash
```
Note: musllinux_1_1 is Alpine Linux 3.12
-->
```
docker run \
-v $(pwd):/io -w /io \
quay.io/pypa/musllinux_1_1_aarch64 \
bash alpine_repro.sh
```

2
node/.npmignore Normal file
View File

@@ -0,0 +1,2 @@
gen_test_data.py
index.node

View File

@@ -8,6 +8,10 @@ A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb)
npm install vectordb
```
This will download the appropriate native library for your platform. We currently
support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
yet support Windows or musl-based Linux (such as Alpine Linux).
## Usage
### Basic Example
@@ -24,6 +28,19 @@ The [examples](./examples) folder contains complete examples.
## Development
Build and install the rust library with:
```bash
npm run build
npm run pack-build
npm install --no-save ./dist/vectordb-*.tgz
```
`npm run build` builds the Rust library, `npm run pack-build` packages the Rust
binary into an npm module called `@vectordb/<platform>` (for example,
`@vectordb/darwin-arm64.node`), and then `npm run install ...` installs that
module.
The LanceDB javascript is built with npm:
```bash

View File

@@ -12,29 +12,20 @@
// See the License for the specific language governing permissions and
// limitations under the License.
const { currentTarget } = require('@neon-rs/load');
let nativeLib;
function getPlatformLibrary() {
if (process.platform === "darwin" && process.arch == "arm64") {
return require('./aarch64-apple-darwin.node');
} else if (process.platform === "darwin" && process.arch == "x64") {
return require('./x86_64-apple-darwin.node');
} else if (process.platform === "linux" && process.arch == "x64") {
return require('./x86_64-unknown-linux-gnu.node');
} else {
throw new Error(`vectordb: unsupported platform ${process.platform}_${process.arch}. Please file a bug report at https://github.com/lancedb/lancedb/issues`)
}
}
try {
nativeLib = require('./index.node')
nativeLib = require(`@vectordb/${currentTarget()}`);
} catch (e) {
if (e.code === "MODULE_NOT_FOUND") {
nativeLib = getPlatformLibrary();
} else {
throw new Error('vectordb: failed to load native library. Please file a bug report at https://github.com/lancedb/lancedb/issues');
}
throw new Error(`vectordb: failed to load native library.
You may need to run \`npm install @vectordb/${currentTarget()}\`.
If that does not work, please file a bug report at https://github.com/lancedb/lancedb/issues
Source error: ${e}`);
}
module.exports = nativeLib
// Dynamic require for runtime.
module.exports = nativeLib;

45
node/package-lock.json generated
View File

@@ -7,12 +7,26 @@
"": {
"name": "vectordb",
"version": "0.1.1",
"cpu": [
"x64",
"arm64"
],
"license": "Apache-2.0",
"os": [
"darwin",
"linux"
],
"dependencies": {
"@apache-arrow/ts": "^12.0.0",
"@neon-rs/load": "^0.0.74",
"@vectordb/darwin-arm64": "0.1.1",
"@vectordb/darwin-x64": "0.1.1",
"@vectordb/linux-x64-gnu": "0.1.1",
"@vectordb/linux-x64-musl": "0.1.1",
"apache-arrow": "^12.0.0"
},
"devDependencies": {
"@neon-rs/cli": "^0.0.74",
"@types/chai": "^4.3.4",
"@types/mocha": "^10.0.1",
"@types/node": "^18.16.2",
@@ -30,6 +44,12 @@
"ts-node": "^10.9.1",
"ts-node-dev": "^2.0.0",
"typescript": "*"
},
"optionalDependencies": {
"@vectordb/darwin-arm64": "0.1.1",
"@vectordb/darwin-x64": "0.1.1",
"@vectordb/linux-x64-gnu": "0.1.1",
"@vectordb/linux-x64-musl": "0.1.1"
}
},
"node_modules/@apache-arrow/ts": {
@@ -197,6 +217,20 @@
"@jridgewell/sourcemap-codec": "^1.4.10"
}
},
"node_modules/@neon-rs/cli": {
"version": "0.0.74",
"resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
"integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
"dev": true,
"bin": {
"neon": "index.js"
}
},
"node_modules/@neon-rs/load": {
"version": "0.0.74",
"resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
"integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
},
"node_modules/@nodelib/fs.scandir": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -4191,6 +4225,17 @@
"@jridgewell/sourcemap-codec": "^1.4.10"
}
},
"@neon-rs/cli": {
"version": "0.0.74",
"resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
"integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
"dev": true
},
"@neon-rs/load": {
"version": "0.0.74",
"resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
"integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
},
"@nodelib/fs.scandir": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",

View File

@@ -1,15 +1,18 @@
{
"name": "vectordb",
"version": "0.1.1",
"version": "0.1.2",
"description": " Serverless, low-latency vector database for AI applications",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
"tsc": "tsc -b",
"build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json-render-diagnostics",
"build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json",
"build-release": "npm run build -- --release",
"cross-release": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cross build --message-format=json --release -p vectordb-node",
"test": "mocha -recursive dist/test",
"lint": "eslint src --ext .js,.ts"
"lint": "eslint src --ext .js,.ts",
"pack-build": "neon pack-build",
"check-npm": "printenv && which node && which npm && npm --version"
},
"repository": {
"type": "git",
@@ -24,6 +27,7 @@
"author": "Lance Devs",
"license": "Apache-2.0",
"devDependencies": {
"@neon-rs/cli": "^0.0.74",
"@types/chai": "^4.3.4",
"@types/mocha": "^10.0.1",
"@types/node": "^18.16.2",
@@ -44,6 +48,33 @@
},
"dependencies": {
"@apache-arrow/ts": "^12.0.0",
"@neon-rs/load": "^0.0.74",
"apache-arrow": "^12.0.0"
},
"os": [
"darwin",
"linux"
],
"cpu": [
"x64",
"arm64"
],
"neon": {
"targets": {
"x86_64-apple-darwin": "@vectordb/darwin-x64",
"aarch64-apple-darwin": "@vectordb/darwin-arm64",
"x86_64-unknown-linux-gnu": "@vectordb/linux-x64-gnu",
"x86_64-unknown-linux-musl": "@vectordb/linux-x64-musl",
"aarch64-unknown-linux-gnu": "@vectordb/linux-arm64-gnu",
"aarch64-unknown-linux-musl": "@vectordb/linux-arm64-musl"
}
},
"optionalDependencies": {
"@vectordb/darwin-arm64": "0.1.2",
"@vectordb/darwin-x64": "0.1.2",
"@vectordb/linux-x64-gnu": "0.1.2",
"@vectordb/linux-x64-musl": "0.1.2",
"@vectordb/linux-arm64-gnu": "0.1.2",
"@vectordb/linux-arm64-musl": "0.1.2"
}
}

View File

@@ -1,6 +1,6 @@
[package]
name = "vectordb-node"
version = "0.1.0"
version = "0.1.2"
description = "Serverless, low-latency vector database for AI applications"
license = "Apache-2.0"
edition = "2018"

View File

@@ -1,6 +1,6 @@
[package]
name = "vectordb"
version = "0.0.1"
version = "0.1.2"
edition = "2021"
description = "Serverless, low-latency vector database for AI applications"
license = "Apache-2.0"