add dbg prints

.github/workflows/make_release_commit.yml

@@ -1,70 +0,0 @@
-name: Create release commit
-
-on:
-  workflow_dispatch:
-    inputs:
-      dry_run:
-        description: 'Just create the local commit/tags but do not push it'
-        required: true
-        default: "false"
-        type: choice
-        options:
-          - "true"
-          - "false"
-      part:
-        description: 'What kind of release is this?'
-        required: true
-        default: 'patch'
-        type: choice
-        options:
-          - patch
-          - minor
-          - major
-
-jobs:
-  bump-version:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out main
-      uses: actions/checkout@v3
-      with:
-        ref: main
-        persist-credentials: false
-        fetch-depth: 0
-        lfs: true
-    - name: Install cargo utils
-      run: cargo install cargo-bump cargo-get
-    - name: Bump vectordb
-      working-directory: rust/vectordb
-      run: |
-        cargo bump ${{ inputs.part }}
-        echo "CRATE_VERSION=$(cargo get version)" >> $GITHUB_ENV
-    - name: Bump rust/ffi/node
-      working-directory: rust/ffi/node
-      run: |
-        cargo bump ${{ inputs.part }}
-        echo "FFI_CRATE_VERSION=$(cargo get version)" >> $GITHUB_ENV 
-    - name: Bump node
-      working-directory: node
-      run: |
-        npm version ${{ inputs.part }}
-        echo "NPM_PACKAGE_VERSION=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV 
-    - name: Create tag
-      run: |
-        if [ "$CRATE_VERSION" != "$FFI_CRATE_VERSION" ]; then
-          echo "Version mismatch between rust/vectordb and rust/ffi/node"
-          exit 1
-        fi
-        if [ "$CRATE_VERSION" != "$NPM_PACKAGE_VERSION" ]; then
-          echo "Version mismatch between rust/vectordb and node"
-          exit 1
-        fi
-        export TAG="v$CRATE_VERSION'"
-        git tag $TAG
-    - name: Push new version and tag
-      if: ${{ inputs.dry_run }} == "false"
-      uses: ad-m/github-push-action@master
-      with:
-        github_token: ${{ secrets.RELEASE_TOKEN }}
-        branch: main
-        tags: true

.github/workflows/release.yml

@@ -1,7 +1,6 @@
 name: Prepare Release
 
-# TODO: bump versions in CI
-# NOTE: Python is a separate release for now.
+# Based on https://github.com/dherman/neon-prebuild-example/blob/eaa4d33d682e5eb7abbc3da7aed153a1b1acb1b3/.github/workflows/publish.yml
 
 on:
   push:
@@ -41,6 +40,34 @@ jobs:
           draft: true
           files: target/package/vectordb-*.crate
           fail_on_unmatched_files: true
+    
+  python:
+    runs-on: ubuntu-latest
+    needs: draft-release
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Build wheel
+        run: |
+          pip install wheel
+          python setup.py sdist bdist_wheel
+      - uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          files: |
+            python/dist/lancedb-*.tar.gz 
+            python/dist/lancedb-*.whl
+          fail_on_unmatched_files: true
 
   node:
     runs-on: ubuntu-latest
@@ -88,9 +115,6 @@ jobs:
       run: |
         cd node
         npm ci
-    - name: Install rustup target
-      if: ${{ matrix.target == 'aarch64-apple-darwin' }}
-      run: rustup target add aarch64-apple-darwin
     - name: Build MacOS native node modules
       run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
     - uses: softprops/action-gh-release@v1
@@ -117,10 +141,6 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v2
-    - name: Change owner to root (for npm)
-      # The docker container is run as root, so we need the files to be owned by root
-      # Otherwise npm is a nightmare: https://github.com/npm/cli/issues/3773
-      run: sudo chown -R root:root .
     - name: Set up QEMU
       if: ${{ matrix.arch == 'aarch64' }}
       uses: docker/setup-qemu-action@v2  
@@ -147,7 +167,7 @@ jobs:
         fail_on_unmatched_files: true
 
   release:
-    needs: [rust, node, node-macos, node-linux]
+    needs: [python, node, node-macos, node-linux, rust]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/download-artifact@v3

Cross.toml

@@ -0,0 +1,39 @@
+# These make sure our builds are compatible with old glibc versions.
+[target.x86_64-unknown-linux-gnu]
+pre-build = [
+    # Install newer gfortran
+    "yum install -y openssl-devel unzip gcc-gfortran",
+    "scl enable devtoolset-11 bash",
+    # protobuf is too old, so we directly download binaries
+    "PB_REL=https://github.com/protocolbuffers/protobuf/releases",
+    "PB_VERSION=23.1",
+    "curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip",
+    "unzip protoc-$PB_VERSION-linux-x86_64.zip -d /usr/local",
+]
+image = "ghcr.io/cross-rs/x86_64-unknown-linux-gnu:main-centos"
+
+[target.aarch64-unknown-linux-gnu]
+pre-build = [
+    "yum install -y openssl-devel unzip",
+    # protobuf is too old, so we directly download binaries
+    "PB_REL=https://github.com/protocolbuffers/protobuf/releases",
+    "PB_VERSION=23.1",
+    "curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip",
+    "unzip protoc-$PB_VERSION-linux-x86_64.zip -d /usr/local",
+]
+# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.aarch64-unknown-linux-gnu.centos
+image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main-centos"
+
+[target.x86_64-unknown-linux-musl]
+# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.x86_64-unknown-linux-musl
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH",
+]
+
+[target.aarch64-unknown-linux-musl]
+# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.aarch64-unknown-linux-musl
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH",
+]

ci/build_linux_artifacts.sh

@@ -18,7 +18,7 @@ setup_dependencies() {
         apk add openssl-dev
     else
         # manylinux2014
-        yum install -y openssl-devel unzip
+        yum install -y openssl-devel unzip   
     fi
 
     if [[ $1 == x86_64* ]]; then
@@ -44,23 +44,32 @@ install_node() {
         # This node version is 15, we need 16 or higher:
         # apk add nodejs-current npm 
         # So instead we install from source (nvm doesn't provide binaries for musl):
-        nvm install -s --no-progress 17
+        nvm install -s 17
     else
-        nvm install --no-progress 17 # latest that supports glibc 2.17
+        nvm install 17 # latest that supports glibc 2.17
     fi
+
+    printenv
+    echo "Node version:"
+    npm --version
+    which npm
+    which node
 }
 
 install_rust() {
     echo "Installing rust..."
     curl https://sh.rustup.rs -sSf | bash -s -- -y
+
+    printenv
+
     export PATH="$PATH:/root/.cargo/bin"
+
+    printenv
 }
 
 build_node_binary() {
     echo "Building node library for $1..."
     pushd node
-
-    npm ci
     
     if [[ $1 == *musl ]]; then
         # This is needed for cargo to allow build cdylibs with musl

ci/build_macos_artifacts.sh

@@ -2,20 +2,6 @@
 # Usage: ./build_macos_artifacts.sh [target]
 # Targets supported: x86_64-apple-darwin aarch64-apple-darwin
 
-prebuild_rust() {
-    # Building here for the sake of easier debugging.
-    pushd rust/ffi/node
-
-    for target in $1
-    do
-        echo "Building rust library for $target"
-        export RUST_BACKTRACE=1
-        cargo build --release --target $target
-    done
-
-    popd
-}
-
 build_node_binaries() {
     pushd node
     
@@ -33,6 +19,4 @@ if [ -n "$1" ]; then
 else
     targets="x86_64-apple-darwin aarch64-apple-darwin"
 fi
-
-prebuild_rust $targets
 build_node_binaries $targets

ci/release_process.md

@@ -1,24 +1,56 @@
-# How to release
 
-This is for the Rust crate and Node module. For now, the Python module is
-released separately.
+How to release the node module
 
-The release is started by bumping the versions and pushing a new tag. To do this
-automatically, use the `make_release_commit` GitHub action.
+### 1. Bump the versions
+
+<!-- TODO: we also need to bump the optional dependencies for node! -->
+
+```shell
+pushd rust/vectordb
+cargo bump minor
+popd
+
+pushd rust/ffi/node
+cargo bump minor
+popd
+
+pushd python
+cargo bump minor
+popd
+
+pushd node
+npm version minor
+popd
+
+git add -u
+git commit -m "Bump versions"
+git push
+```
+
+### 2. Push a new tag
+
+```shell
+git tag vX.X.X
+git push --tag vX.X.X
+```
 
 When the tag is pushed, GitHub actions will start building the libraries and
-will upload them to a draft release.
+will upload them to a draft release. Wait for those jobs to complete.
 
-While those jobs are running, edit the release notes as needed. For example, 
-bring relevant new features and bugfixes to the top of the notes and the testing
-and CI changes to the bottom.
+### 3. Publish the release
 
-Once the jobs have finished, the release will be marked as not draft and the
-artifacts will be released to crates.io, NPM, and PyPI.
+Once the jobs are complete, you can edit the 
+
+2. Push a tag, such as vX.X.X. Once the tag is pushrf, GitHub actions will start
+   building the native libraries and uploading them to a draft release. Wait for
+   those jobs to complete.
+3. If the libraries are successful, edit the changelog and then publish the
+   release. Once you publish, a new action will start and upload all the 
+   release artifacts to npm.
 
 ## Manual process
 
-You can also build the artifacts locally on a MacOS machine.
+You can build the artifacts locally on a MacOS machine.
 
 ### Build the MacOS release libraries
 
@@ -48,12 +80,11 @@ docker run \
 
 You can change `ARCH` to `x86_64`.
 
-Similar script for musl binaries (not yet working):
+Similar script for musl binaries:
 
 ```shell
 ARCH=aarch64
 docker run \
-    --user $(id -u) \
     -v $(pwd):/io -w /io \
     quay.io/pypa/musllinux_1_1_$ARCH \
     bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-musl

node/README.md

@@ -9,8 +9,7 @@ npm install vectordb
 ```
 
 This will download the appropriate native library for your platform. We currently
-support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
-yet support Windows or musl-based Linux (such as Alpine Linux).
+support x86_64 Linux, Intel MacOS, and ARM (M1/M2) MacOS.
 
 ## Usage
 

node/package.json

@@ -11,8 +11,7 @@
     "cross-release": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cross build --message-format=json --release -p vectordb-node",
     "test": "mocha -recursive dist/test",
     "lint": "eslint src --ext .js,.ts",
-    "pack-build": "neon pack-build",
-    "check-npm": "printenv && which node && which npm && npm --version"
+    "pack-build": "neon pack-build"
   },
   "repository": {
     "type": "git",