diff --git a/.cargo/config.toml b/.cargo/config.toml index 709609e9..71c10c2a 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -34,6 +34,10 @@ rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c" [target.x86_64-unknown-linux-musl] rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=-crt-static,+avx2,+fma,+f16c"] +[target.aarch64-unknown-linux-musl] +linker = "aarch64-linux-musl-gcc" +rustflags = ["-C", "target-feature=-crt-static"] + [target.aarch64-apple-darwin] rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm,+dotprod"] @@ -44,4 +48,4 @@ rustflags = ["-Ctarget-feature=+crt-static"] # Experimental target for Arm64 Windows [target.aarch64-pc-windows-msvc] -rustflags = ["-Ctarget-feature=+crt-static"] \ No newline at end of file +rustflags = ["-Ctarget-feature=+crt-static"] diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index aa69ae5b..1f9ecd62 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -1,500 +1,29 @@ name: NPM Publish +env: + MACOSX_DEPLOYMENT_TARGET: '10.13' + CARGO_INCREMENTAL: '0' + +permissions: + contents: write + id-token: write + on: push: + branches: + - main tags: - "v*" + pull_request: + # This should trigger a dry run (we skip the final publish step) + paths: + - .github/workflows/npm-publish.yml + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - node: - name: vectordb Typescript - runs-on: ubuntu-latest - # Only runs on tags that matches the make-release action - if: startsWith(github.ref, 'refs/tags/v') - defaults: - run: - shell: bash - working-directory: node - steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 - with: - node-version: 20 - cache: "npm" - cache-dependency-path: node/package-lock.json - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler libssl-dev - - name: Build - run: | - npm ci - npm run tsc - npm pack - - name: Upload Linux Artifacts - uses: actions/upload-artifact@v4 - with: - name: node-package - path: | - node/vectordb-*.tgz - - node-macos: - name: vectordb ${{ matrix.config.arch }} - strategy: - matrix: - config: - - arch: x86_64-apple-darwin - runner: macos-13 - - arch: aarch64-apple-darwin - # xlarge is implicitly arm64. - runner: macos-14 - runs-on: ${{ matrix.config.runner }} - # Only runs on tags that matches the make-release action - if: startsWith(github.ref, 'refs/tags/v') - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install system dependencies - run: brew install protobuf - - name: Install npm dependencies - run: | - cd node - npm ci - - name: Build MacOS native node modules - run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }} - - name: Upload Darwin Artifacts - uses: actions/upload-artifact@v4 - with: - name: node-native-darwin-${{ matrix.config.arch }} - path: | - node/dist/lancedb-vectordb-darwin*.tgz - - nodejs-macos: - name: lancedb ${{ matrix.config.arch }} - strategy: - matrix: - config: - - arch: x86_64-apple-darwin - runner: macos-13 - - arch: aarch64-apple-darwin - # xlarge is implicitly arm64. - runner: macos-14 - runs-on: ${{ matrix.config.runner }} - # Only runs on tags that matches the make-release action - if: startsWith(github.ref, 'refs/tags/v') - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install system dependencies - run: brew install protobuf - - name: Install npm dependencies - run: | - cd nodejs - npm ci - - name: Build MacOS native nodejs modules - run: bash ci/build_macos_artifacts_nodejs.sh ${{ matrix.config.arch }} - - name: Upload Darwin Artifacts - uses: actions/upload-artifact@v4 - with: - name: nodejs-native-darwin-${{ matrix.config.arch }} - path: | - nodejs/dist/*.node - - node-linux-gnu: - name: vectordb (${{ matrix.config.arch}}-unknown-linux-gnu) - runs-on: ${{ matrix.config.runner }} - # Only runs on tags that matches the make-release action - if: startsWith(github.ref, 'refs/tags/v') - strategy: - fail-fast: false - matrix: - config: - - arch: x86_64 - runner: ubuntu-latest - - arch: aarch64 - # For successful fat LTO builds, we need a large runner to avoid OOM errors. - runner: warp-ubuntu-latest-arm64-4x - steps: - - name: Checkout - uses: actions/checkout@v4 - # To avoid OOM errors on ARM, we create a swap file. - - name: Configure aarch64 build - if: ${{ matrix.config.arch == 'aarch64' }} - run: | - free -h - sudo fallocate -l 16G /swapfile - sudo chmod 600 /swapfile - sudo mkswap /swapfile - sudo swapon /swapfile - echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab - # print info - swapon --show - free -h - - name: Build Linux Artifacts - run: | - bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-unknown-linux-gnu - - name: Upload Linux Artifacts - uses: actions/upload-artifact@v4 - with: - name: node-native-linux-${{ matrix.config.arch }}-gnu - path: | - node/dist/lancedb-vectordb-linux*.tgz - - nodejs-linux-gnu: - name: lancedb (${{ matrix.config.arch}}-unknown-linux-gnu - runs-on: ${{ matrix.config.runner }} - # Only runs on tags that matches the make-release action - if: startsWith(github.ref, 'refs/tags/v') - strategy: - fail-fast: false - matrix: - config: - - arch: x86_64 - runner: ubuntu-latest - - arch: aarch64 - # For successful fat LTO builds, we need a large runner to avoid OOM errors. - runner: buildjet-16vcpu-ubuntu-2204-arm - steps: - - name: Checkout - uses: actions/checkout@v4 - # Buildjet aarch64 runners have only 1.5 GB RAM per core, vs 3.5 GB per core for - # x86_64 runners. To avoid OOM errors on ARM, we create a swap file. - - name: Configure aarch64 build - if: ${{ matrix.config.arch == 'aarch64' }} - run: | - free -h - sudo fallocate -l 16G /swapfile - sudo chmod 600 /swapfile - sudo mkswap /swapfile - sudo swapon /swapfile - echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab - # print info - swapon --show - free -h - - name: Build Linux Artifacts - run: | - bash ci/build_linux_artifacts_nodejs.sh ${{ matrix.config.arch }} - - name: Upload Linux Artifacts - uses: actions/upload-artifact@v4 - with: - name: nodejs-native-linux-${{ matrix.config.arch }}-gnu - path: | - nodejs/dist/*.node - # The generic files are the same in all distros so we just pick - # one to do the upload. - - name: Upload Generic Artifacts - if: ${{ matrix.config.arch == 'x86_64' }} - uses: actions/upload-artifact@v4 - with: - name: nodejs-dist - path: | - nodejs/dist/* - !nodejs/dist/*.node - - nodejs-linux-musl: - name: lancedb (${{ matrix.config.arch}}-unknown-linux-musl - runs-on: ubuntu-latest - container: alpine:edge - # Only runs on tags that matches the make-release action - if: startsWith(github.ref, 'refs/tags/v') - strategy: - fail-fast: false - matrix: - config: - - arch: x86_64 - - arch: aarch64 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install common dependencies - run: | - apk add protobuf-dev curl clang mold grep npm bash - curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y - echo "source $HOME/.cargo/env" >> saved_env - echo "export CC=clang" >> saved_env - echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=-crt-static,+avx2,+fma,+f16c -Clinker=clang -Clink-arg=-fuse-ld=mold'" >> saved_env - - name: Configure aarch64 build - if: ${{ matrix.config.arch == 'aarch64' }} - run: | - source "$HOME/.cargo/env" - rustup target add aarch64-unknown-linux-musl - crt=$(realpath $(dirname $(rustup which rustc))/../lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained) - sysroot_lib=/usr/aarch64-unknown-linux-musl/usr/lib - apk_url=https://dl-cdn.alpinelinux.org/alpine/latest-stable/main/aarch64/ - curl -sSf $apk_url > apk_list - for pkg in gcc libgcc musl; do curl -sSf $apk_url$(cat apk_list | grep -oP '(?<=")'$pkg'-\d.*?(?=")') | tar zxf -; done - mkdir -p $sysroot_lib - echo 'GROUP ( libgcc_s.so.1 -lgcc )' > $sysroot_lib/libgcc_s.so - cp usr/lib/libgcc_s.so.1 $sysroot_lib - cp usr/lib/gcc/aarch64-alpine-linux-musl/*/libgcc.a $sysroot_lib - cp lib/ld-musl-aarch64.so.1 $sysroot_lib/libc.so - echo '!' > $sysroot_lib/libdl.a - (cd $crt && cp crti.o crtbeginS.o crtendS.o crtn.o -t $sysroot_lib) - echo "export CARGO_BUILD_TARGET=aarch64-unknown-linux-musl" >> saved_env - echo "export RUSTFLAGS='-Ctarget-feature=-crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=--target=aarch64-unknown-linux-musl -Clink-arg=--sysroot=/usr/aarch64-unknown-linux-musl -Clink-arg=-lc'" >> saved_env - - name: Build Linux Artifacts - run: | - source ./saved_env - bash ci/manylinux_node/build_lancedb.sh ${{ matrix.config.arch }} - - name: Upload Linux Artifacts - uses: actions/upload-artifact@v4 - with: - name: nodejs-native-linux-${{ matrix.config.arch }}-musl - path: | - nodejs/dist/*.node - - node-windows: - name: vectordb ${{ matrix.target }} - runs-on: windows-2022 - # Only runs on tags that matches the make-release action - if: startsWith(github.ref, 'refs/tags/v') - strategy: - fail-fast: false - matrix: - target: [x86_64-pc-windows-msvc] - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install Protoc v21.12 - working-directory: C:\ - run: | - New-Item -Path 'C:\protoc' -ItemType Directory - Set-Location C:\protoc - Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip - 7z x protoc.zip - Add-Content $env:GITHUB_PATH "C:\protoc\bin" - shell: powershell - - name: Install npm dependencies - run: | - cd node - npm ci - - name: Build Windows native node modules - run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }} - - name: Upload Windows Artifacts - uses: actions/upload-artifact@v4 - with: - name: node-native-windows - path: | - node/dist/lancedb-vectordb-win32*.tgz - - nodejs-windows: - name: lancedb ${{ matrix.target }} - runs-on: windows-2022 - # Only runs on tags that matches the make-release action - if: startsWith(github.ref, 'refs/tags/v') - strategy: - fail-fast: false - matrix: - target: [x86_64-pc-windows-msvc] - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install Protoc v21.12 - working-directory: C:\ - run: | - New-Item -Path 'C:\protoc' -ItemType Directory - Set-Location C:\protoc - Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip - 7z x protoc.zip - Add-Content $env:GITHUB_PATH "C:\protoc\bin" - shell: powershell - - name: Install npm dependencies - run: | - cd nodejs - npm ci - - name: Build Windows native node modules - run: .\ci\build_windows_artifacts_nodejs.ps1 ${{ matrix.target }} - - name: Upload Windows Artifacts - uses: actions/upload-artifact@v4 - with: - name: nodejs-native-windows - path: | - nodejs/dist/*.node - - nodejs-windows-arm64: - name: lancedb ${{ matrix.config.arch }}-pc-windows-msvc - # Only runs on tags that matches the make-release action - # if: startsWith(github.ref, 'refs/tags/v') - runs-on: ubuntu-latest - container: alpine:edge - strategy: - fail-fast: false - matrix: - config: - # - arch: x86_64 - - arch: aarch64 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install dependencies - run: | - apk add protobuf-dev curl clang lld llvm19 grep npm bash msitools sed - curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y - echo "source $HOME/.cargo/env" >> saved_env - echo "export CC=clang" >> saved_env - echo "export AR=llvm-ar" >> saved_env - source "$HOME/.cargo/env" - rustup target add ${{ matrix.config.arch }}-pc-windows-msvc - (mkdir -p sysroot && cd sysroot && sh ../ci/sysroot-${{ matrix.config.arch }}-pc-windows-msvc.sh) - echo "export C_INCLUDE_PATH=/usr/${{ matrix.config.arch }}-pc-windows-msvc/usr/include" >> saved_env - echo "export CARGO_BUILD_TARGET=${{ matrix.config.arch }}-pc-windows-msvc" >> saved_env - printf '#!/bin/sh\ncargo "$@"' > $HOME/.cargo/bin/cargo-xwin - chmod u+x $HOME/.cargo/bin/cargo-xwin - - name: Configure x86_64 build - if: ${{ matrix.config.arch == 'x86_64' }} - run: | - echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=+crt-static,+avx2,+fma,+f16c -Clinker=lld -Clink-arg=/LIBPATH:/usr/x86_64-pc-windows-msvc/usr/lib'" >> saved_env - - name: Configure aarch64 build - if: ${{ matrix.config.arch == 'aarch64' }} - run: | - echo "export RUSTFLAGS='-Ctarget-feature=+crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=lld -Clink-arg=/LIBPATH:/usr/aarch64-pc-windows-msvc/usr/lib -Clink-arg=arm64rt.lib'" >> saved_env - - name: Build Windows Artifacts - run: | - source ./saved_env - bash ci/manylinux_node/build_lancedb.sh ${{ matrix.config.arch }} - - name: Upload Windows Artifacts - uses: actions/upload-artifact@v4 - with: - name: nodejs-native-windows-${{ matrix.config.arch }} - path: | - nodejs/dist/*.node - - release: - name: vectordb NPM Publish - needs: [node, node-macos, node-linux-gnu, node-windows] - runs-on: ubuntu-latest - # Only runs on tags that matches the make-release action - if: startsWith(github.ref, 'refs/tags/v') - steps: - - uses: actions/download-artifact@v4 - with: - pattern: node-* - - name: Display structure of downloaded files - run: ls -R - - uses: actions/setup-node@v3 - with: - node-version: 20 - registry-url: "https://registry.npmjs.org" - - name: Publish to NPM - env: - NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }} - run: | - # Tag beta as "preview" instead of default "latest". See lancedb - # npm publish step for more info. - if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then - PUBLISH_ARGS="--tag preview" - fi - - mv */*.tgz . - for filename in *.tgz; do - npm publish $PUBLISH_ARGS $filename - done - - name: Notify Slack Action - uses: ravsamhq/notify-slack-action@2.3.0 - if: ${{ always() }} - with: - status: ${{ job.status }} - notify_when: "failure" - notification_title: "{workflow} is failing" - env: - SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} - - release-nodejs: - name: lancedb NPM Publish - needs: [nodejs-macos, nodejs-linux-gnu, nodejs-linux-musl, nodejs-windows, nodejs-windows-arm64] - runs-on: ubuntu-latest - # Only runs on tags that matches the make-release action - if: startsWith(github.ref, 'refs/tags/v') - defaults: - run: - shell: bash - working-directory: nodejs - steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - name: nodejs-dist - path: nodejs/dist - - uses: actions/download-artifact@v4 - name: Download arch-specific binaries - with: - pattern: nodejs-* - path: nodejs/nodejs-artifacts - merge-multiple: true - - name: Display structure of downloaded files - run: find . - - uses: actions/setup-node@v3 - with: - node-version: 20 - registry-url: "https://registry.npmjs.org" - - name: Install napi-rs - run: npm install -g @napi-rs/cli - - name: Prepare artifacts - run: npx napi artifacts -d nodejs-artifacts - - name: Display structure of staged files - run: find npm - - name: Publish to NPM - env: - NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }} - # By default, things are published to the latest tag. This is what is - # installed by default if the user does not specify a version. This is - # good for stable releases, but for pre-releases, we want to publish to - # the "preview" tag so they can install with `npm install lancedb@preview`. - # See: https://medium.com/@mbostock/prereleases-and-npm-e778fc5e2420 - run: | - if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then - npm publish --access public --tag preview - else - npm publish --access public - fi - - name: Notify Slack Action - uses: ravsamhq/notify-slack-action@2.3.0 - if: ${{ always() }} - with: - status: ${{ job.status }} - notify_when: "failure" - notification_title: "{workflow} is failing" - env: - SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} - - update-package-lock: - if: startsWith(github.ref, 'refs/tags/v') - needs: [release] - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: main - token: ${{ secrets.LANCEDB_RELEASE_TOKEN }} - fetch-depth: 0 - lfs: true - - uses: ./.github/workflows/update_package_lock - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - - update-package-lock-nodejs: - if: startsWith(github.ref, 'refs/tags/v') - needs: [release-nodejs] - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: main - token: ${{ secrets.LANCEDB_RELEASE_TOKEN }} - fetch-depth: 0 - lfs: true - - uses: ./.github/workflows/update_package_lock_nodejs - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - gh-release: if: startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest @@ -561,3 +90,474 @@ jobs: generate_release_notes: false name: Node/Rust LanceDB v${{ steps.extract_version.outputs.version }} body: ${{ steps.release_notes.outputs.changelog }} + + build-lancedb: + strategy: + fail-fast: false + matrix: + settings: + - target: x86_64-apple-darwin + host: macos-latest + features: "," + pre_build: |- + brew install protobuf + rustup target add x86_64-apple-darwin + - target: aarch64-apple-darwin + host: macos-latest + features: fp16kernels + pre_build: brew install protobuf + - target: x86_64-pc-windows-msvc + host: windows-latest + features: "," + pre_build: |- + choco install --no-progress protoc ninja nasm + tail -n 1000 /c/ProgramData/chocolatey/logs/chocolatey.log + # There is an issue where choco doesn't add nasm to the path + export PATH="$PATH:/c/Program Files/NASM" + nasm -v + - target: aarch64-pc-windows-msvc + host: windows-latest + features: "," + pre_build: |- + choco install --no-progress protoc + rustup target add aarch64-pc-windows-msvc + - target: x86_64-unknown-linux-gnu + host: ubuntu-latest + features: fp16kernels + # https://github.com/napi-rs/napi-rs/blob/main/debian.Dockerfile + docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian + pre_build: |- + set -e && + apt-get update && + apt-get install -y protobuf-compiler pkg-config + + # TODO: re-enable x64 musl builds. I could not figure out why, but it + # consistently made GHA runners non-responsive at the end of build. Example: + # https://github.com/lancedb/lancedb/actions/runs/13980431071/job/39144319470?pr=2250 + + # - target: x86_64-unknown-linux-musl + # # This one seems to need some extra memory + # host: ubuntu-2404-8x-x64 + # # https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile + # docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine + # features: "," + # pre_build: |- + # set -e && + # apk add protobuf-dev curl && + # ln -s /usr/lib/gcc/x86_64-alpine-linux-musl/14.2.0/crtbeginS.o /usr/lib/crtbeginS.o && + # ln -s /usr/lib/libgcc_s.so /usr/lib/libgcc.so + + - target: aarch64-unknown-linux-gnu + host: ubuntu-2404-8x-x64 + # https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile + docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64 + # TODO: enable fp16kernels after https://github.com/lancedb/lance/pull/3559 + features: "," + pre_build: |- + set -e && + apt-get update && + apt-get install -y protobuf-compiler pkg-config && + # https://github.com/aws/aws-lc-rs/issues/737#issuecomment-2725918627 + ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc/aarch64-unknown-linux-gnu/4.8.5/crtbeginS.o /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/crtbeginS.o && + ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/gcc && + rustup target add aarch64-unknown-linux-gnu + - target: aarch64-unknown-linux-musl + host: ubuntu-2404-8x-x64 + # https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile + docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine + features: "," + pre_build: |- + set -e && + apk add protobuf-dev && + rustup target add aarch64-unknown-linux-musl && + export CC="/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc" && + export CXX="/aarch64-linux-musl-cross/bin/aarch64-linux-musl-g++" + name: build - ${{ matrix.settings.target }} + runs-on: ${{ matrix.settings.host }} + defaults: + run: + working-directory: nodejs + steps: + - uses: actions/checkout@v4 + - name: Setup node + uses: actions/setup-node@v4 + if: ${{ !matrix.settings.docker }} + with: + node-version: 20 + cache: npm + cache-dependency-path: nodejs/package-lock.json + - name: Install + uses: dtolnay/rust-toolchain@stable + if: ${{ !matrix.settings.docker }} + with: + toolchain: stable + targets: ${{ matrix.settings.target }} + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + .cargo-cache + target/ + key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }} + - name: Setup toolchain + run: ${{ matrix.settings.setup }} + if: ${{ matrix.settings.setup }} + shell: bash + - name: Install dependencies + run: npm ci + - name: Build in docker + uses: addnab/docker-run-action@v3 + if: ${{ matrix.settings.docker }} + with: + image: ${{ matrix.settings.docker }} + options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \ + -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache \ + -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index \ + -v ${{ github.workspace }}:/build -w /build/nodejs" + run: | + set -e + ${{ matrix.settings.pre_build }} + npx napi build --platform --release --no-const-enum \ + --features ${{ matrix.settings.features }} \ + --target ${{ matrix.settings.target }} \ + --dts ../lancedb/native.d.ts \ + --js ../lancedb/native.js \ + --strip \ + dist/ + - name: Build + run: | + ${{ matrix.settings.pre_build }} + npx napi build --platform --release --no-const-enum \ + --features ${{ matrix.settings.features }} \ + --target ${{ matrix.settings.target }} \ + --dts ../lancedb/native.d.ts \ + --js ../lancedb/native.js \ + --strip \ + $EXTRA_ARGS \ + dist/ + if: ${{ !matrix.settings.docker }} + shell: bash + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: lancedb-${{ matrix.settings.target }} + path: nodejs/dist/*.node + if-no-files-found: error + # The generic files are the same in all distros so we just pick + # one to do the upload. + - name: Make generic artifacts + if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }} + run: npm run tsc + - name: Upload Generic Artifacts + if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }} + uses: actions/upload-artifact@v4 + with: + name: nodejs-dist + path: | + nodejs/dist/* + !nodejs/dist/*.node + test-lancedb: + name: "Test: ${{ matrix.settings.target }} - node@${{ matrix.node }}" + needs: + - build-lancedb + strategy: + fail-fast: false + matrix: + settings: + # TODO: Get tests passing on Windows (failing from test tmpdir issue) + # - host: windows-latest + # target: x86_64-pc-windows-msvc + - host: macos-latest + target: aarch64-apple-darwin + - target: x86_64-unknown-linux-gnu + host: ubuntu-latest + - target: aarch64-unknown-linux-gnu + host: buildjet-16vcpu-ubuntu-2204-arm + node: + - '20' + runs-on: ${{ matrix.settings.host }} + defaults: + run: + shell: bash + working-directory: nodejs + steps: + - uses: actions/checkout@v4 + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + cache: npm + cache-dependency-path: nodejs/package-lock.json + - name: Install dependencies + run: npm ci + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: lancedb-${{ matrix.settings.target }} + path: nodejs/dist/ + # For testing purposes: + # run-id: 13982782871 + # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo + - uses: actions/download-artifact@v4 + with: + name: nodejs-dist + path: nodejs/dist + # For testing purposes: + # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo + # run-id: 13982782871 + - name: List packages + run: ls -R dist + - name: Move built files + run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/ + - name: Test bindings + run: npm test + publish: + name: Publish + runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: nodejs + needs: + - test-lancedb + steps: + - uses: actions/checkout@v4 + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + cache-dependency-path: nodejs/package-lock.json + - name: Install dependencies + run: npm ci + - uses: actions/download-artifact@v4 + with: + name: nodejs-dist + path: nodejs/dist + # For testing purposes: + # run-id: 13982782871 + # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo + - uses: actions/download-artifact@v4 + name: Download arch-specific binaries + with: + pattern: lancedb-* + path: nodejs/nodejs-artifacts + merge-multiple: true + # For testing purposes: + # run-id: 13982782871 + # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo + - name: Display structure of downloaded files + run: find dist && find nodejs-artifacts + - name: Move artifacts + run: npx napi artifacts -d nodejs-artifacts + - name: List packages + run: find npm + - name: Publish + env: + NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }} + DRY_RUN: ${{ !startsWith(github.ref, 'refs/tags/v') }} + run: | + ARGS="--access public" + if [[ $DRY_RUN == "true" ]]; then + ARGS="$ARGS --dry-run" + fi + if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then + ARGS="$ARGS --tag preview" + fi + npm publish $ARGS + + + # ---------------------------------------------------------------------------- + # vectordb release (legacy) + # ---------------------------------------------------------------------------- + # TODO: delete this when we drop vectordb + node: + name: vectordb Typescript + runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: node + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/setup-node@v3 + with: + node-version: 20 + cache: "npm" + cache-dependency-path: node/package-lock.json + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y protobuf-compiler libssl-dev + - name: Build + run: | + npm ci + npm run tsc + npm pack + - name: Upload Linux Artifacts + uses: actions/upload-artifact@v4 + with: + name: node-package + path: | + node/vectordb-*.tgz + + node-macos: + name: vectordb ${{ matrix.config.arch }} + strategy: + matrix: + config: + - arch: x86_64-apple-darwin + runner: macos-13 + - arch: aarch64-apple-darwin + # xlarge is implicitly arm64. + runner: macos-14 + runs-on: ${{ matrix.config.runner }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install system dependencies + run: brew install protobuf + - name: Install npm dependencies + run: | + cd node + npm ci + - name: Build MacOS native node modules + run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }} + - name: Upload Darwin Artifacts + uses: actions/upload-artifact@v4 + with: + name: node-native-darwin-${{ matrix.config.arch }} + path: | + node/dist/lancedb-vectordb-darwin*.tgz + + node-linux-gnu: + name: vectordb (${{ matrix.config.arch}}-unknown-linux-gnu) + runs-on: ${{ matrix.config.runner }} + strategy: + fail-fast: false + matrix: + config: + - arch: x86_64 + runner: ubuntu-latest + - arch: aarch64 + # For successful fat LTO builds, we need a large runner to avoid OOM errors. + runner: warp-ubuntu-latest-arm64-4x + steps: + - name: Checkout + uses: actions/checkout@v4 + # To avoid OOM errors on ARM, we create a swap file. + - name: Configure aarch64 build + if: ${{ matrix.config.arch == 'aarch64' }} + run: | + free -h + sudo fallocate -l 16G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab + # print info + swapon --show + free -h + - name: Build Linux Artifacts + run: | + bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-unknown-linux-gnu + - name: Upload Linux Artifacts + uses: actions/upload-artifact@v4 + with: + name: node-native-linux-${{ matrix.config.arch }}-gnu + path: | + node/dist/lancedb-vectordb-linux*.tgz + + node-windows: + name: vectordb ${{ matrix.target }} + runs-on: windows-2022 + strategy: + fail-fast: false + matrix: + target: [x86_64-pc-windows-msvc] + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install Protoc v21.12 + working-directory: C:\ + run: | + New-Item -Path 'C:\protoc' -ItemType Directory + Set-Location C:\protoc + Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip + 7z x protoc.zip + Add-Content $env:GITHUB_PATH "C:\protoc\bin" + shell: powershell + - name: Install npm dependencies + run: | + cd node + npm ci + - name: Build Windows native node modules + run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }} + - name: Upload Windows Artifacts + uses: actions/upload-artifact@v4 + with: + name: node-native-windows + path: | + node/dist/lancedb-vectordb-win32*.tgz + + release: + name: vectordb NPM Publish + needs: [node, node-macos, node-linux-gnu, node-windows] + runs-on: ubuntu-latest + # Only runs on tags that matches the make-release action + if: startsWith(github.ref, 'refs/tags/v') + steps: + - uses: actions/download-artifact@v4 + with: + pattern: node-* + - name: Display structure of downloaded files + run: ls -R + - uses: actions/setup-node@v3 + with: + node-version: 20 + registry-url: "https://registry.npmjs.org" + - name: Publish to NPM + env: + NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }} + run: | + # Tag beta as "preview" instead of default "latest". See lancedb + # npm publish step for more info. + if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then + PUBLISH_ARGS="--tag preview" + fi + + mv */*.tgz . + for filename in *.tgz; do + npm publish $PUBLISH_ARGS $filename + done + - name: Notify Slack Action + uses: ravsamhq/notify-slack-action@2.3.0 + if: ${{ always() }} + with: + status: ${{ job.status }} + notify_when: "failure" + notification_title: "{workflow} is failing" + env: + SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} + + update-package-lock: + if: startsWith(github.ref, 'refs/tags/v') + needs: [release] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: main + token: ${{ secrets.LANCEDB_RELEASE_TOKEN }} + fetch-depth: 0 + lfs: true + - uses: ./.github/workflows/update_package_lock + with: + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/Cargo.lock b/Cargo.lock index 17658e3b..cbe92007 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -433,7 +433,7 @@ dependencies = [ "bytes", "derive_builder", "futures", - "rand 0.8.5", + "rand", "reqwest", "reqwest-eventsource", "secrecy", @@ -1014,7 +1014,7 @@ dependencies = [ "getrandom 0.2.15", "instant", "pin-project-lite", - "rand 0.8.5", + "rand", "tokio", ] @@ -1168,9 +1168,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.7.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17679a8d69b6d7fd9cd9801a536cec9fa5e5970b69f9d4747f70b39b031f5e7" +checksum = "675f87afced0413c9bb02843499dbbd3882a237645883f71a2b59644a6d2f753" dependencies = [ "arrayref", "arrayvec", @@ -1269,7 +1269,7 @@ dependencies = [ "memmap2 0.9.5", "num-traits", "num_cpus", - "rand 0.8.5", + "rand", "rand_distr", "rayon", "safetensors", @@ -1304,7 +1304,7 @@ dependencies = [ "candle-nn", "fancy-regex", "num-traits", - "rand 0.8.5", + "rand", "rayon", "serde", "serde_json", @@ -1688,7 +1688,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" dependencies = [ "generic-array", - "rand_core 0.6.4", + "rand_core", "subtle", "zeroize", ] @@ -1699,7 +1699,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "rand_core 0.6.4", + "rand_core", "subtle", ] @@ -1834,7 +1834,7 @@ dependencies = [ "log", "object_store", "parking_lot", - "rand 0.8.5", + "rand", "regex", "sqlparser 0.53.0", "tempfile", @@ -1919,7 +1919,7 @@ dependencies = [ "log", "object_store", "parking_lot", - "rand 0.8.5", + "rand", "tempfile", "url", ] @@ -1979,7 +1979,7 @@ dependencies = [ "itertools 0.14.0", "log", "md-5", - "rand 0.8.5", + "rand", "regex", "sha2", "unicode-segmentation", @@ -2261,9 +2261,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.4.0" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", "serde", @@ -2414,7 +2414,7 @@ dependencies = [ "generic-array", "group", "pkcs8", - "rand_core 0.6.4", + "rand_core", "sec1", "subtle", "zeroize", @@ -2597,7 +2597,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" dependencies = [ - "rand_core 0.6.4", + "rand_core", "subtle", ] @@ -2675,7 +2675,7 @@ name = "fsst" version = "0.25.0" source = "git+https://github.com/lancedb/lance.git?tag=v0.25.0-beta.5#f02095ddb4a57f15769e028919f343e38866d155" dependencies = [ - "rand 0.8.5", + "rand", ] [[package]] @@ -2935,16 +2935,14 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" dependencies = [ "cfg-if", - "js-sys", "libc", - "r-efi", - "wasi 0.14.2+wasi-0.2.4", - "wasm-bindgen", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets 0.52.6", ] [[package]] @@ -2966,7 +2964,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" dependencies = [ "ff", - "rand_core 0.6.4", + "rand_core", "subtle", ] @@ -3018,7 +3016,7 @@ dependencies = [ "cfg-if", "crunchy", "num-traits", - "rand 0.8.5", + "rand", "rand_distr", ] @@ -3087,7 +3085,7 @@ dependencies = [ "libc", "log", "num_cpus", - "rand 0.8.5", + "rand", "reqwest", "serde", "serde_json", @@ -3695,7 +3693,7 @@ dependencies = [ "pin-project", "prost", "prost-types", - "rand 0.8.5", + "rand", "roaring", "serde", "serde_json", @@ -3723,7 +3721,7 @@ dependencies = [ "getrandom 0.2.15", "half", "num-traits", - "rand 0.8.5", + "rand", ] [[package]] @@ -3752,7 +3750,7 @@ dependencies = [ "object_store", "pin-project", "prost", - "rand 0.8.5", + "rand", "roaring", "serde_json", "snafu", @@ -3821,7 +3819,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.8.5", + "rand", "seq-macro", "snafu", "tokio", @@ -3904,7 +3902,7 @@ dependencies = [ "object_store", "prost", "prost-build", - "rand 0.8.5", + "rand", "rayon", "roaring", "serde", @@ -3948,7 +3946,7 @@ dependencies = [ "path_abs", "pin-project", "prost", - "rand 0.8.5", + "rand", "shellexpand", "snafu", "tokio", @@ -3974,7 +3972,7 @@ dependencies = [ "lazy_static", "log", "num-traits", - "rand 0.8.5", + "rand", "rayon", "tokio", "tracing", @@ -4008,7 +4006,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.8.5", + "rand", "rangemap", "roaring", "serde", @@ -4029,7 +4027,7 @@ dependencies = [ "arrow-schema", "lance-arrow", "num-traits", - "rand 0.8.5", + "rand", ] [[package]] @@ -4085,7 +4083,7 @@ dependencies = [ "pin-project", "polars", "polars-arrow", - "rand 0.8.5", + "rand", "random_word", "regex", "reqwest", @@ -4886,7 +4884,7 @@ dependencies = [ "parking_lot", "percent-encoding", "quick-xml", - "rand 0.8.5", + "rand", "reqwest", "ring", "rustls-pemfile 2.2.0", @@ -5087,7 +5085,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared", - "rand 0.8.5", + "rand", ] [[package]] @@ -5259,7 +5257,7 @@ dependencies = [ "polars-error", "polars-row", "polars-utils", - "rand 0.8.5", + "rand", "rand_distr", "rayon", "regex", @@ -5458,7 +5456,7 @@ dependencies = [ "polars-error", "polars-lazy", "polars-plan", - "rand 0.8.5", + "rand", "serde", "serde_json", "sqlparser 0.39.0", @@ -5736,12 +5734,11 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.7" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3bd15a6f2967aef83887dcb9fec0014580467e33720d073560cf015a5683012" +checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" dependencies = [ "bytes", - "cfg_aliases", "pin-project-lite", "quinn-proto", "quinn-udp", @@ -5751,18 +5748,17 @@ dependencies = [ "thiserror 2.0.12", "tokio", "tracing", - "web-time", ] [[package]] name = "quinn-proto" -version = "0.11.10" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc" +checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" dependencies = [ "bytes", - "getrandom 0.3.2", - "rand 0.9.0", + "getrandom 0.2.15", + "rand", "ring", "rustc-hash 2.1.1", "rustls 0.23.25", @@ -5797,12 +5793,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "r-efi" -version = "5.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" - [[package]] name = "radium" version = "0.7.0" @@ -5816,19 +5806,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" -dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.3", - "zerocopy 0.8.23", + "rand_chacha", + "rand_core", ] [[package]] @@ -5838,17 +5817,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.3", + "rand_core", ] [[package]] @@ -5860,15 +5829,6 @@ dependencies = [ "getrandom 0.2.15", ] -[[package]] -name = "rand_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" -dependencies = [ - "getrandom 0.3.2", -] - [[package]] name = "rand_distr" version = "0.4.3" @@ -5876,7 +5836,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" dependencies = [ "num-traits", - "rand 0.8.5", + "rand", ] [[package]] @@ -5889,7 +5849,7 @@ dependencies = [ "brotli", "once_cell", "paste", - "rand 0.8.5", + "rand", "unicase", ] @@ -6052,9 +6012,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.15" +version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" +checksum = "989e327e510263980e231de548a33e63d34962d29ae61b467389a1a09627a254" dependencies = [ "async-compression", "base64 0.22.1", @@ -6234,9 +6194,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.3" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" +checksum = "f7178faa4b75a30e269c71e61c353ce2748cf3d76f0c44c393f4e60abf49b825" dependencies = [ "bitflags 2.9.0", "errno", @@ -6640,7 +6600,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" dependencies = [ "digest", - "rand_core 0.6.4", + "rand_core", ] [[package]] @@ -6792,9 +6752,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stacker" -version = "0.1.20" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601f9201feb9b09c00266478bf459952b9ef9a6b94edb2f21eba14ab681a60a9" +checksum = "d9156ebd5870ef293bfb43f91c7a74528d363ec0d424afe24160ed5a4343d08a" dependencies = [ "cc", "cfg-if", @@ -7136,14 +7096,14 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.19.1" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" +checksum = "488960f40a3fd53d72c2a29a58722561dee8afdd175bd88e3db4677d7b2ba600" dependencies = [ "fastrand", - "getrandom 0.3.2", + "getrandom 0.3.1", "once_cell", - "rustix 1.0.3", + "rustix 1.0.2", "windows-sys 0.59.0", ] @@ -7199,9 +7159,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.40" +version = "0.3.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d9c75b47bdff86fa3334a3db91356b8d7d86a9b839dab7d0bdc5c3d3a077618" +checksum = "dad298b01a40a23aac4580b67e3dbedb7cc8402f3592d7f49469de2ea4aecdd8" dependencies = [ "deranged", "itoa", @@ -7214,15 +7174,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.4" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" +checksum = "765c97a5b985b7c11d7bc27fa927dc4fe6af3a6dfb021d28deb60d3bf51e76ef" [[package]] name = "time-macros" -version = "0.2.21" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29aa485584182073ed57fd5004aa09c371f021325014694e432313345865fd04" +checksum = "e8093bc3e81c3bc5f7879de09619d06c9a5a5e45ca44dfeeb7225bae38005c5c" dependencies = [ "num-conv", "time-core", @@ -7280,7 +7240,7 @@ dependencies = [ "monostate", "onig", "paste", - "rand 0.8.5", + "rand", "rayon", "rayon-cond", "regex", @@ -7619,7 +7579,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" dependencies = [ - "getrandom 0.3.2", + "getrandom 0.3.1", "serde", ] @@ -7668,9 +7628,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasi" -version = "0.14.2+wasi-0.2.4" +version = "0.13.3+wasi-0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" dependencies = [ "wit-bindgen-rt", ] @@ -7897,9 +7857,9 @@ dependencies = [ [[package]] name = "windows-link" -version = "0.1.1" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" +checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3" [[package]] name = "windows-registry" @@ -7907,7 +7867,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ - "windows-result 0.3.2", + "windows-result 0.3.1", "windows-strings 0.3.1", "windows-targets 0.53.0", ] @@ -7923,9 +7883,9 @@ dependencies = [ [[package]] name = "windows-result" -version = "0.3.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +checksum = "06374efe858fab7e4f881500e6e86ec8bc28f9462c47e5a9941a0142ad86b189" dependencies = [ "windows-link", ] @@ -8238,9 +8198,9 @@ dependencies = [ [[package]] name = "wit-bindgen-rt" -version = "0.39.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" dependencies = [ "bitflags 2.9.0", ] diff --git a/ci/build_linux_artifacts_nodejs.sh b/ci/build_linux_artifacts_nodejs.sh deleted file mode 100755 index e47f5365..00000000 --- a/ci/build_linux_artifacts_nodejs.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -set -e -ARCH=${1:-x86_64} - -# We pass down the current user so that when we later mount the local files -# into the container, the files are accessible by the current user. -pushd ci/manylinux_node -docker build \ - -t lancedb-node-manylinux-$ARCH \ - --build-arg="ARCH=$ARCH" \ - --build-arg="DOCKER_USER=$(id -u)" \ - --progress=plain \ - . -popd - -# We turn on memory swap to avoid OOM killer -docker run \ - -v $(pwd):/io -w /io \ - --memory-swap=-1 \ - lancedb-node-manylinux-$ARCH \ - bash ci/manylinux_node/build_lancedb.sh $ARCH diff --git a/ci/build_macos_artifacts_nodejs.sh b/ci/build_macos_artifacts_nodejs.sh deleted file mode 100644 index 8deb2834..00000000 --- a/ci/build_macos_artifacts_nodejs.sh +++ /dev/null @@ -1,34 +0,0 @@ -# Builds the macOS artifacts (nodejs binaries). -# Usage: ./ci/build_macos_artifacts_nodejs.sh [target] -# Targets supported: x86_64-apple-darwin aarch64-apple-darwin -set -e - -prebuild_rust() { - # Building here for the sake of easier debugging. - pushd rust/lancedb - echo "Building rust library for $1" - export RUST_BACKTRACE=1 - cargo build --release --target $1 - popd -} - -build_node_binaries() { - pushd nodejs - echo "Building nodejs library for $1" - export RUST_TARGET=$1 - npm run build-release - popd -} - -if [ -n "$1" ]; then - targets=$1 -else - targets="x86_64-apple-darwin aarch64-apple-darwin" -fi - -echo "Building artifacts for targets: $targets" -for target in $targets - do - prebuild_rust $target - build_node_binaries $target -done diff --git a/ci/manylinux_node/Dockerfile b/ci/manylinux_node/Dockerfile index b96231b9..1d06d4f6 100644 --- a/ci/manylinux_node/Dockerfile +++ b/ci/manylinux_node/Dockerfile @@ -1,5 +1,5 @@ # Many linux dockerfile with Rust, Node, and Lance dependencies installed. -# This container allows building the node modules native libraries in an +# This container allows building the node modules native libraries in an # environment with a very old glibc, so that we are compatible with a wide # range of linux distributions. ARG ARCH=x86_64 @@ -9,10 +9,6 @@ FROM quay.io/pypa/manylinux_2_28_${ARCH} ARG ARCH=x86_64 ARG DOCKER_USER=default_user -# Install static openssl -COPY install_openssl.sh install_openssl.sh -RUN ./install_openssl.sh ${ARCH} > /dev/null - # Protobuf is also installed as root. COPY install_protobuf.sh install_protobuf.sh RUN ./install_protobuf.sh ${ARCH} @@ -21,7 +17,7 @@ ENV DOCKER_USER=${DOCKER_USER} # Create a group and user, but only if it doesn't exist RUN echo ${ARCH} && id -u ${DOCKER_USER} >/dev/null 2>&1 || adduser --user-group --create-home --uid ${DOCKER_USER} build_user -# We switch to the user to install Rust and Node, since those like to be +# We switch to the user to install Rust and Node, since those like to be # installed at the user level. USER ${DOCKER_USER} diff --git a/ci/manylinux_node/build_lancedb.sh b/ci/manylinux_node/build_lancedb.sh deleted file mode 100644 index 597319c4..00000000 --- a/ci/manylinux_node/build_lancedb.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# Builds the nodejs module for manylinux. Invoked by ci/build_linux_artifacts_nodejs.sh. -set -e -ARCH=${1:-x86_64} - -if [ "$ARCH" = "x86_64" ]; then - export OPENSSL_LIB_DIR=/usr/local/lib64/ -else - export OPENSSL_LIB_DIR=/usr/local/lib/ -fi -export OPENSSL_STATIC=1 -export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl - -#Alpine doesn't have .bashrc -FILE=$HOME/.bashrc && test -f $FILE && source $FILE - -cd nodejs -npm ci -npm run build-release diff --git a/ci/manylinux_node/build_vectordb.sh b/ci/manylinux_node/build_vectordb.sh index e5b81e89..8d703816 100755 --- a/ci/manylinux_node/build_vectordb.sh +++ b/ci/manylinux_node/build_vectordb.sh @@ -4,14 +4,6 @@ set -e ARCH=${1:-x86_64} TARGET_TRIPLE=${2:-x86_64-unknown-linux-gnu} -if [ "$ARCH" = "x86_64" ]; then - export OPENSSL_LIB_DIR=/usr/local/lib64/ -else - export OPENSSL_LIB_DIR=/usr/local/lib/ -fi -export OPENSSL_STATIC=1 -export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl - #Alpine doesn't have .bashrc FILE=$HOME/.bashrc && test -f $FILE && source $FILE diff --git a/ci/manylinux_node/install_openssl.sh b/ci/manylinux_node/install_openssl.sh deleted file mode 100755 index 87822316..00000000 --- a/ci/manylinux_node/install_openssl.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# Builds openssl from source so we can statically link to it - -# this is to avoid the error we get with the system installation: -# /usr/bin/ld: : version node not found for symbol SSLeay@@OPENSSL_1.0.1 -# /usr/bin/ld: failed to set dynamic section sizes: Bad value -set -e - -git clone -b OpenSSL_1_1_1v \ - --single-branch \ - https://github.com/openssl/openssl.git - -pushd openssl - -if [[ $1 == x86_64* ]]; then - ARCH=linux-x86_64 -else - # gnu target - ARCH=linux-aarch64 -fi - -./Configure no-shared $ARCH - -make - -make install \ No newline at end of file diff --git a/docs/src/js/README.md b/docs/src/js/README.md index 2728d523..91f61526 100644 --- a/docs/src/js/README.md +++ b/docs/src/js/README.md @@ -15,11 +15,9 @@ npm install @lancedb/lancedb This will download the appropriate native library for your platform. We currently support: -- Linux (x86_64 and aarch64) +- Linux (x86_64 and aarch64 on glibc and musl) - MacOS (Intel and ARM/M1/M2) -- Windows (x86_64 only) - -We do not yet support musl-based Linux (such as Alpine Linux) or aarch64 Windows. +- Windows (x86_64 and aarch64) ## Usage diff --git a/nodejs/.npmignore b/nodejs/.npmignore index 2d0d1d82..41f6643a 100644 --- a/nodejs/.npmignore +++ b/nodejs/.npmignore @@ -22,3 +22,4 @@ build.rs jest.config.js tsconfig.json typedoc.json +typedoc_post_process.js diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index e7400fc9..7960e01b 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -18,7 +18,7 @@ arrow-array.workspace = true arrow-schema.workspace = true env_logger.workspace = true futures.workspace = true -lancedb = { path = "../rust/lancedb", features = ["remote"] } +lancedb = { path = "../rust/lancedb" } napi = { version = "2.16.8", default-features = false, features = [ "napi9", "async" @@ -30,3 +30,8 @@ log.workspace = true [build-dependencies] napi-build = "2.1" + +[features] +default = ["remote"] +fp16kernels = ["lancedb/fp16kernels"] +remote = ["lancedb/remote"] diff --git a/nodejs/README.md b/nodejs/README.md index 4446855d..15df64c0 100644 --- a/nodejs/README.md +++ b/nodejs/README.md @@ -11,11 +11,9 @@ npm install @lancedb/lancedb This will download the appropriate native library for your platform. We currently support: -- Linux (x86_64 and aarch64) +- Linux (x86_64 and aarch64 on glibc and musl) - MacOS (Intel and ARM/M1/M2) -- Windows (x86_64 only) - -We do not yet support musl-based Linux (such as Alpine Linux) or aarch64 Windows. +- Windows (x86_64 and aarch64) ## Usage diff --git a/nodejs/package.json b/nodejs/package.json index 70c01881..26e96708 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -29,7 +29,6 @@ "aarch64-apple-darwin", "x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu", - "x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl", "x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc" @@ -74,8 +73,10 @@ "artifacts": "napi artifacts", "build:debug": "napi build --platform --no-const-enum --dts ../lancedb/native.d.ts --js ../lancedb/native.js lancedb", "build:release": "napi build --platform --no-const-enum --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js dist/", - "build": "npm run build:debug && tsc -b && shx cp lancedb/native.d.ts dist/native.d.ts && shx cp lancedb/*.node dist/", - "build-release": "npm run build:release && tsc -b && shx cp lancedb/native.d.ts dist/native.d.ts", + "build": "npm run build:debug && npm run tsc && shx cp lancedb/*.node dist/", + "build-release": "npm run build:release && npm run tsc", + "tsc": "tsc -b", + "posttsc": "shx cp lancedb/native.d.ts dist/native.d.ts", "lint-ci": "biome ci .", "docs": "typedoc --plugin typedoc-plugin-markdown --treatWarningsAsErrors --out ../docs/src/js lancedb/index.ts", "postdocs": "node typedoc_post_process.js",