rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-01-04 10:52:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(node) Move native packages to @lancedb NPM org (#341)

Browse Source 
- Move native packages to @lancedb org
- Move package-lock.json update to a reusable action and created a target to run it manually.
This commit is contained in:
gsilvestrin
2023-07-20 12:54:39 -07:00
committed by GitHub
parent 0cf40c8da3
commit a2bb497135
8 changed files with 138 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
.github/workflows/make-release-commit.yml vendored
View File

@@ -25,38 +25,25 @@ jobs:
bump-version:
runs-on: ubuntu-latest
steps:
- name: Check out main
uses: actions/checkout@v3
with:
ref: main
persist-credentials: false
fetch-depth: 0
lfs: true
- name: Set git configs for bumpversion
shell: bash
run: |
git config user.name 'Lance Release'
git config user.email 'lance-dev@lancedb.com'
- name: Set up Python 3.10
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Bump version, create tag and commit
run: |
pip install bump2version
bumpversion --verbose ${{ inputs.part }}
- name: Update package-lock.json file
run: |
npm install
git add package-lock.json
# Add this change to the commit created by bumpversion
git commit --amend --no-edit
working-directory: node
- name: Push new version and tag
if: ${{ inputs.dry_run }} == "false"
uses: ad-m/github-push-action@master
with:
github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
branch: main
tags: true
- name: Check out main
uses: actions/checkout@v3
with:
ref: main
fetch-depth: 0
lfs: true
- name: Set git configs for bumpversion
shell: bash
run: |
git config user.name 'Lance Release'
git config user.email 'lance-dev@lancedb.com'
- name: Set up Python 3.10
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Bump version, create tag and commit
run: |
pip install bump2version
bumpversion --verbose ${{ inputs.part }}
git push
- uses: ./.github/workflows/update_package_lock

4
.github/workflows/node.yml vendored
View File

@@ -70,7 +70,7 @@ jobs:
npm run tsc
npm run build
npm run pack-build
npm install --no-save ./dist/vectordb-*.tgz
npm install --no-save ./dist/lancedb-vectordb-*.tgz
# Remove index.node to test with dependency installed
rm index.node
- name: Test
@@ -101,7 +101,7 @@ jobs:
npm run tsc
npm run build
npm run pack-build
npm install --no-save ./dist/vectordb-*.tgz
npm install --no-save ./dist/lancedb-vectordb-*.tgz
# Remove index.node to test with dependency installed
rm index.node
- name: Test

116
.github/workflows/npm-publish.yml vendored
View File

@@ -35,7 +35,7 @@ jobs:
with:
name: node-package
path: |
node/vectordb-*.tgz
node/lancedb-vectordb-*.tgz
node-macos:
runs-on: macos-12
@@ -46,25 +46,25 @@ jobs:
matrix:
target: [x86_64-apple-darwin, aarch64-apple-darwin]
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install system dependencies
run: brew install protobuf
- name: Install npm dependencies
run: |
cd node
npm ci
- name: Install rustup target
if: ${{ matrix.target == 'aarch64-apple-darwin' }}
run: rustup target add aarch64-apple-darwin
- name: Build MacOS native node modules
run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
- name: Upload Darwin Artifacts
uses: actions/upload-artifact@v3
with:
name: darwin-native
path: |
node/dist/vectordb-darwin*.tgz
- name: Checkout
uses: actions/checkout@v3
- name: Install system dependencies
run: brew install protobuf
- name: Install npm dependencies
run: |
cd node
npm ci
- name: Install rustup target
if: ${{ matrix.target == 'aarch64-apple-darwin' }}
run: rustup target add aarch64-apple-darwin
- name: Build MacOS native node modules
run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
- name: Upload Darwin Artifacts
uses: actions/upload-artifact@v3
with:
name: native-darwin
path: |
node/dist/lancedb-vectordb-darwin*.tgz
node-linux:
name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }})
@@ -84,37 +84,37 @@ jobs:
# Building on aarch64 is too slow for now
# - aarch64
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Change owner to root (for npm)
# The docker container is run as root, so we need the files to be owned by root
# Otherwise npm is a nightmare: https://github.com/npm/cli/issues/3773
run: sudo chown -R root:root .
- name: Set up QEMU
if: ${{ matrix.arch == 'aarch64' }}
uses: docker/setup-qemu-action@v2
with:
platforms: arm64
- name: Build Linux GNU native node modules
if: ${{ matrix.libc == 'gnu' }}
run: |
docker run \
-v $(pwd):/io -w /io \
rust:1.70-bookworm \
bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu
- name: Build musl Linux native node modules
if: ${{ matrix.libc == 'musl' }}
run: |
docker run --platform linux/arm64/v8 \
-v $(pwd):/io -w /io \
quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
- name: Upload Linux Artifacts
uses: actions/upload-artifact@v3
with:
name: linux-native
path: |
node/dist/vectordb-linux*.tgz
- name: Checkout
uses: actions/checkout@v3
- name: Change owner to root (for npm)
# The docker container is run as root, so we need the files to be owned by root
# Otherwise npm is a nightmare: https://github.com/npm/cli/issues/3773
run: sudo chown -R root:root .
- name: Set up QEMU
if: ${{ matrix.arch == 'aarch64' }}
uses: docker/setup-qemu-action@v2
with:
platforms: arm64
- name: Build Linux GNU native node modules
if: ${{ matrix.libc == 'gnu' }}
run: |
docker run \
-v $(pwd):/io -w /io \
rust:1.70-bookworm \
bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu
- name: Build musl Linux native node modules
if: ${{ matrix.libc == 'musl' }}
run: |
docker run --platform linux/arm64/v8 \
-v $(pwd):/io -w /io \
quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
- name: Upload Linux Artifacts
uses: actions/upload-artifact@v3
with:
name: native-linux
path: |
node/dist/lancedb-vectordb-linux*.tgz
node-windows:
runs-on: windows-2022
@@ -145,12 +145,12 @@ jobs:
- name: Upload Windows Artifacts
uses: actions/upload-artifact@v3
with:
name: windows-native
name: native-windows
path: |
node/dist/vectordb-win32*.tgz
node/dist/lancedb-vectordb-win32*.tgz
release:
needs: [node, node-macos, node-linux]
needs: [node, node-macos, node-linux, node-windows]
runs-on: ubuntu-latest
# Only runs on tags that matches the make-release action
if: startsWith(github.ref, 'refs/tags/v')
@@ -170,3 +170,11 @@ jobs:
for filename in *.tgz; do
npm publish $filename
done
update-package-lock:
needs: [release]
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- uses: ./.github/workflows/update_package_lock

22
.github/workflows/update_package_lock/action.yml vendored Normal file
View File

@@ -0,0 +1,22 @@
name: update_package_lock
description: "Update node's package.lock"
runs:
using: "composite"
steps:
- uses: actions/setup-node@v3
with:
node-version: 20
- name: Set git configs
shell: bash
run: |
git config user.name 'Lance Release'
git config user.email 'lance-dev@lancedb.com'
- name: Update package-lock.json file
working-directory: ./node
run: |
npm install
git add package-lock.json
git commit -m "Updating package-lock.json"
git push
shell: bash

12
.github/workflows/update_package_lock_run.yml vendored Normal file
View File

@@ -0,0 +1,12 @@
name: Update package-lock.json
on:
workflow_dispatch:
jobs:
publish:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- uses: ./.github/workflows/update_package_lock

8
node/native.js
View File

@@ -17,7 +17,7 @@ const { currentTarget } = require('@neon-rs/load');
let nativeLib;
try {
nativeLib = require(`vectordb-${currentTarget()}`);
nativeLib = require(`@lancedb/vectordb-${currentTarget()}`);
} catch (e) {
try {
// Might be developing locally, so try that. But don't expose that error
@@ -25,12 +25,12 @@ try {
nativeLib = require("./index.node");
} catch {
throw new Error(`vectordb: failed to load native library.
You may need to run \`npm install vectordb-${currentTarget()}\`.
You may need to run \`npm install @lancedb/vectordb-${currentTarget()}\`.
If that does not work, please file a bug report at https://github.com/lancedb/lancedb/issues
Source error: ${e}`);
}
}
}
// Dynamic require for runtime.

64
node/package-lock.json generated
View File

@@ -51,11 +51,11 @@
"typescript": "*"
},
"optionalDependencies": {
"vectordb-darwin-arm64": "0.1.15",
"vectordb-darwin-x64": "0.1.15",
"vectordb-linux-arm64-gnu": "0.1.15",
"vectordb-linux-x64-gnu": "0.1.15",
"vectordb-win32-x64-msvc": "0.1.15"
"@lancedb/vectordb-darwin-arm64": "0.1.15",
"@lancedb/vectordb-darwin-x64": "0.1.15",
"@lancedb/vectordb-linux-arm64-gnu": "0.1.15",
"@lancedb/vectordb-linux-x64-gnu": "0.1.15",
"@lancedb/vectordb-win32-x64-msvc": "0.1.15"
}
},
"node_modules/@apache-arrow/ts": {
@@ -4297,42 +4297,6 @@
"integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==",
"dev": true
},
"node_modules/vectordb-darwin-arm64": {
"version": "0.1.15",
"resolved": "https://registry.npmjs.org/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.1.15.tgz",
"integrity": "sha512-15bX+ZMojTvLoMXmWRKj+Im4905B8c+2ZPt0xd0mdRnIcR0XKUq3XHgj5vqSN3xhAzAND/q7472J1Y/OXyYLeg==",
"cpu": [
"arm64"
],
"optional": true,
"os": [
"darwin"
]
},
"node_modules/vectordb-darwin-x64": {
"version": "0.1.15",
"resolved": "https://registry.npmjs.org/vectordb-darwin-x64/-/vectordb-darwin-x64-0.1.15.tgz",
"integrity": "sha512-jbub+YGto6UEn8omEz9XJTbrGTtZzsVydMTDBCWSSh/Cg3ZklkY0La1NNZPN5XPvmK3WcxxocxlPz5ttcLoZmA==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"darwin"
]
},
"node_modules/vectordb-linux-x64-gnu": {
"version": "0.1.15",
"resolved": "https://registry.npmjs.org/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.1.15.tgz",
"integrity": "sha512-IPnKbiEI4py+w81urN4XAMVYFn7tikvLZziY4z0bQrCHllNwnD2DVr61mpAEIGP7kHyXBdsVWEdyAkRGXzcxPw==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"linux"
]
},
"node_modules/vscode-oniguruma": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz",
@@ -7638,24 +7602,6 @@
"integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==",
"dev": true
},
"vectordb-darwin-arm64": {
"version": "0.1.15",
"resolved": "https://registry.npmjs.org/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.1.15.tgz",
"integrity": "sha512-15bX+ZMojTvLoMXmWRKj+Im4905B8c+2ZPt0xd0mdRnIcR0XKUq3XHgj5vqSN3xhAzAND/q7472J1Y/OXyYLeg==",
"optional": true
},
"vectordb-darwin-x64": {
"version": "0.1.15",
"resolved": "https://registry.npmjs.org/vectordb-darwin-x64/-/vectordb-darwin-x64-0.1.15.tgz",
"integrity": "sha512-jbub+YGto6UEn8omEz9XJTbrGTtZzsVydMTDBCWSSh/Cg3ZklkY0La1NNZPN5XPvmK3WcxxocxlPz5ttcLoZmA==",
"optional": true
},
"vectordb-linux-x64-gnu": {
"version": "0.1.15",
"resolved": "https://registry.npmjs.org/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.1.15.tgz",
"integrity": "sha512-IPnKbiEI4py+w81urN4XAMVYFn7tikvLZziY4z0bQrCHllNwnD2DVr61mpAEIGP7kHyXBdsVWEdyAkRGXzcxPw==",
"optional": true
},
"vscode-oniguruma": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz",

20
node/package.json
View File

@@ -70,18 +70,18 @@
],
"neon": {
"targets": {
"x86_64-apple-darwin": "vectordb-darwin-x64",
"aarch64-apple-darwin": "vectordb-darwin-arm64",
"x86_64-unknown-linux-gnu": "vectordb-linux-x64-gnu",
"aarch64-unknown-linux-gnu": "vectordb-linux-arm64-gnu",
"x86_64-pc-windows-msvc": "vectordb-win32-x64-msvc"
"x86_64-apple-darwin": "@lancedb/vectordb-darwin-x64",
"aarch64-apple-darwin": "@lancedb/vectordb-darwin-arm64",
"x86_64-unknown-linux-gnu": "@lancedb/vectordb-linux-x64-gnu",
"aarch64-unknown-linux-gnu": "@lancedb/vectordb-linux-arm64-gnu",
"x86_64-pc-windows-msvc": "@lancedb/vectordb-win32-x64-msvc"
}
},
"optionalDependencies": {
"vectordb-darwin-arm64": "0.1.15",
"vectordb-darwin-x64": "0.1.15",
"vectordb-linux-arm64-gnu": "0.1.15",
"vectordb-linux-x64-gnu": "0.1.15",
"vectordb-win32-x64-msvc": "0.1.15"
"@lancedb/vectordb-darwin-arm64": "0.1.15",
"@lancedb/vectordb-darwin-x64": "0.1.15",
"@lancedb/vectordb-linux-arm64-gnu": "0.1.15",
"@lancedb/vectordb-linux-x64-gnu": "0.1.15",
"@lancedb/vectordb-win32-x64-msvc": "0.1.15"
}
}