diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index faa94136..d3e1bb1d 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -2,7 +2,7 @@ name: NPM Publish on: release: - types: [ published ] + types: [published] jobs: node: @@ -19,7 +19,7 @@ jobs: - uses: actions/setup-node@v3 with: node-version: 20 - cache: 'npm' + cache: "npm" cache-dependency-path: node/package-lock.json - name: Install dependencies run: | @@ -31,7 +31,7 @@ jobs: npm run tsc npm pack - name: Upload Linux Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: node-package path: | @@ -61,12 +61,41 @@ jobs: - name: Build MacOS native node modules run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }} - name: Upload Darwin Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: native-darwin + name: node-native-darwin path: | node/dist/lancedb-vectordb-darwin*.tgz + nodejs-macos: + strategy: + matrix: + config: + - arch: x86_64-apple-darwin + runner: macos-13 + - arch: aarch64-apple-darwin + # xlarge is implicitly arm64. + runner: macos-14 + runs-on: ${{ matrix.config.runner }} + # Only runs on tags that matches the make-release action + if: startsWith(github.ref, 'refs/tags/v') + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install system dependencies + run: brew install protobuf + - name: Install npm dependencies + run: | + cd nodejs + npm ci + - name: Build MacOS native nodejs modules + run: bash ci/build_macos_artifacts_nodejs.sh ${{ matrix.config.arch }} + - name: Upload Darwin Artifacts + uses: actions/upload-artifact@v4 + with: + name: nodejs-native-darwin-${{ matrix.config.arch }} + path: | + nodejs/dist/*.node node-linux: name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu @@ -103,12 +132,63 @@ jobs: run: | bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }} - name: Upload Linux Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: native-linux + name: node-native-linux path: | node/dist/lancedb-vectordb-linux*.tgz + nodejs-linux: + name: nodejs-linux (${{ matrix.config.arch}}-unknown-linux-gnu + runs-on: ${{ matrix.config.runner }} + # Only runs on tags that matches the make-release action + if: startsWith(github.ref, 'refs/tags/v') + strategy: + fail-fast: false + matrix: + config: + - arch: x86_64 + runner: ubuntu-latest + - arch: aarch64 + # For successful fat LTO builds, we need a large runner to avoid OOM errors. + runner: buildjet-16vcpu-ubuntu-2204-arm + steps: + - name: Checkout + uses: actions/checkout@v4 + # Buildjet aarch64 runners have only 1.5 GB RAM per core, vs 3.5 GB per core for + # x86_64 runners. To avoid OOM errors on ARM, we create a swap file. + - name: Configure aarch64 build + if: ${{ matrix.config.arch == 'aarch64' }} + run: | + free -h + sudo fallocate -l 16G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab + # print info + swapon --show + free -h + - name: Build Linux Artifacts + run: | + bash ci/build_linux_artifacts_nodejs.sh ${{ matrix.config.arch }} + - name: Upload Linux Artifacts + uses: actions/upload-artifact@v4 + with: + name: nodejs-native-linux-${{ matrix.config.arch }} + path: | + nodejs/dist/*.node + # The generic files are the same in all distros so we just pick + # one to do the upload. + - name: Upload Generic Artifacts + if: ${{ matrix.config.arch == 'x86_64' }} + uses: actions/upload-artifact@v4 + with: + name: nodejs-dist + path: | + nodejs/dist/* + !nodejs/dist/*.node + node-windows: runs-on: windows-2022 # Only runs on tags that matches the make-release action @@ -136,25 +216,60 @@ jobs: - name: Build Windows native node modules run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }} - name: Upload Windows Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: native-windows + name: node-native-windows path: | node/dist/lancedb-vectordb-win32*.tgz + nodejs-windows: + runs-on: windows-2022 + # Only runs on tags that matches the make-release action + if: startsWith(github.ref, 'refs/tags/v') + strategy: + fail-fast: false + matrix: + target: [x86_64-pc-windows-msvc] + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install Protoc v21.12 + working-directory: C:\ + run: | + New-Item -Path 'C:\protoc' -ItemType Directory + Set-Location C:\protoc + Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip + 7z x protoc.zip + Add-Content $env:GITHUB_PATH "C:\protoc\bin" + shell: powershell + - name: Install npm dependencies + run: | + cd nodejs + npm ci + - name: Build Windows native node modules + run: .\ci\build_windows_artifacts_nodejs.ps1 ${{ matrix.target }} + - name: Upload Windows Artifacts + uses: actions/upload-artifact@v4 + with: + name: nodejs-native-windows + path: | + nodejs/dist/*.node + release: needs: [node, node-macos, node-linux, node-windows] runs-on: ubuntu-latest # Only runs on tags that matches the make-release action if: startsWith(github.ref, 'refs/tags/v') steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 + with: + pattern: node-* - name: Display structure of downloaded files run: ls -R - uses: actions/setup-node@v3 with: node-version: 20 - registry-url: 'https://registry.npmjs.org' + registry-url: "https://registry.npmjs.org" - name: Publish to NPM env: NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }} @@ -164,6 +279,45 @@ jobs: npm publish $filename done + release-nodejs: + needs: [nodejs-macos, nodejs-linux, nodejs-windows] + runs-on: ubuntu-latest + # Only runs on tags that matches the make-release action + if: startsWith(github.ref, 'refs/tags/v') + defaults: + run: + shell: bash + working-directory: nodejs + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: nodejs-dist + path: nodejs/dist + - uses: actions/download-artifact@v4 + name: Download arch-specific binaries + with: + pattern: nodejs-* + path: nodejs/nodejs-artifacts + merge-multiple: true + - name: Display structure of downloaded files + run: find . + - uses: actions/setup-node@v3 + with: + node-version: 20 + registry-url: "https://registry.npmjs.org" + - name: Install napi-rs + run: npm install -g @napi-rs/cli + - name: Prepare artifacts + run: npx napi artifacts -d nodejs-artifacts + - name: Display structure of staged files + run: find npm + - name: Publish to NPM + env: + NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }} + run: npm publish --access public + update-package-lock: needs: [release] runs-on: ubuntu-latest @@ -178,3 +332,18 @@ jobs: - uses: ./.github/workflows/update_package_lock with: github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }} + + update-package-lock-nodejs: + needs: [release-nodejs] + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: main + persist-credentials: false + fetch-depth: 0 + lfs: true + - uses: ./.github/workflows/update_package_lock_nodejs + with: + github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }} diff --git a/.github/workflows/update_package_lock_nodejs/action.yml b/.github/workflows/update_package_lock_nodejs/action.yml new file mode 100644 index 00000000..33746433 --- /dev/null +++ b/.github/workflows/update_package_lock_nodejs/action.yml @@ -0,0 +1,33 @@ +name: update_package_lock_nodejs +description: "Update nodejs's package.lock" + +inputs: + github_token: + required: true + description: "github token for the repo" + +runs: + using: "composite" + steps: + - uses: actions/setup-node@v3 + with: + node-version: 20 + - name: Set git configs + shell: bash + run: | + git config user.name 'Lance Release' + git config user.email 'lance-dev@lancedb.com' + - name: Update package-lock.json file + working-directory: ./nodejs + run: | + npm install + git add package-lock.json + git commit -m "Updating package-lock.json" + shell: bash + - name: Push changes + if: ${{ inputs.dry_run }} == "false" + uses: ad-m/github-push-action@master + with: + github_token: ${{ inputs.github_token }} + branch: main + tags: true diff --git a/.github/workflows/update_package_lock_run_nodejs.yml b/.github/workflows/update_package_lock_run_nodejs.yml new file mode 100644 index 00000000..e0bfda48 --- /dev/null +++ b/.github/workflows/update_package_lock_run_nodejs.yml @@ -0,0 +1,19 @@ +name: Update NodeJs package-lock.json + +on: + workflow_dispatch: + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: main + persist-credentials: false + fetch-depth: 0 + lfs: true + - uses: ./.github/workflows/update_package_lock_nodejs + with: + github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }} diff --git a/.gitignore b/.gitignore index 46e13e7b..da3b594f 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ python/dist node/dist node/examples/**/package-lock.json node/examples/**/dist +nodejs/lancedb/native* dist ## Rust diff --git a/ci/build_linux_artifacts_nodejs.sh b/ci/build_linux_artifacts_nodejs.sh new file mode 100755 index 00000000..65438223 --- /dev/null +++ b/ci/build_linux_artifacts_nodejs.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e +ARCH=${1:-x86_64} + +# We pass down the current user so that when we later mount the local files +# into the container, the files are accessible by the current user. +pushd ci/manylinux_nodejs +docker build \ + -t lancedb-nodejs-manylinux \ + --build-arg="ARCH=$ARCH" \ + --build-arg="DOCKER_USER=$(id -u)" \ + --progress=plain \ + . +popd + +# We turn on memory swap to avoid OOM killer +docker run \ + -v $(pwd):/io -w /io \ + --memory-swap=-1 \ + lancedb-nodejs-manylinux \ + bash ci/manylinux_nodejs/build.sh $ARCH diff --git a/ci/build_macos_artifacts_nodejs.sh b/ci/build_macos_artifacts_nodejs.sh new file mode 100644 index 00000000..8deb2834 --- /dev/null +++ b/ci/build_macos_artifacts_nodejs.sh @@ -0,0 +1,34 @@ +# Builds the macOS artifacts (nodejs binaries). +# Usage: ./ci/build_macos_artifacts_nodejs.sh [target] +# Targets supported: x86_64-apple-darwin aarch64-apple-darwin +set -e + +prebuild_rust() { + # Building here for the sake of easier debugging. + pushd rust/lancedb + echo "Building rust library for $1" + export RUST_BACKTRACE=1 + cargo build --release --target $1 + popd +} + +build_node_binaries() { + pushd nodejs + echo "Building nodejs library for $1" + export RUST_TARGET=$1 + npm run build-release + popd +} + +if [ -n "$1" ]; then + targets=$1 +else + targets="x86_64-apple-darwin aarch64-apple-darwin" +fi + +echo "Building artifacts for targets: $targets" +for target in $targets + do + prebuild_rust $target + build_node_binaries $target +done diff --git a/ci/build_windows_artifacts_nodejs.ps1 b/ci/build_windows_artifacts_nodejs.ps1 new file mode 100644 index 00000000..b960f306 --- /dev/null +++ b/ci/build_windows_artifacts_nodejs.ps1 @@ -0,0 +1,41 @@ +# Builds the Windows artifacts (nodejs binaries). +# Usage: .\ci\build_windows_artifacts_nodejs.ps1 [target] +# Targets supported: +# - x86_64-pc-windows-msvc +# - i686-pc-windows-msvc + +function Prebuild-Rust { + param ( + [string]$target + ) + + # Building here for the sake of easier debugging. + Push-Location -Path "rust/lancedb" + Write-Host "Building rust library for $target" + $env:RUST_BACKTRACE=1 + cargo build --release --target $target + Pop-Location +} + +function Build-NodeBinaries { + param ( + [string]$target + ) + + Push-Location -Path "nodejs" + Write-Host "Building nodejs library for $target" + $env:RUST_TARGET=$target + npm run build-release + Pop-Location +} + +$targets = $args[0] +if (-not $targets) { + $targets = "x86_64-pc-windows-msvc" +} + +Write-Host "Building artifacts for targets: $targets" +foreach ($target in $targets) { + Prebuild-Rust $target + Build-NodeBinaries $target +} diff --git a/ci/manylinux_nodejs/Dockerfile b/ci/manylinux_nodejs/Dockerfile new file mode 100644 index 00000000..65249c96 --- /dev/null +++ b/ci/manylinux_nodejs/Dockerfile @@ -0,0 +1,31 @@ +# Many linux dockerfile with Rust, Node, and Lance dependencies installed. +# This container allows building the node modules native libraries in an +# environment with a very old glibc, so that we are compatible with a wide +# range of linux distributions. +ARG ARCH=x86_64 + +FROM quay.io/pypa/manylinux2014_${ARCH} + +ARG ARCH=x86_64 +ARG DOCKER_USER=default_user + +# Install static openssl +COPY install_openssl.sh install_openssl.sh +RUN ./install_openssl.sh ${ARCH} > /dev/null + +# Protobuf is also installed as root. +COPY install_protobuf.sh install_protobuf.sh +RUN ./install_protobuf.sh ${ARCH} + +ENV DOCKER_USER=${DOCKER_USER} +# Create a group and user +RUN echo ${ARCH} && adduser --user-group --create-home --uid ${DOCKER_USER} build_user + +# We switch to the user to install Rust and Node, since those like to be +# installed at the user level. +USER ${DOCKER_USER} + +COPY prepare_manylinux_node.sh prepare_manylinux_node.sh +RUN cp /prepare_manylinux_node.sh $HOME/ && \ + cd $HOME && \ + ./prepare_manylinux_node.sh ${ARCH} diff --git a/ci/manylinux_nodejs/build.sh b/ci/manylinux_nodejs/build.sh new file mode 100755 index 00000000..b1fd60f8 --- /dev/null +++ b/ci/manylinux_nodejs/build.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Builds the nodejs module for manylinux. Invoked by ci/build_linux_artifacts_nodejs.sh. +set -e +ARCH=${1:-x86_64} + +if [ "$ARCH" = "x86_64" ]; then + export OPENSSL_LIB_DIR=/usr/local/lib64/ +else + export OPENSSL_LIB_DIR=/usr/local/lib/ +fi +export OPENSSL_STATIC=1 +export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl + +source $HOME/.bashrc + +cd nodejs +npm ci +npm run build-release diff --git a/ci/manylinux_nodejs/install_openssl.sh b/ci/manylinux_nodejs/install_openssl.sh new file mode 100755 index 00000000..59c6d873 --- /dev/null +++ b/ci/manylinux_nodejs/install_openssl.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Builds openssl from source so we can statically link to it + +# this is to avoid the error we get with the system installation: +# /usr/bin/ld: : version node not found for symbol SSLeay@@OPENSSL_1.0.1 +# /usr/bin/ld: failed to set dynamic section sizes: Bad value +set -e + +git clone -b OpenSSL_1_1_1u \ + --single-branch \ + https://github.com/openssl/openssl.git + +pushd openssl + +if [[ $1 == x86_64* ]]; then + ARCH=linux-x86_64 +else + # gnu target + ARCH=linux-aarch64 +fi + +./Configure no-shared $ARCH + +make + +make install diff --git a/ci/manylinux_nodejs/install_protobuf.sh b/ci/manylinux_nodejs/install_protobuf.sh new file mode 100755 index 00000000..a02cd2c8 --- /dev/null +++ b/ci/manylinux_nodejs/install_protobuf.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Installs protobuf compiler. Should be run as root. +set -e + +if [[ $1 == x86_64* ]]; then + ARCH=x86_64 +else + # gnu target + ARCH=aarch_64 +fi + +PB_REL=https://github.com/protocolbuffers/protobuf/releases +PB_VERSION=23.1 +curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip +unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local diff --git a/ci/manylinux_nodejs/prepare_manylinux_node.sh b/ci/manylinux_nodejs/prepare_manylinux_node.sh new file mode 100755 index 00000000..f32c0fb4 --- /dev/null +++ b/ci/manylinux_nodejs/prepare_manylinux_node.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e + +install_node() { + echo "Installing node..." + + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash + + source "$HOME"/.bashrc + + nvm install --no-progress 16 +} + +install_rust() { + echo "Installing rust..." + curl https://sh.rustup.rs -sSf | bash -s -- -y + export PATH="$PATH:/root/.cargo/bin" +} + +install_node +install_rust diff --git a/nodejs/lancedb/arrow.ts b/nodejs/lancedb/arrow.ts index df4129b0..24902974 100644 --- a/nodejs/lancedb/arrow.ts +++ b/nodejs/lancedb/arrow.ts @@ -106,6 +106,9 @@ export class MakeArrowTableOptions { * An enhanced version of the {@link makeTable} function from Apache Arrow * that supports nested fields and embeddings columns. * + * (typically you do not need to call this function. It will be called automatically + * when creating a table or adding data to it) + * * This function converts an array of Record (row-major JS objects) * to an Arrow Table (a columnar structure) * diff --git a/nodejs/lancedb/embedding/index.ts b/nodejs/lancedb/embedding/index.ts new file mode 100644 index 00000000..ac9ead3c --- /dev/null +++ b/nodejs/lancedb/embedding/index.ts @@ -0,0 +1,2 @@ +export { EmbeddingFunction, isEmbeddingFunction } from "./embedding_function"; +export { OpenAIEmbeddingFunction } from "./openai"; diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index 4877a6c2..40bcc645 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -18,9 +18,34 @@ import { ConnectionOptions, } from "./native.js"; -export { ConnectionOptions, WriteOptions, Query } from "./native.js"; -export { Connection, CreateTableOptions } from "./connection"; -export { Table, AddDataOptions } from "./table"; +export { + WriteOptions, + WriteMode, + AddColumnsSql, + ColumnAlteration, + ConnectionOptions, +} from "./native.js"; +export { + makeArrowTable, + MakeArrowTableOptions, + Data, + VectorColumnOptions, +} from "./arrow"; +export { + Connection, + CreateTableOptions, + TableNamesOptions, +} from "./connection"; +export { + ExecutableQuery, + Query, + QueryBase, + VectorQuery, + RecordBatchIterator, +} from "./query"; +export { Index, IndexOptions, IvfPqOptions } from "./indices"; +export { Table, AddDataOptions, IndexConfig, UpdateOptions } from "./table"; +export * as embedding from "./embedding"; /** * Connect to a LanceDB instance at the given URI. diff --git a/nodejs/lancedb/native.d.ts b/nodejs/lancedb/native.d.ts deleted file mode 100644 index f025484b..00000000 --- a/nodejs/lancedb/native.d.ts +++ /dev/null @@ -1,147 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ - -/* auto-generated by NAPI-RS */ - -/** A description of an index currently configured on a column */ -export interface IndexConfig { - /** The type of the index */ - indexType: string - /** - * The columns in the index - * - * Currently this is always an array of size 1. In the future there may - * be more columns to represent composite indices. - */ - columns: Array -} -/** - * A definition of a column alteration. The alteration changes the column at - * `path` to have the new name `name`, to be nullable if `nullable` is true, - * and to have the data type `data_type`. At least one of `rename` or `nullable` - * must be provided. - */ -export interface ColumnAlteration { - /** - * The path to the column to alter. This is a dot-separated path to the column. - * If it is a top-level column then it is just the name of the column. If it is - * a nested column then it is the path to the column, e.g. "a.b.c" for a column - * `c` nested inside a column `b` nested inside a column `a`. - */ - path: string - /** - * The new name of the column. If not provided then the name will not be changed. - * This must be distinct from the names of all other columns in the table. - */ - rename?: string - /** Set the new nullability. Note that a nullable column cannot be made non-nullable. */ - nullable?: boolean -} -/** A definition of a new column to add to a table. */ -export interface AddColumnsSql { - /** The name of the new column. */ - name: string - /** - * The values to populate the new column with, as a SQL expression. - * The expression can reference other columns in the table. - */ - valueSql: string -} -export interface ConnectionOptions { - apiKey?: string - hostOverride?: string - /** - * (For LanceDB OSS only): The interval, in seconds, at which to check for - * updates to the table from other processes. If None, then consistency is not - * checked. For performance reasons, this is the default. For strong - * consistency, set this to zero seconds. Then every read will check for - * updates from other processes. As a compromise, you can set this to a - * non-zero value for eventual consistency. If more than that interval - * has passed since the last check, then the table will be checked for updates. - * Note: this consistency only applies to read operations. Write operations are - * always consistent. - */ - readConsistencyInterval?: number -} -/** Write mode for writing a table. */ -export const enum WriteMode { - Create = 'Create', - Append = 'Append', - Overwrite = 'Overwrite' -} -/** Write options when creating a Table. */ -export interface WriteOptions { - mode?: WriteMode -} -export function connect(uri: string, options: ConnectionOptions): Promise -export class Connection { - /** Create a new Connection instance from the given URI. */ - static new(uri: string, options: ConnectionOptions): Promise - display(): string - isOpen(): boolean - close(): void - /** List all tables in the dataset. */ - tableNames(startAfter?: string | undefined | null, limit?: number | undefined | null): Promise> - /** - * Create table from a Apache Arrow IPC (file) buffer. - * - * Parameters: - * - name: The name of the table. - * - buf: The buffer containing the IPC file. - * - */ - createTable(name: string, buf: Buffer, mode: string): Promise - createEmptyTable(name: string, schemaBuf: Buffer, mode: string): Promise
- openTable(name: string): Promise
- /** Drop table with the name. Or raise an error if the table does not exist. */ - dropTable(name: string): Promise -} -export class Index { - static ivfPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index - static btree(): Index -} -/** Typescript-style Async Iterator over RecordBatches */ -export class RecordBatchIterator { - next(): Promise -} -export class Query { - onlyIf(predicate: string): void - select(columns: Array<[string, string]>): void - limit(limit: number): void - nearestTo(vector: Float32Array): VectorQuery - execute(): Promise -} -export class VectorQuery { - column(column: string): void - distanceType(distanceType: string): void - postfilter(): void - refineFactor(refineFactor: number): void - nprobes(nprobe: number): void - bypassVectorIndex(): void - onlyIf(predicate: string): void - select(columns: Array<[string, string]>): void - limit(limit: number): void - execute(): Promise -} -export class Table { - display(): string - isOpen(): boolean - close(): void - /** Return Schema as empty Arrow IPC file. */ - schema(): Promise - add(buf: Buffer, mode: string): Promise - countRows(filter?: string | undefined | null): Promise - delete(predicate: string): Promise - createIndex(index: Index | undefined | null, column: string, replace?: boolean | undefined | null): Promise - update(onlyIf: string | undefined | null, columns: Array<[string, string]>): Promise - query(): Query - vectorSearch(vector: Float32Array): VectorQuery - addColumns(transforms: Array): Promise - alterColumns(alterations: Array): Promise - dropColumns(columns: Array): Promise - version(): Promise - checkout(version: number): Promise - checkoutLatest(): Promise - restore(): Promise - listIndices(): Promise> -} diff --git a/nodejs/lancedb/native.js b/nodejs/lancedb/native.js deleted file mode 100644 index e8ea9295..00000000 --- a/nodejs/lancedb/native.js +++ /dev/null @@ -1,329 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/* prettier-ignore */ - -/* auto-generated by NAPI-RS */ - -const { existsSync, readFileSync } = require('fs') -const { join } = require("path"); - -const { platform, arch } = process; - -let nativeBinding = null; -let localFileExisted = false; -let loadError = null; - -function isMusl() { - // For Node 10 - if (!process.report || typeof process.report.getReport !== "function") { - try { - const lddPath = require("child_process") - .execSync("which ldd") - .toString() - .trim(); - return readFileSync(lddPath, "utf8").includes("musl"); - } catch (e) { - return true; - } - } else { - const { glibcVersionRuntime } = process.report.getReport().header; - return !glibcVersionRuntime; - } -} - -switch (platform) { - case "android": - switch (arch) { - case "arm64": - localFileExisted = existsSync( - join(__dirname, "lancedb-nodejs.android-arm64.node"), - ); - try { - if (localFileExisted) { - nativeBinding = require("./lancedb-nodejs.android-arm64.node"); - } else { - nativeBinding = require("lancedb-android-arm64"); - } - } catch (e) { - loadError = e; - } - break; - case "arm": - localFileExisted = existsSync( - join(__dirname, "lancedb-nodejs.android-arm-eabi.node"), - ); - try { - if (localFileExisted) { - nativeBinding = require("./lancedb-nodejs.android-arm-eabi.node"); - } else { - nativeBinding = require("lancedb-android-arm-eabi"); - } - } catch (e) { - loadError = e; - } - break; - default: - throw new Error(`Unsupported architecture on Android ${arch}`); - } - break; - case "win32": - switch (arch) { - case "x64": - localFileExisted = existsSync( - join(__dirname, "lancedb-nodejs.win32-x64-msvc.node"), - ); - try { - if (localFileExisted) { - nativeBinding = require("./lancedb-nodejs.win32-x64-msvc.node"); - } else { - nativeBinding = require("lancedb-win32-x64-msvc"); - } - } catch (e) { - loadError = e; - } - break; - case "ia32": - localFileExisted = existsSync( - join(__dirname, "lancedb-nodejs.win32-ia32-msvc.node"), - ); - try { - if (localFileExisted) { - nativeBinding = require("./lancedb-nodejs.win32-ia32-msvc.node"); - } else { - nativeBinding = require("lancedb-win32-ia32-msvc"); - } - } catch (e) { - loadError = e; - } - break; - case "arm64": - localFileExisted = existsSync( - join(__dirname, "lancedb-nodejs.win32-arm64-msvc.node"), - ); - try { - if (localFileExisted) { - nativeBinding = require("./lancedb-nodejs.win32-arm64-msvc.node"); - } else { - nativeBinding = require("lancedb-win32-arm64-msvc"); - } - } catch (e) { - loadError = e; - } - break; - default: - throw new Error(`Unsupported architecture on Windows: ${arch}`); - } - break; - case "darwin": - localFileExisted = existsSync( - join(__dirname, "lancedb-nodejs.darwin-universal.node"), - ); - try { - if (localFileExisted) { - nativeBinding = require("./lancedb-nodejs.darwin-universal.node"); - } else { - nativeBinding = require("lancedb-darwin-universal"); - } - break; - } catch {} - switch (arch) { - case "x64": - localFileExisted = existsSync( - join(__dirname, "lancedb-nodejs.darwin-x64.node"), - ); - try { - if (localFileExisted) { - nativeBinding = require("./lancedb-nodejs.darwin-x64.node"); - } else { - nativeBinding = require("lancedb-darwin-x64"); - } - } catch (e) { - loadError = e; - } - break; - case "arm64": - localFileExisted = existsSync( - join(__dirname, "lancedb-nodejs.darwin-arm64.node"), - ); - try { - if (localFileExisted) { - nativeBinding = require("./lancedb-nodejs.darwin-arm64.node"); - } else { - nativeBinding = require("lancedb-darwin-arm64"); - } - } catch (e) { - loadError = e; - } - break; - default: - throw new Error(`Unsupported architecture on macOS: ${arch}`); - } - break; - case "freebsd": - if (arch !== "x64") { - throw new Error(`Unsupported architecture on FreeBSD: ${arch}`); - } - localFileExisted = existsSync( - join(__dirname, "lancedb-nodejs.freebsd-x64.node"), - ); - try { - if (localFileExisted) { - nativeBinding = require("./lancedb-nodejs.freebsd-x64.node"); - } else { - nativeBinding = require("lancedb-freebsd-x64"); - } - } catch (e) { - loadError = e; - } - break; - case "linux": - switch (arch) { - case "x64": - if (isMusl()) { - localFileExisted = existsSync( - join(__dirname, "lancedb-nodejs.linux-x64-musl.node"), - ); - try { - if (localFileExisted) { - nativeBinding = require("./lancedb-nodejs.linux-x64-musl.node"); - } else { - nativeBinding = require("lancedb-linux-x64-musl"); - } - } catch (e) { - loadError = e; - } - } else { - localFileExisted = existsSync( - join(__dirname, "lancedb-nodejs.linux-x64-gnu.node"), - ); - try { - if (localFileExisted) { - nativeBinding = require("./lancedb-nodejs.linux-x64-gnu.node"); - } else { - nativeBinding = require("lancedb-linux-x64-gnu"); - } - } catch (e) { - loadError = e; - } - } - break; - case "arm64": - if (isMusl()) { - localFileExisted = existsSync( - join(__dirname, "lancedb-nodejs.linux-arm64-musl.node"), - ); - try { - if (localFileExisted) { - nativeBinding = require("./lancedb-nodejs.linux-arm64-musl.node"); - } else { - nativeBinding = require("lancedb-linux-arm64-musl"); - } - } catch (e) { - loadError = e; - } - } else { - localFileExisted = existsSync( - join(__dirname, "lancedb-nodejs.linux-arm64-gnu.node"), - ); - try { - if (localFileExisted) { - nativeBinding = require("./lancedb-nodejs.linux-arm64-gnu.node"); - } else { - nativeBinding = require("lancedb-linux-arm64-gnu"); - } - } catch (e) { - loadError = e; - } - } - break; - case "arm": - localFileExisted = existsSync( - join(__dirname, "lancedb-nodejs.linux-arm-gnueabihf.node"), - ); - try { - if (localFileExisted) { - nativeBinding = require("./lancedb-nodejs.linux-arm-gnueabihf.node"); - } else { - nativeBinding = require("lancedb-linux-arm-gnueabihf"); - } - } catch (e) { - loadError = e; - } - break; - case "riscv64": - if (isMusl()) { - localFileExisted = existsSync( - join(__dirname, "lancedb-nodejs.linux-riscv64-musl.node"), - ); - try { - if (localFileExisted) { - nativeBinding = require("./lancedb-nodejs.linux-riscv64-musl.node"); - } else { - nativeBinding = require("lancedb-linux-riscv64-musl"); - } - } catch (e) { - loadError = e; - } - } else { - localFileExisted = existsSync( - join(__dirname, "lancedb-nodejs.linux-riscv64-gnu.node"), - ); - try { - if (localFileExisted) { - nativeBinding = require("./lancedb-nodejs.linux-riscv64-gnu.node"); - } else { - nativeBinding = require("lancedb-linux-riscv64-gnu"); - } - } catch (e) { - loadError = e; - } - } - break; - case "s390x": - localFileExisted = existsSync( - join(__dirname, "lancedb-nodejs.linux-s390x-gnu.node"), - ); - try { - if (localFileExisted) { - nativeBinding = require("./lancedb-nodejs.linux-s390x-gnu.node"); - } else { - nativeBinding = require("lancedb-linux-s390x-gnu"); - } - } catch (e) { - loadError = e; - } - break; - default: - throw new Error(`Unsupported architecture on Linux: ${arch}`); - } - break; - default: - throw new Error(`Unsupported OS: ${platform}, architecture: ${arch}`); -} - -if (!nativeBinding) { - if (loadError) { - throw loadError; - } - throw new Error(`Failed to load native binding`); -} - -const { - Connection, - Index, - RecordBatchIterator, - Query, - VectorQuery, - Table, - WriteMode, - connect, -} = nativeBinding; - -module.exports.Connection = Connection; -module.exports.Index = Index; -module.exports.RecordBatchIterator = RecordBatchIterator; -module.exports.Query = Query; -module.exports.VectorQuery = VectorQuery; -module.exports.Table = Table; -module.exports.WriteMode = WriteMode; -module.exports.connect = connect; diff --git a/nodejs/lancedb/query.ts b/nodejs/lancedb/query.ts index 990f6370..23bf2b82 100644 --- a/nodejs/lancedb/query.ts +++ b/nodejs/lancedb/query.ts @@ -20,7 +20,7 @@ import { VectorQuery as NativeVectorQuery, } from "./native"; import { type IvfPqOptions } from "./indices"; -class RecordBatchIterator implements AsyncIterator { +export class RecordBatchIterator implements AsyncIterator { private promisedInner?: Promise; private inner?: NativeBatchIterator; diff --git a/nodejs/npm/darwin-arm64/README.md b/nodejs/npm/darwin-arm64/README.md index b324e37b..75f026e1 100644 --- a/nodejs/npm/darwin-arm64/README.md +++ b/nodejs/npm/darwin-arm64/README.md @@ -1,3 +1,3 @@ -# `lancedb-darwin-arm64` +# `@lancedb/lancedb-darwin-arm64` -This is the **aarch64-apple-darwin** binary for `lancedb` +This is the **aarch64-apple-darwin** binary for `@lancedb/lancedb` diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index e4636f4d..8b3f09a0 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,5 +1,5 @@ { - "name": "lancedb-darwin-arm64", + "name": "@lancedb/lancedb-darwin-arm64", "version": "0.4.3", "os": [ "darwin" @@ -11,7 +11,7 @@ "files": [ "lancedb.darwin-arm64.node" ], - "license": "MIT", + "license": "Apache 2.0", "engines": { "node": ">= 18" } diff --git a/nodejs/npm/darwin-x64/README.md b/nodejs/npm/darwin-x64/README.md index dee23695..f82b0cf7 100644 --- a/nodejs/npm/darwin-x64/README.md +++ b/nodejs/npm/darwin-x64/README.md @@ -1,3 +1,3 @@ -# `lancedb-darwin-x64` +# `@lancedb/lancedb-darwin-x64` -This is the **x86_64-apple-darwin** binary for `lancedb` +This is the **x86_64-apple-darwin** binary for `@lancedb/lancedb` diff --git a/nodejs/npm/darwin-x64/package.json b/nodejs/npm/darwin-x64/package.json index 56f8ae37..46b6eb24 100644 --- a/nodejs/npm/darwin-x64/package.json +++ b/nodejs/npm/darwin-x64/package.json @@ -1,5 +1,5 @@ { - "name": "lancedb-darwin-x64", + "name": "@lancedb/lancedb-darwin-x64", "version": "0.4.3", "os": [ "darwin" @@ -11,7 +11,7 @@ "files": [ "lancedb.darwin-x64.node" ], - "license": "MIT", + "license": "Apache 2.0", "engines": { "node": ">= 18" } diff --git a/nodejs/npm/linux-arm64-gnu/README.md b/nodejs/npm/linux-arm64-gnu/README.md index b2fda68d..d46549a2 100644 --- a/nodejs/npm/linux-arm64-gnu/README.md +++ b/nodejs/npm/linux-arm64-gnu/README.md @@ -1,3 +1,3 @@ -# `lancedb-linux-arm64-gnu` +# `@lancedb/lancedb-linux-arm64-gnu` -This is the **aarch64-unknown-linux-gnu** binary for `lancedb` +This is the **aarch64-unknown-linux-gnu** binary for `@lancedb/lancedb` diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 1bed8dd6..486793f5 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,5 +1,5 @@ { - "name": "lancedb-linux-arm64-gnu", + "name": "@lancedb/lancedb-linux-arm64-gnu", "version": "0.4.3", "os": [ "linux" @@ -11,9 +11,9 @@ "files": [ "lancedb.linux-arm64-gnu.node" ], - "license": "MIT", + "license": "Apache 2.0", "engines": { - "node": ">= 10" + "node": ">= 18" }, "libc": [ "glibc" diff --git a/nodejs/npm/linux-x64-gnu/README.md b/nodejs/npm/linux-x64-gnu/README.md index e4a94cdd..bf3c706f 100644 --- a/nodejs/npm/linux-x64-gnu/README.md +++ b/nodejs/npm/linux-x64-gnu/README.md @@ -1,3 +1,3 @@ -# `lancedb-linux-x64-gnu` +# `@lancedb/lancedb-linux-x64-gnu` -This is the **x86_64-unknown-linux-gnu** binary for `lancedb` +This is the **x86_64-unknown-linux-gnu** binary for `@lancedb/lancedb` diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 1c4ddc22..95d82520 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,5 +1,5 @@ { - "name": "lancedb-linux-x64-gnu", + "name": "@lancedb/lancedb-linux-x64-gnu", "version": "0.4.3", "os": [ "linux" @@ -11,9 +11,9 @@ "files": [ "lancedb.linux-x64-gnu.node" ], - "license": "MIT", + "license": "Apache 2.0", "engines": { - "node": ">= 10" + "node": ">= 18" }, "libc": [ "glibc" diff --git a/nodejs/npm/win32-x64-msvc/README.md b/nodejs/npm/win32-x64-msvc/README.md new file mode 100644 index 00000000..42b06d84 --- /dev/null +++ b/nodejs/npm/win32-x64-msvc/README.md @@ -0,0 +1,3 @@ +# `@lancedb/lancedb-win32-x64-msvc` + +This is the **x86_64-pc-windows-msvc** binary for `@lancedb/lancedb` diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json new file mode 100644 index 00000000..1b77a695 --- /dev/null +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -0,0 +1,18 @@ +{ + "name": "@lancedb/lancedb-win32-x64-msvc", + "version": "0.4.3", + "os": [ + "win32" + ], + "cpu": [ + "x64" + ], + "main": "lancedb.win32-x64-msvc.node", + "files": [ + "lancedb.win32-x64-msvc.node" + ], + "license": "Apache 2.0", + "engines": { + "node": ">= 18" + } +} diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index b8d05a7e..43d8125c 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -1,11 +1,11 @@ { - "name": "lancedb", + "name": "@lancedb/lancedb", "version": "0.4.3", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "lancedb", + "name": "@lancedb/lancedb", "version": "0.4.3", "cpu": [ "x64", @@ -15,8 +15,12 @@ "os": [ "darwin", "linux", - "windows" + "win32" ], + "dependencies": { + "apache-arrow": "^15.0.0", + "openai": "^4.29.2" + }, "devDependencies": { "@napi-rs/cli": "^2.18.0", "@types/jest": "^29.1.2", @@ -29,6 +33,7 @@ "eslint-plugin-jsdoc": "^48.2.1", "jest": "^29.7.0", "prettier": "^3.1.0", + "shx": "^0.3.4", "tmp": "^0.2.3", "ts-jest": "^29.1.2", "typedoc": "^0.25.7", @@ -40,14 +45,11 @@ "node": ">= 18" }, "optionalDependencies": { - "lancedb-darwin-arm64": "0.4.3", - "lancedb-darwin-x64": "0.4.3", - "lancedb-linux-arm64-gnu": "0.4.3", - "lancedb-linux-x64-gnu": "0.4.3", - "openai": "^4.28.4" - }, - "peerDependencies": { - "apache-arrow": "^15.0.0" + "@lancedb/lancedb-darwin-arm64": "0.4.3", + "@lancedb/lancedb-darwin-x64": "0.4.3", + "@lancedb/lancedb-linux-arm64-gnu": "0.4.3", + "@lancedb/lancedb-linux-x64-gnu": "0.4.3", + "@lancedb/lancedb-win32-x64-msvc": "0.4.3" } }, "node_modules/@75lb/deep-merge": { @@ -1317,6 +1319,66 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@lancedb/lancedb-darwin-arm64": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-darwin-arm64/-/lancedb-darwin-arm64-0.4.3.tgz", + "integrity": "sha512-+kxuWUK9vtLBbjFMkIKeQ32kxK2tgvZRCQaU1I3RJ3+dLmDIVeIj+KJSlMelkKa2QC4JoyHQi9Ty1PdS2DojmQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 18" + } + }, + "node_modules/@lancedb/lancedb-darwin-x64": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-darwin-x64/-/lancedb-darwin-x64-0.4.3.tgz", + "integrity": "sha512-JYvsSYxTOa/7OMojulz9h0gN2FwvypG/6l6dpLkViZ5LDvRcfVyDTzOLcOJkFn+db4TKeBOVyMWnnpDKaB+jLA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 18" + } + }, + "node_modules/@lancedb/lancedb-linux-x64-gnu": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-x64-gnu/-/lancedb-linux-x64-gnu-0.4.3.tgz", + "integrity": "sha512-jDANHchWNGmu1wfAyBk0apoFlLxtJ7FRc31pAQ3tKE4fwlgG7bUcaTX6s5C3vMNWXnyQLQtVuWZNXi2nVj879g==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 18" + } + }, + "node_modules/@lancedb/lancedb-win32-x64-msvc": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-win32-x64-msvc/-/lancedb-win32-x64-msvc-0.4.3.tgz", + "integrity": "sha512-qADveXyv4YzllIbOOq8soqFfL7p7I35uhrD3PcTvj4Qxuo6q7pgQWQz2Mt3kGBpyPkH2yE4wWAGJhayShLRbiQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 18" + } + }, "node_modules/@napi-rs/cli": { "version": "2.18.0", "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.18.0.tgz", @@ -1396,7 +1458,6 @@ "version": "0.5.6", "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.6.tgz", "integrity": "sha512-aYX01Ke9hunpoCexYAgQucEpARGQ5w/cqHFrIR+e9gdKb1QWTsVJuTJ2ozQzIAxLyRQe/m+2RqzkyOOGiMKRQA==", - "peer": true, "dependencies": { "tslib": "^2.4.0" } @@ -1445,8 +1506,7 @@ "node_modules/@types/command-line-args": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.3.tgz", - "integrity": "sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw==", - "peer": true + "integrity": "sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw==" }, "node_modules/@types/command-line-usage": { "version": "5.0.2", @@ -1514,7 +1574,6 @@ "version": "2.6.11", "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.11.tgz", "integrity": "sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==", - "optional": true, "dependencies": { "@types/node": "*", "form-data": "^4.0.0" @@ -1783,7 +1842,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "optional": true, "dependencies": { "event-target-shim": "^5.0.0" }, @@ -1816,7 +1874,6 @@ "version": "4.5.0", "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz", "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==", - "optional": true, "dependencies": { "humanize-ms": "^1.2.1" }, @@ -1913,7 +1970,6 @@ "version": "15.0.0", "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-15.0.0.tgz", "integrity": "sha512-e6aunxNKM+woQf137ny3tp/xbLjFJS2oGQxQhYGqW6dGeIwNV1jOeEAeR6sS2jwAI2qLO83gYIP2MBz02Gw5Xw==", - "peer": true, "dependencies": { "@swc/helpers": "^0.5.2", "@types/command-line-args": "^5.2.1", @@ -2001,8 +2057,7 @@ "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "optional": true + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "node_modules/babel-jest": { "version": "29.7.0", @@ -2129,8 +2184,7 @@ "node_modules/base-64": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz", - "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==", - "optional": true + "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==" }, "node_modules/brace-expansion": { "version": "1.1.11", @@ -2296,7 +2350,6 @@ "version": "0.0.2", "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", - "optional": true, "engines": { "node": "*" } @@ -2357,7 +2410,6 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "optional": true, "dependencies": { "delayed-stream": "~1.0.0" }, @@ -2469,7 +2521,6 @@ "version": "0.0.2", "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", - "optional": true, "engines": { "node": "*" } @@ -2530,7 +2581,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "optional": true, "engines": { "node": ">=0.4.0" } @@ -2557,7 +2607,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz", "integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==", - "optional": true, "dependencies": { "base-64": "^0.1.0", "md5": "^2.3.0" @@ -2862,7 +2911,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "optional": true, "engines": { "node": ">=6" } @@ -3024,7 +3072,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", - "optional": true, "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -3037,14 +3084,12 @@ "node_modules/form-data-encoder": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", - "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", - "optional": true + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==" }, "node_modules/formdata-node": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", - "optional": true, "dependencies": { "node-domexception": "1.0.0", "web-streams-polyfill": "4.0.0-beta.3" @@ -3057,7 +3102,6 @@ "version": "4.0.0-beta.3", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", - "optional": true, "engines": { "node": ">= 14" } @@ -3272,7 +3316,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "optional": true, "dependencies": { "ms": "^2.0.0" } @@ -3355,6 +3398,15 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "dev": true }, + "node_modules/interpret": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", + "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", + "dev": true, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -3364,8 +3416,7 @@ "node_modules/is-buffer": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", - "optional": true + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" }, "node_modules/is-builtin-module": { "version": "3.2.1", @@ -4458,7 +4509,6 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", - "optional": true, "dependencies": { "charenc": "0.0.2", "crypt": "0.0.2", @@ -4497,7 +4547,6 @@ "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "optional": true, "engines": { "node": ">= 0.6" } @@ -4506,7 +4555,6 @@ "version": "2.1.35", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "optional": true, "dependencies": { "mime-db": "1.52.0" }, @@ -4538,8 +4586,7 @@ "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "optional": true + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/natural-compare": { "version": "1.4.0", @@ -4567,7 +4614,6 @@ "url": "https://paypal.me/jimmywarting" } ], - "optional": true, "engines": { "node": ">=10.5.0" } @@ -4576,7 +4622,6 @@ "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "optional": true, "dependencies": { "whatwg-url": "^5.0.0" }, @@ -4623,10 +4668,9 @@ } }, "node_modules/openai": { - "version": "4.28.4", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.28.4.tgz", - "integrity": "sha512-RNIwx4MT/F0zyizGcwS+bXKLzJ8QE9IOyigDG/ttnwB220d58bYjYFp0qjvGwEFBO6+pvFVIDABZPGDl46RFsg==", - "optional": true, + "version": "4.29.2", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.29.2.tgz", + "integrity": "sha512-cPkT6zjEcE4qU5OW/SoDDuXEsdOLrXlAORhzmaguj5xZSPlgKvLhi27sFWhLKj07Y6WKNWxcwIbzm512FzTBNQ==", "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", @@ -4643,10 +4687,9 @@ } }, "node_modules/openai/node_modules/@types/node": { - "version": "18.19.20", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.20.tgz", - "integrity": "sha512-SKXZvI375jkpvAj8o+5U2518XQv76mAsixqfXiVyWyXZbVWQK25RurFovYpVIxVzul0rZoH58V/3SkEnm7s3qA==", - "optional": true, + "version": "18.19.26", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.26.tgz", + "integrity": "sha512-+wiMJsIwLOYCvUqSdKTrfkS8mpTp+MPINe6+Np4TAGFWWRWiBQ5kSq9nZGCSPkzx9mvT+uEukzpX4MOSCydcvw==", "dependencies": { "undici-types": "~5.26.4" } @@ -4996,6 +5039,18 @@ "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==", "dev": true }, + "node_modules/rechoir": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", + "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==", + "dev": true, + "dependencies": { + "resolve": "^1.1.6" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/repeat-string": { "version": "1.6.1", "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", @@ -5145,6 +5200,23 @@ "node": ">=8" } }, + "node_modules/shelljs": { + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", + "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", + "dev": true, + "dependencies": { + "glob": "^7.0.0", + "interpret": "^1.0.0", + "rechoir": "^0.6.2" + }, + "bin": { + "shjs": "bin/shjs" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/shiki": { "version": "0.14.7", "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.14.7.tgz", @@ -5157,6 +5229,22 @@ "vscode-textmate": "^8.0.0" } }, + "node_modules/shx": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/shx/-/shx-0.3.4.tgz", + "integrity": "sha512-N6A9MLVqjxZYcVn8hLmtneQWIJtp8IKzMP4eMnx+nqkvXoqinUPCbUFLp2UcWTEIUONhlk0ewxr/jaVGlc+J+g==", + "dev": true, + "dependencies": { + "minimist": "^1.2.3", + "shelljs": "^0.8.5" + }, + "bin": { + "shx": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", @@ -5432,8 +5520,7 @@ "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "optional": true + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, "node_modules/ts-api-utils": { "version": "1.0.3", @@ -5929,7 +6016,6 @@ "version": "3.3.3", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", - "optional": true, "engines": { "node": ">= 8" } @@ -5937,14 +6023,12 @@ "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "optional": true + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" }, "node_modules/whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "optional": true, "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" diff --git a/nodejs/package.json b/nodejs/package.json index d7d7580f..bb8a8c89 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -1,17 +1,18 @@ { - "name": "lancedb", + "name": "@lancedb/lancedb", "version": "0.4.3", "main": "./dist/index.js", "types": "./dist/index.d.ts", "napi": { - "name": "lancedb-nodejs", + "name": "lancedb", "triples": { "defaults": false, "additional": [ "aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-apple-darwin", - "x86_64-unknown-linux-gnu" + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc" ] } }, @@ -28,6 +29,7 @@ "eslint-plugin-jsdoc": "^48.2.1", "jest": "^29.7.0", "prettier": "^3.1.0", + "shx": "^0.3.4", "tmp": "^0.2.3", "ts-jest": "^29.1.2", "typedoc": "^0.25.7", @@ -48,13 +50,14 @@ "os": [ "darwin", "linux", - "windows" + "win32" ], "scripts": { "artifacts": "napi artifacts", - "build:native": "napi build --platform --release --js lancedb/native.js --dts lancedb/native.d.ts dist/", "build:debug": "napi build --platform --dts ../lancedb/native.d.ts --js ../lancedb/native.js dist/", - "build": "npm run build:debug && tsc -b", + "build:release": "napi build --platform --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js dist/", + "build": "npm run build:debug && tsc -b && shx cp lancedb/native.d.ts dist/native.d.ts", + "build-release": "npm run build:release && tsc -b && shx cp lancedb/native.d.ts dist/native.d.ts", "chkformat": "prettier . --check", "docs": "typedoc --plugin typedoc-plugin-markdown lancedb/index.ts", "lint": "eslint lancedb && eslint __test__", @@ -64,13 +67,14 @@ "version": "napi version" }, "optionalDependencies": { - "lancedb-darwin-arm64": "0.4.3", - "lancedb-darwin-x64": "0.4.3", - "lancedb-linux-arm64-gnu": "0.4.3", - "lancedb-linux-x64-gnu": "0.4.3", - "openai": "^4.28.4" + "@lancedb/lancedb-darwin-arm64": "0.4.3", + "@lancedb/lancedb-darwin-x64": "0.4.3", + "@lancedb/lancedb-linux-arm64-gnu": "0.4.3", + "@lancedb/lancedb-linux-x64-gnu": "0.4.3", + "@lancedb/lancedb-win32-x64-msvc": "0.4.3" }, - "peerDependencies": { + "dependencies": { + "openai": "^4.29.2", "apache-arrow": "^15.0.0" } }