Bump version: 0.4.13 → 0.4.14

This will start publishing `@lancedb/lancedb` with the new nodejs
package on our releases.

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.13
+current_version = 0.4.14
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True

.github/workflows/npm-publish.yml

@@ -2,7 +2,7 @@ name: NPM Publish
 
 on:
   release:
-    types: [ published ]
+    types: [published]
 
 jobs:
   node:
@@ -19,7 +19,7 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: 20
-          cache: 'npm'
+          cache: "npm"
           cache-dependency-path: node/package-lock.json
       - name: Install dependencies
         run: |
@@ -31,7 +31,7 @@ jobs:
           npm run tsc
           npm pack
       - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: node-package
           path: |
@@ -61,12 +61,41 @@ jobs:
       - name: Build MacOS native node modules
         run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }}
       - name: Upload Darwin Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: native-darwin
+          name: node-native-darwin
           path: |
             node/dist/lancedb-vectordb-darwin*.tgz
 
+  nodejs-macos:
+    strategy:
+      matrix:
+        config:
+          - arch: x86_64-apple-darwin
+            runner: macos-13
+          - arch: aarch64-apple-darwin
+            # xlarge is implicitly arm64.
+            runner: macos-14
+    runs-on: ${{ matrix.config.runner }}
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install system dependencies
+        run: brew install protobuf
+      - name: Install npm dependencies
+        run: |
+          cd nodejs
+          npm ci
+      - name: Build MacOS native nodejs modules
+        run: bash ci/build_macos_artifacts_nodejs.sh ${{ matrix.config.arch }}
+      - name: Upload Darwin Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-native-darwin-${{ matrix.config.arch }}
+          path: |
+            nodejs/dist/*.node
 
   node-linux:
     name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu
@@ -103,12 +132,63 @@ jobs:
         run: |
           bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}
       - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: native-linux
+          name: node-native-linux
           path: |
             node/dist/lancedb-vectordb-linux*.tgz
 
+  nodejs-linux:
+    name: nodejs-linux (${{ matrix.config.arch}}-unknown-linux-gnu
+    runs-on: ${{ matrix.config.runner }}
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - arch: x86_64
+            runner: ubuntu-latest
+          - arch: aarch64
+            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
+            runner: buildjet-16vcpu-ubuntu-2204-arm
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      # Buildjet aarch64 runners have only 1.5 GB RAM per core, vs 3.5 GB per core for
+      # x86_64 runners. To avoid OOM errors on ARM, we create a swap file.
+      - name: Configure aarch64 build
+        if: ${{ matrix.config.arch == 'aarch64' }}
+        run: |
+          free -h
+          sudo fallocate -l 16G /swapfile
+          sudo chmod 600 /swapfile
+          sudo mkswap /swapfile
+          sudo swapon /swapfile
+          echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab
+          # print info
+          swapon --show
+          free -h
+      - name: Build Linux Artifacts
+        run: |
+          bash ci/build_linux_artifacts_nodejs.sh ${{ matrix.config.arch }}
+      - name: Upload Linux Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-native-linux-${{ matrix.config.arch }}
+          path: |
+            nodejs/dist/*.node
+      # The generic files are the same in all distros so we just pick
+      # one to do the upload.
+      - name: Upload Generic Artifacts
+        if: ${{ matrix.config.arch == 'x86_64' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-dist
+          path: |
+            nodejs/dist/*
+            !nodejs/dist/*.node
+
   node-windows:
     runs-on: windows-2022
     # Only runs on tags that matches the make-release action
@@ -136,25 +216,60 @@ jobs:
       - name: Build Windows native node modules
         run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }}
       - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: native-windows
+          name: node-native-windows
           path: |
             node/dist/lancedb-vectordb-win32*.tgz
 
+  nodejs-windows:
+    runs-on: windows-2022
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64-pc-windows-msvc]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install Protoc v21.12
+        working-directory: C:\
+        run: |
+          New-Item -Path 'C:\protoc' -ItemType Directory
+          Set-Location C:\protoc
+          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+          7z x protoc.zip
+          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+        shell: powershell
+      - name: Install npm dependencies
+        run: |
+          cd nodejs
+          npm ci
+      - name: Build Windows native node modules
+        run: .\ci\build_windows_artifacts_nodejs.ps1 ${{ matrix.target }}
+      - name: Upload Windows Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-native-windows
+          path: |
+            nodejs/dist/*.node
+
   release:
     needs: [node, node-macos, node-linux, node-windows]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: node-*
       - name: Display structure of downloaded files
         run: ls -R
       - uses: actions/setup-node@v3
         with:
           node-version: 20
-          registry-url: 'https://registry.npmjs.org'
+          registry-url: "https://registry.npmjs.org"
       - name: Publish to NPM
         env:
           NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
@@ -164,6 +279,45 @@ jobs:
             npm publish $filename
           done
 
+  release-nodejs:
+    needs: [nodejs-macos, nodejs-linux, nodejs-windows]
+    runs-on: ubuntu-latest
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    defaults:
+      run:
+        shell: bash
+        working-directory: nodejs
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: nodejs-dist
+          path: nodejs/dist
+      - uses: actions/download-artifact@v4
+        name: Download arch-specific binaries
+        with:
+          pattern: nodejs-*
+          path: nodejs/nodejs-artifacts
+          merge-multiple: true
+      - name: Display structure of downloaded files
+        run: find .
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          registry-url: "https://registry.npmjs.org"
+      - name: Install napi-rs
+        run: npm install -g @napi-rs/cli
+      - name: Prepare artifacts
+        run: npx napi artifacts -d nodejs-artifacts
+      - name: Display structure of staged files
+        run: find npm
+      - name: Publish to NPM
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
+        run: npm publish --access public
+
   update-package-lock:
     needs: [release]
     runs-on: ubuntu-latest
@@ -178,3 +332,18 @@ jobs:
       - uses: ./.github/workflows/update_package_lock
         with:
           github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+
+  update-package-lock-nodejs:
+    needs: [release-nodejs]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          persist-credentials: false
+          fetch-depth: 0
+          lfs: true
+      - uses: ./.github/workflows/update_package_lock_nodejs
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.github/workflows/update_package_lock_nodejs/action.yml

@@ -0,0 +1,33 @@
+name: update_package_lock_nodejs
+description: "Update nodejs's package.lock"
+
+inputs:
+  github_token:
+    required: true
+    description: "github token for the repo"
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 20
+    - name: Set git configs
+      shell: bash
+      run: |
+        git config user.name 'Lance Release'
+        git config user.email 'lance-dev@lancedb.com'
+    - name: Update package-lock.json file
+      working-directory: ./nodejs
+      run: |
+        npm install
+        git add package-lock.json
+        git commit -m "Updating package-lock.json"
+      shell: bash
+    - name: Push changes
+      if: ${{ inputs.dry_run }} == "false"
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ inputs.github_token }}
+        branch: main
+        tags: true

.github/workflows/update_package_lock_run_nodejs.yml

@@ -0,0 +1,19 @@
+name: Update NodeJs package-lock.json
+
+on:
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          persist-credentials: false
+          fetch-depth: 0
+          lfs: true
+      - uses: ./.github/workflows/update_package_lock_nodejs
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.gitignore

@@ -34,6 +34,7 @@ python/dist
 node/dist
 node/examples/**/package-lock.json
 node/examples/**/dist
+nodejs/lancedb/native*
 dist
 
 ## Rust

Cargo.toml

@@ -39,3 +39,5 @@ pin-project = "1.0.7"
 snafu = "0.7.4"
 url = "2"
 num-traits = "0.2"
+regex = "1.10"
+lazy_static = "1"

ci/build_linux_artifacts_nodejs.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+ARCH=${1:-x86_64}
+
+# We pass down the current user so that when we later mount the local files
+# into the container, the files are accessible by the current user.
+pushd ci/manylinux_nodejs
+docker build \
+    -t lancedb-nodejs-manylinux \
+    --build-arg="ARCH=$ARCH" \
+    --build-arg="DOCKER_USER=$(id -u)" \
+    --progress=plain \
+    .
+popd
+
+# We turn on memory swap to avoid OOM killer
+docker run \
+    -v $(pwd):/io -w /io \
+    --memory-swap=-1 \
+    lancedb-nodejs-manylinux \
+    bash ci/manylinux_nodejs/build.sh $ARCH

ci/build_macos_artifacts_nodejs.sh

@@ -0,0 +1,34 @@
+# Builds the macOS artifacts (nodejs binaries).
+# Usage: ./ci/build_macos_artifacts_nodejs.sh [target]
+# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
+set -e
+
+prebuild_rust() {
+    # Building here for the sake of easier debugging.
+    pushd rust/lancedb
+    echo "Building rust library for $1"
+    export RUST_BACKTRACE=1
+    cargo build --release --target $1
+    popd
+}
+
+build_node_binaries() {
+    pushd nodejs
+    echo "Building nodejs library for $1"
+    export RUST_TARGET=$1
+    npm run build-release
+    popd
+}
+
+if [ -n "$1" ]; then
+    targets=$1
+else
+    targets="x86_64-apple-darwin aarch64-apple-darwin"
+fi
+
+echo "Building artifacts for targets: $targets"
+for target in $targets
+    do
+    prebuild_rust $target
+    build_node_binaries $target
+done

ci/build_windows_artifacts_nodejs.ps1

@@ -0,0 +1,41 @@
+# Builds the Windows artifacts (nodejs binaries).
+# Usage:  .\ci\build_windows_artifacts_nodejs.ps1 [target]
+# Targets supported:
+# - x86_64-pc-windows-msvc
+# - i686-pc-windows-msvc
+
+function Prebuild-Rust {
+    param (
+        [string]$target
+    )
+
+    # Building here for the sake of easier debugging.
+    Push-Location -Path "rust/lancedb"
+    Write-Host "Building rust library for $target"
+    $env:RUST_BACKTRACE=1
+    cargo build --release --target $target
+    Pop-Location
+}
+
+function Build-NodeBinaries {
+    param (
+        [string]$target
+    )
+
+    Push-Location -Path "nodejs"
+    Write-Host "Building nodejs library for $target"
+    $env:RUST_TARGET=$target
+    npm run build-release
+    Pop-Location
+}
+
+$targets = $args[0]
+if (-not $targets) {
+    $targets = "x86_64-pc-windows-msvc"
+}
+
+Write-Host "Building artifacts for targets: $targets"
+foreach ($target in $targets) {
+    Prebuild-Rust $target
+    Build-NodeBinaries $target
+}

ci/manylinux_nodejs/Dockerfile

@@ -0,0 +1,31 @@
+# Many linux dockerfile with Rust, Node, and Lance dependencies installed.
+# This container allows building the node modules native libraries in an
+# environment with a very old glibc, so that we are compatible with a wide
+# range of linux distributions.
+ARG ARCH=x86_64
+
+FROM quay.io/pypa/manylinux2014_${ARCH}
+
+ARG ARCH=x86_64
+ARG DOCKER_USER=default_user
+
+# Install static openssl
+COPY install_openssl.sh install_openssl.sh
+RUN ./install_openssl.sh ${ARCH} > /dev/null
+
+# Protobuf is also installed as root.
+COPY install_protobuf.sh install_protobuf.sh
+RUN ./install_protobuf.sh ${ARCH}
+
+ENV DOCKER_USER=${DOCKER_USER}
+# Create a group and user
+RUN echo ${ARCH} && adduser --user-group --create-home --uid ${DOCKER_USER} build_user
+
+# We switch to the user to install Rust and Node, since those like to be
+# installed at the user level.
+USER ${DOCKER_USER}
+
+COPY prepare_manylinux_node.sh prepare_manylinux_node.sh
+RUN cp /prepare_manylinux_node.sh $HOME/ && \
+    cd $HOME && \
+    ./prepare_manylinux_node.sh ${ARCH}

ci/manylinux_nodejs/build.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Builds the nodejs module for manylinux. Invoked by ci/build_linux_artifacts_nodejs.sh.
+set -e
+ARCH=${1:-x86_64}
+
+if [ "$ARCH" = "x86_64" ]; then
+    export OPENSSL_LIB_DIR=/usr/local/lib64/
+else
+    export OPENSSL_LIB_DIR=/usr/local/lib/
+fi
+export OPENSSL_STATIC=1
+export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
+
+source $HOME/.bashrc
+
+cd nodejs
+npm ci
+npm run build-release

ci/manylinux_nodejs/install_openssl.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Builds openssl from source so we can statically link to it
+
+# this is to avoid the error we get with the system installation:
+# /usr/bin/ld: <library>: version node not found for symbol SSLeay@@OPENSSL_1.0.1
+# /usr/bin/ld: failed to set dynamic section sizes: Bad value
+set -e
+
+git clone -b OpenSSL_1_1_1u \
+    --single-branch \
+    https://github.com/openssl/openssl.git
+
+pushd openssl
+
+if [[ $1 == x86_64* ]]; then
+    ARCH=linux-x86_64
+else
+    # gnu target
+    ARCH=linux-aarch64
+fi
+
+./Configure no-shared $ARCH
+
+make
+
+make install

ci/manylinux_nodejs/install_protobuf.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Installs protobuf compiler. Should be run as root.
+set -e
+
+if [[ $1 == x86_64* ]]; then
+    ARCH=x86_64
+else
+    # gnu target
+    ARCH=aarch_64
+fi
+
+PB_REL=https://github.com/protocolbuffers/protobuf/releases
+PB_VERSION=23.1
+curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
+unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local

ci/manylinux_nodejs/prepare_manylinux_node.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+
+install_node() {
+    echo "Installing node..."
+
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+
+    source "$HOME"/.bashrc
+
+    nvm install --no-progress 16
+}
+
+install_rust() {
+    echo "Installing rust..."
+    curl https://sh.rustup.rs -sSf | bash -s -- -y
+    export PATH="$PATH:/root/.cargo/bin"
+}
+
+install_node
+install_rust

docs/src/extra_js/init_ask_ai_widget.js

@@ -1,11 +1,79 @@
-document.addEventListener("DOMContentLoaded", function () {
+// Creates an SVG robot icon (from Lucide)
-    var script = document.createElement("script");
+function robotSVG() {
-    script.src = "https://widget.kapa.ai/kapa-widget.bundle.js";
+  var svg = document.createElementNS("http://www.w3.org/2000/svg", "svg");
-    script.setAttribute("data-website-id", "c5881fae-cec0-490b-b45e-d83d131d4f25");
+  svg.setAttribute("width", "24");
-    script.setAttribute("data-project-name", "LanceDB");
+  svg.setAttribute("height", "24");
-    script.setAttribute("data-project-color", "#000000");
+  svg.setAttribute("viewBox", "0 0 24 24");
-    script.setAttribute("data-project-logo", "https://avatars.githubusercontent.com/u/108903835?s=200&v=4");
+  svg.setAttribute("fill", "none");
-  script.setAttribute("data-modal-example-questions","Help me create an IVF_PQ index,How do I do an exhaustive search?,How do I create a LanceDB table?,Can I use my own embedding function?");
+  svg.setAttribute("stroke", "currentColor");
-    script.async = true;
+  svg.setAttribute("stroke-width", "2");
-    document.head.appendChild(script);
+  svg.setAttribute("stroke-linecap", "round");
-  });
+  svg.setAttribute("stroke-linejoin", "round");
+  svg.setAttribute("class", "lucide lucide-bot-message-square");
+
+  var path1 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path1.setAttribute("d", "M12 6V2H8");
+  svg.appendChild(path1);
+
+  var path2 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path2.setAttribute("d", "m8 18-4 4V8a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2Z");
+  svg.appendChild(path2);
+
+  var path3 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path3.setAttribute("d", "M2 12h2");
+  svg.appendChild(path3);
+
+  var path4 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path4.setAttribute("d", "M9 11v2");
+  svg.appendChild(path4);
+
+  var path5 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path5.setAttribute("d", "M15 11v2");
+  svg.appendChild(path5);
+
+  var path6 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path6.setAttribute("d", "M20 12h2");
+  svg.appendChild(path6);
+
+  return svg
+}
+
+// Creates the Fluidic Chatbot buttom
+function fluidicButton() {
+  var btn = document.createElement("a");
+  btn.href = "https://asklancedb.com";
+  btn.target = "_blank";
+  btn.style.position = "fixed";
+  btn.style.fontWeight = "bold";
+  btn.style.fontSize = ".8rem";
+  btn.style.right = "10px";
+  btn.style.bottom = "10px";
+  btn.style.width = "80px";
+  btn.style.height = "80px";
+  btn.style.background = "linear-gradient(135deg, #7C5EFF 0%, #625eff 100%)";
+  btn.style.color = "white";
+  btn.style.borderRadius = "5px";
+  btn.style.display = "flex";
+  btn.style.flexDirection = "column";
+  btn.style.justifyContent = "center";
+  btn.style.alignItems = "center";
+  btn.style.zIndex = "1000";
+  btn.style.opacity = "0";
+  btn.style.boxShadow = "0 0 0 rgba(0, 0, 0, 0)";
+  btn.style.transition = "opacity 0.2s ease-in, box-shadow 0.2s ease-in";
+
+  setTimeout(function() {
+      btn.style.opacity = "1";
+      btn.style.boxShadow = "0 0 .2rem #0000001a,0 .2rem .4rem #0003"
+  }, 0);
+
+  return btn
+}
+
+document.addEventListener("DOMContentLoaded", function() {
+  var btn = fluidicButton()
+  btn.appendChild(robotSVG());
+  var text = document.createTextNode("Ask AI");
+  btn.appendChild(text);
+  document.body.appendChild(btn);
+});

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.4.13",
+  "version": "0.4.14",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -88,10 +88,10 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.4.13",
+    "@lancedb/vectordb-darwin-arm64": "0.4.14",
-    "@lancedb/vectordb-darwin-x64": "0.4.13",
+    "@lancedb/vectordb-darwin-x64": "0.4.14",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.4.13",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.4.14",
-    "@lancedb/vectordb-linux-x64-gnu": "0.4.13",
+    "@lancedb/vectordb-linux-x64-gnu": "0.4.14",
-    "@lancedb/vectordb-win32-x64-msvc": "0.4.13"
+    "@lancedb/vectordb-win32-x64-msvc": "0.4.14"
   }
 }

node/src/index.ts

@@ -24,6 +24,7 @@ import { RemoteConnection } from './remote'
 import { Query } from './query'
 import { isEmbeddingFunction } from './embedding/embedding_function'
 import { type Literal, toSQL } from './util'
+import { type HttpMiddleware } from './middleware'
 
 const {
   databaseNew,
@@ -302,6 +303,18 @@ export interface Connection {
    * @param name The name of the table to drop.
    */
   dropTable(name: string): Promise<void>
+
+  /**
+   * Instrument the behavior of this Connection with middleware.
+   *
+   * The middleware will be called in the order they are added.
+   *
+   * Currently this functionality is only supported for remote Connections.
+   *
+   * @param {HttpMiddleware} - Middleware which will instrument the Connection.
+   * @returns - this Connection instrumented by the passed middleware
+   */
+  withMiddleware(middleware: HttpMiddleware): Connection
 }
 
 /**
@@ -541,6 +554,18 @@ export interface Table<T = number[]> {
    *                    names (e.g. "a").
    */
   dropColumns(columnNames: string[]): Promise<void>
+
+  /**
+   * Instrument the behavior of this Table with middleware.
+   *
+   * The middleware will be called in the order they are added.
+   *
+   * Currently this functionality is only supported for remote tables.
+   *
+   * @param {HttpMiddleware} - Middleware which will instrument the Table.
+   * @returns - this Table instrumented by the passed middleware
+   */
+  withMiddleware(middleware: HttpMiddleware): Table<T>
 }
 
 /**
@@ -795,6 +820,10 @@ export class LocalConnection implements Connection {
   async dropTable (name: string): Promise<void> {
     await databaseDropTable.call(this._db, name)
   }
+
+  withMiddleware (middleware: HttpMiddleware): Connection {
+    return this
+  }
 }
 
 export class LocalTable<T = number[]> implements Table<T> {
@@ -1105,6 +1134,10 @@ export class LocalTable<T = number[]> implements Table<T> {
   async dropColumns (columnNames: string[]): Promise<void> {
     return tableDropColumns.call(this._tbl, columnNames)
   }
+
+  withMiddleware (middleware: HttpMiddleware): Table<T> {
+    return this
+  }
 }
 
 export interface CleanupStats {

node/src/middleware.ts

@@ -0,0 +1,58 @@
+// Copyright 2024 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * Middleware for Remote LanceDB Connection or Table
+ */
+export interface HttpMiddleware {
+  /**
+   * A callback that can be used to instrument the behavior of http requests to remote
+   * tables. It can be used to add headers, modify the request, or even short-circuit
+   * the request and return a response without making the request to the remote endpoint.
+   * It can also be used to modify the response from the remote endpoint.
+   *
+   * @param {RemoteResponse} res - Request to the remote endpoint
+   * @param {onRemoteRequestNext} next - Callback to advance the middleware chain
+   */
+  onRemoteRequest(
+    req: RemoteRequest,
+    next: (req: RemoteRequest) => Promise<RemoteResponse>,
+  ): Promise<RemoteResponse>
+};
+
+export enum Method {
+  GET,
+  POST
+}
+
+/**
+ * A LanceDB Remote HTTP Request
+ */
+export interface RemoteRequest {
+  uri: string
+  method: Method
+  headers: Map<string, string>
+  params?: Map<string, string>
+  body?: any
+}
+
+/**
+ * A LanceDB Remote HTTP Response
+ */
+export interface RemoteResponse {
+  status: number
+  statusText: string
+  headers: Map<string, string>
+  body: () => Promise<any>
+}

node/src/remote/client.ts

@@ -12,13 +12,101 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import axios, { type AxiosResponse } from 'axios'
+import axios, { type AxiosResponse, type ResponseType } from 'axios'
 
 import { tableFromIPC, type Table as ArrowTable } from 'apache-arrow'
 
+import { type RemoteResponse, type RemoteRequest, Method } from '../middleware'
+
+interface HttpLancedbClientMiddleware {
+  onRemoteRequest(
+    req: RemoteRequest,
+    next: (req: RemoteRequest) => Promise<RemoteResponse>,
+  ): Promise<RemoteResponse>
+}
+
+/**
+ * Invoke the middleware chain and at the end call the remote endpoint
+ */
+async function callWithMiddlewares (
+  req: RemoteRequest,
+  middlewares: HttpLancedbClientMiddleware[],
+  opts?: MiddlewareInvocationOptions
+): Promise<RemoteResponse> {
+  async function call (
+    i: number,
+    req: RemoteRequest
+  ): Promise<RemoteResponse> {
+    // if we have reached the end of the middleware chain, make the request
+    if (i > middlewares.length) {
+      const headers = Object.fromEntries(req.headers.entries())
+      const params = Object.fromEntries(req.params?.entries() ?? [])
+      const timeout = 10000
+      let res
+      if (req.method === Method.POST) {
+        res = await axios.post(
+          req.uri,
+          req.body,
+          {
+            headers,
+            params,
+            timeout,
+            responseType: opts?.responseType
+          }
+        )
+      } else {
+        res = await axios.get(
+          req.uri,
+          {
+            headers,
+            params,
+            timeout
+          }
+        )
+      }
+
+      return toLanceRes(res)
+    }
+
+    // call next middleware in chain
+    return await middlewares[i - 1].onRemoteRequest(
+      req,
+      async (req) => {
+        return await call(i + 1, req)
+      }
+    )
+  }
+
+  return await call(1, req)
+}
+
+interface MiddlewareInvocationOptions {
+  responseType?: ResponseType
+}
+
+/**
+ * Marshall the library response into a LanceDB response
+ */
+function toLanceRes (res: AxiosResponse): RemoteResponse {
+  const headers = new Map()
+  for (const h in res.headers) {
+    headers.set(h, res.headers[h])
+  }
+
+  return {
+    status: res.status,
+    statusText: res.statusText,
+    headers,
+    body: async () => {
+      return res.data
+    }
+  }
+}
+
 export class HttpLancedbClient {
   private readonly _url: string
   private readonly _apiKey: () => string
+  private readonly _middlewares: HttpLancedbClientMiddleware[]
 
   public constructor (
     url: string,
@@ -27,6 +115,7 @@ export class HttpLancedbClient {
   ) {
     this._url = url
     this._apiKey = () => apiKey
+    this._middlewares = []
   }
 
   get uri (): string {
@@ -43,74 +132,61 @@ export class HttpLancedbClient {
     columns?: string[],
     filter?: string
   ): Promise<ArrowTable<any>> {
-    const response = await axios.post(
+    const result = await this.post(
-              `${this._url}/v1/table/${tableName}/query/`,
+      `/v1/table/${tableName}/query/`,
-              {
+      {
-                vector,
+        vector,
-                k,
+        k,
-                nprobes,
+        nprobes,
-                refineFactor,
+        refineFactor,
-                columns,
+        columns,
-                filter,
+        filter,
-                prefilter
+        prefilter
-              },
+      },
-              {
+      undefined,
-                headers: {
+      undefined,
-                  'Content-Type': 'application/json',
+      'arraybuffer'
-                  'x-api-key': this._apiKey(),
+    )
-                  ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
+    const table = tableFromIPC(await result.body())
-                },
-                responseType: 'arraybuffer',
-                timeout: 10000
-              }
-    ).catch((err) => {
-      console.error('error: ', err)
-      if (err.response === undefined) {
-        throw new Error(`Network Error: ${err.message as string}`)
-      }
-      return err.response
-    })
-    if (response.status !== 200) {
-      const errorData = new TextDecoder().decode(response.data)
-      throw new Error(
-        `Server Error, status: ${response.status as number}, ` +
-        `message: ${response.statusText as string}: ${errorData}`
-      )
-    }
-
-    const table = tableFromIPC(response.data)
     return table
   }
 
   /**
    * Sent GET request.
    */
-  public async get (path: string, params?: Record<string, string | number>): Promise<AxiosResponse> {
+  public async get (path: string, params?: Record<string, string>): Promise<RemoteResponse> {
-    const response = await axios.get(
+    const req = {
-      `${this._url}${path}`,
+      uri: `${this._url}${path}`,
-      {
+      method: Method.GET,
-        headers: {
+      headers: new Map(Object.entries({
-          'Content-Type': 'application/json',
+        'Content-Type': 'application/json',
-          'x-api-key': this._apiKey(),
+        'x-api-key': this._apiKey(),
-          ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
+        ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
-        },
+      })),
-        params,
+      params: new Map(Object.entries(params ?? {}))
-        timeout: 10000
+    }
-      }
+
-    ).catch((err) => {
+    let response
+    try {
+      response = await callWithMiddlewares(req, this._middlewares)
+      return response
+    } catch (err: any) {
       console.error('error: ', err)
       if (err.response === undefined) {
         throw new Error(`Network Error: ${err.message as string}`)
       }
-      return err.response
+
-    })
+      response = toLanceRes(err.response)
+    }
+
     if (response.status !== 200) {
-      const errorData = new TextDecoder().decode(response.data)
+      const errorData = new TextDecoder().decode(await response.body())
       throw new Error(
-        `Server Error, status: ${response.status as number}, ` +
+        `Server Error, status: ${response.status}, ` +
-        `message: ${response.statusText as string}: ${errorData}`
+        `message: ${response.statusText}: ${errorData}`
       )
     }
+
     return response
   }
 
@@ -120,35 +196,65 @@ export class HttpLancedbClient {
   public async post (
     path: string,
     data?: any,
-    params?: Record<string, string | number>,
+    params?: Record<string, string>,
-    content?: string | undefined
+    content?: string | undefined,
-  ): Promise<AxiosResponse> {
+    responseType?: ResponseType | undefined
-    const response = await axios.post(
+  ): Promise<RemoteResponse> {
-        `${this._url}${path}`,
+    const req = {
-        data,
+      uri: `${this._url}${path}`,
-        {
+      method: Method.POST,
-          headers: {
+      headers: new Map(Object.entries({
-            'Content-Type': content ?? 'application/json',
+        'Content-Type': content ?? 'application/json',
-            'x-api-key': this._apiKey(),
+        'x-api-key': this._apiKey(),
-            ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
+        ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
-          },
+      })),
-          params,
+      params: new Map(Object.entries(params ?? {})),
-          timeout: 30000
+      body: data
-        }
+    }
-    ).catch((err) => {
+
+    let response
+    try {
+      response = await callWithMiddlewares(req, this._middlewares, { responseType })
+
+      // return response
+    } catch (err: any) {
       console.error('error: ', err)
       if (err.response === undefined) {
         throw new Error(`Network Error: ${err.message as string}`)
       }
-      return err.response
+      response = toLanceRes(err.response)
-    })
+    }
+
     if (response.status !== 200) {
-      const errorData = new TextDecoder().decode(response.data)
+      const errorData = new TextDecoder().decode(await response.body())
       throw new Error(
-          `Server Error, status: ${response.status as number}, ` +
+        `Server Error, status: ${response.status}, ` +
-          `message: ${response.statusText as string}: ${errorData}`
+        `message: ${response.statusText}: ${errorData}`
       )
     }
+
     return response
   }
+
+  /**
+   * Instrument this client with middleware
+   * @param mw - The middleware that instruments the client
+   * @returns - an instance of this client instrumented with the middleware
+   */
+  public withMiddleware (mw: HttpLancedbClientMiddleware): HttpLancedbClient {
+    const wrapped = this.clone()
+    wrapped._middlewares.push(mw)
+    return wrapped
+  }
+
+  /**
+   * Make a clone of this client
+   */
+  private clone (): HttpLancedbClient {
+    const clone = new HttpLancedbClient(this._url, this._apiKey(), this._dbName)
+    for (const mw of this._middlewares) {
+      clone._middlewares.push(mw)
+    }
+    return clone
+  }
 }

node/src/remote/index.ts

@@ -39,12 +39,13 @@ import {
   fromTableToStreamBuffer
 } from '../arrow'
 import { toSQL } from '../util'
+import { type HttpMiddleware } from '../middleware'
 
 /**
  * Remote connection.
  */
 export class RemoteConnection implements Connection {
-  private readonly _client: HttpLancedbClient
+  private _client: HttpLancedbClient
   private readonly _dbName: string
 
   constructor (opts: ConnectionOptions) {
@@ -84,10 +85,11 @@ export class RemoteConnection implements Connection {
     limit: number = 10
   ): Promise<string[]> {
     const response = await this._client.get('/v1/table/', {
-      limit,
+      limit: `${limit}`,
       page_token: pageToken
     })
-    return response.data.tables
+    const body = await response.body()
+    return body.tables
   }
 
   async openTable (name: string): Promise<Table>
@@ -163,7 +165,7 @@ export class RemoteConnection implements Connection {
       throw new Error(
         `Server Error, status: ${res.status}, ` +
           // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-          `message: ${res.statusText}: ${res.data}`
+          `message: ${res.statusText}: ${await res.body()}`
       )
     }
 
@@ -177,6 +179,17 @@ export class RemoteConnection implements Connection {
   async dropTable (name: string): Promise<void> {
     await this._client.post(`/v1/table/${name}/drop/`)
   }
+
+  withMiddleware (middleware: HttpMiddleware): Connection {
+    const wrapped = this.clone()
+    wrapped._client = wrapped._client.withMiddleware(middleware)
+    return wrapped
+  }
+
+  private clone (): RemoteConnection {
+    const clone: RemoteConnection = Object.create(RemoteConnection.prototype)
+    return Object.assign(clone, this)
+  }
 }
 
 export class RemoteQuery<T = number[]> extends Query<T> {
@@ -229,7 +242,7 @@ export class RemoteQuery<T = number[]> extends Query<T> {
 // we are using extend until we have next next version release
 // Table and Connection has both been refactored to interfaces
 export class RemoteTable<T = number[]> implements Table<T> {
-  private readonly _client: HttpLancedbClient
+  private _client: HttpLancedbClient
   private readonly _embeddings?: EmbeddingFunction<T>
   private readonly _name: string
 
@@ -256,15 +269,15 @@ export class RemoteTable<T = number[]> implements Table<T> {
   get schema (): Promise<any> {
     return this._client
       .post(`/v1/table/${this._name}/describe/`)
-      .then((res) => {
+      .then(async (res) => {
         if (res.status !== 200) {
           throw new Error(
             `Server Error, status: ${res.status}, ` +
               // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-              `message: ${res.statusText}: ${res.data}`
+              `message: ${res.statusText}: ${await res.body()}`
           )
         }
-        return res.data?.schema
+        return (await res.body())?.schema
       })
   }
 
@@ -320,7 +333,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
       throw new Error(
         `Server Error, status: ${res.status}, ` +
           // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-          `message: ${res.statusText}: ${res.data}`
+          `message: ${res.statusText}: ${await res.body()}`
       )
     }
   }
@@ -346,7 +359,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
       throw new Error(
         `Server Error, status: ${res.status}, ` +
           // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-          `message: ${res.statusText}: ${res.data}`
+          `message: ${res.statusText}: ${await res.body()}`
       )
     }
     return tbl.numRows
@@ -372,7 +385,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
       throw new Error(
         `Server Error, status: ${res.status}, ` +
           // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-          `message: ${res.statusText}: ${res.data}`
+          `message: ${res.statusText}: ${await res.body()}`
       )
     }
     return tbl.numRows
@@ -415,7 +428,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
       throw new Error(
         `Server Error, status: ${res.status}, ` +
           // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-          `message: ${res.statusText}: ${res.data}`
+          `message: ${res.statusText}: ${await res.body()}`
       )
     }
   }
@@ -436,14 +449,14 @@ export class RemoteTable<T = number[]> implements Table<T> {
       throw new Error(
         `Server Error, status: ${res.status}, ` +
           // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-          `message: ${res.statusText}: ${res.data}`
+          `message: ${res.statusText}: ${await res.body()}`
       )
     }
   }
 
   async countRows (): Promise<number> {
     const result = await this._client.post(`/v1/table/${this._name}/describe/`)
-    return result.data?.stats?.num_rows
+    return (await result.body())?.stats?.num_rows
   }
 
   async delete (filter: string): Promise<void> {
@@ -476,7 +489,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
     const results = await this._client.post(
       `/v1/table/${this._name}/index/list/`
     )
-    return results.data.indexes?.map((index: any) => ({
+    return (await results.body()).indexes?.map((index: any) => ({
       columns: index.columns,
       name: index.index_name,
       uuid: index.index_uuid
@@ -487,9 +500,10 @@ export class RemoteTable<T = number[]> implements Table<T> {
     const results = await this._client.post(
       `/v1/table/${this._name}/index/${indexUuid}/stats/`
     )
+    const body = await results.body()
     return {
-      numIndexedRows: results.data.num_indexed_rows,
+      numIndexedRows: body?.num_indexed_rows,
-      numUnindexedRows: results.data.num_unindexed_rows
+      numUnindexedRows: body?.num_unindexed_rows
     }
   }
 
@@ -504,4 +518,15 @@ export class RemoteTable<T = number[]> implements Table<T> {
   async dropColumns (columnNames: string[]): Promise<void> {
     throw new Error('Drop columns is not yet supported in LanceDB Cloud.')
   }
+
+  withMiddleware(middleware: HttpMiddleware): Table<T> {
+    const wrapped = this.clone()
+    wrapped._client = wrapped._client.withMiddleware(middleware)
+    return wrapped
+  }
+
+  private clone (): RemoteTable<T> {
+    const clone: RemoteTable<T> = Object.create(RemoteTable.prototype)
+    return Object.assign(clone, this)
+  }
 }

nodejs/lancedb/arrow.ts

@@ -106,6 +106,9 @@ export class MakeArrowTableOptions {
  * An enhanced version of the {@link makeTable} function from Apache Arrow
  * that supports nested fields and embeddings columns.
  *
+ * (typically you do not need to call this function.  It will be called automatically
+ * when creating a table or adding data to it)
+ *
  * This function converts an array of Record<String, any> (row-major JS objects)
  * to an Arrow Table (a columnar structure)
  *

nodejs/lancedb/embedding/index.ts

@@ -0,0 +1,2 @@
+export { EmbeddingFunction, isEmbeddingFunction } from "./embedding_function";
+export { OpenAIEmbeddingFunction } from "./openai";

nodejs/lancedb/index.ts

@@ -18,9 +18,34 @@ import {
   ConnectionOptions,
 } from "./native.js";
 
-export { ConnectionOptions, WriteOptions, Query } from "./native.js";
+export {
-export { Connection, CreateTableOptions } from "./connection";
+  WriteOptions,
-export { Table, AddDataOptions } from "./table";
+  WriteMode,
+  AddColumnsSql,
+  ColumnAlteration,
+  ConnectionOptions,
+} from "./native.js";
+export {
+  makeArrowTable,
+  MakeArrowTableOptions,
+  Data,
+  VectorColumnOptions,
+} from "./arrow";
+export {
+  Connection,
+  CreateTableOptions,
+  TableNamesOptions,
+} from "./connection";
+export {
+  ExecutableQuery,
+  Query,
+  QueryBase,
+  VectorQuery,
+  RecordBatchIterator,
+} from "./query";
+export { Index, IndexOptions, IvfPqOptions } from "./indices";
+export { Table, AddDataOptions, IndexConfig, UpdateOptions } from "./table";
+export * as embedding from "./embedding";
 
 /**
  * Connect to a LanceDB instance at the given URI.

nodejs/lancedb/native.d.ts

@@ -1,147 +0,0 @@
-/* tslint:disable */
-/* eslint-disable */
-
-/* auto-generated by NAPI-RS */
-
-/** A description of an index currently configured on a column */
-export interface IndexConfig {
-  /** The type of the index */
-  indexType: string
-  /**
-   * The columns in the index
-   *
-   * Currently this is always an array of size 1.  In the future there may
-   * be more columns to represent composite indices.
-   */
-  columns: Array<string>
-}
-/**
- *  A definition of a column alteration. The alteration changes the column at
- * `path` to have the new name `name`, to be nullable if `nullable` is true,
- * and to have the data type `data_type`. At least one of `rename` or `nullable`
- * must be provided.
- */
-export interface ColumnAlteration {
-  /**
-   * The path to the column to alter. This is a dot-separated path to the column.
-   * If it is a top-level column then it is just the name of the column. If it is
-   * a nested column then it is the path to the column, e.g. "a.b.c" for a column
-   * `c` nested inside a column `b` nested inside a column `a`.
-   */
-  path: string
-  /**
-   * The new name of the column. If not provided then the name will not be changed.
-   * This must be distinct from the names of all other columns in the table.
-   */
-  rename?: string
-  /** Set the new nullability. Note that a nullable column cannot be made non-nullable. */
-  nullable?: boolean
-}
-/** A definition of a new column to add to a table. */
-export interface AddColumnsSql {
-  /** The name of the new column. */
-  name: string
-  /**
-   * The values to populate the new column with, as a SQL expression.
-   * The expression can reference other columns in the table.
-   */
-  valueSql: string
-}
-export interface ConnectionOptions {
-  apiKey?: string
-  hostOverride?: string
-  /**
-   * (For LanceDB OSS only): The interval, in seconds, at which to check for
-   * updates to the table from other processes. If None, then consistency is not
-   * checked. For performance reasons, this is the default. For strong
-   * consistency, set this to zero seconds. Then every read will check for
-   * updates from other processes. As a compromise, you can set this to a
-   * non-zero value for eventual consistency. If more than that interval
-   * has passed since the last check, then the table will be checked for updates.
-   * Note: this consistency only applies to read operations. Write operations are
-   * always consistent.
-   */
-  readConsistencyInterval?: number
-}
-/** Write mode for writing a table. */
-export const enum WriteMode {
-  Create = 'Create',
-  Append = 'Append',
-  Overwrite = 'Overwrite'
-}
-/** Write options when creating a Table. */
-export interface WriteOptions {
-  mode?: WriteMode
-}
-export function connect(uri: string, options: ConnectionOptions): Promise<Connection>
-export class Connection {
-  /** Create a new Connection instance from the given URI. */
-  static new(uri: string, options: ConnectionOptions): Promise<Connection>
-  display(): string
-  isOpen(): boolean
-  close(): void
-  /** List all tables in the dataset. */
-  tableNames(startAfter?: string | undefined | null, limit?: number | undefined | null): Promise<Array<string>>
-  /**
-   * Create table from a Apache Arrow IPC (file) buffer.
-   *
-   * Parameters:
-   * - name: The name of the table.
-   * - buf: The buffer containing the IPC file.
-   *
-   */
-  createTable(name: string, buf: Buffer, mode: string): Promise<Table>
-  createEmptyTable(name: string, schemaBuf: Buffer, mode: string): Promise<Table>
-  openTable(name: string): Promise<Table>
-  /** Drop table with the name. Or raise an error if the table does not exist. */
-  dropTable(name: string): Promise<void>
-}
-export class Index {
-  static ivfPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
-  static btree(): Index
-}
-/** Typescript-style Async Iterator over RecordBatches  */
-export class RecordBatchIterator {
-  next(): Promise<Buffer | null>
-}
-export class Query {
-  onlyIf(predicate: string): void
-  select(columns: Array<[string, string]>): void
-  limit(limit: number): void
-  nearestTo(vector: Float32Array): VectorQuery
-  execute(): Promise<RecordBatchIterator>
-}
-export class VectorQuery {
-  column(column: string): void
-  distanceType(distanceType: string): void
-  postfilter(): void
-  refineFactor(refineFactor: number): void
-  nprobes(nprobe: number): void
-  bypassVectorIndex(): void
-  onlyIf(predicate: string): void
-  select(columns: Array<[string, string]>): void
-  limit(limit: number): void
-  execute(): Promise<RecordBatchIterator>
-}
-export class Table {
-  display(): string
-  isOpen(): boolean
-  close(): void
-  /** Return Schema as empty Arrow IPC file. */
-  schema(): Promise<Buffer>
-  add(buf: Buffer, mode: string): Promise<void>
-  countRows(filter?: string | undefined | null): Promise<number>
-  delete(predicate: string): Promise<void>
-  createIndex(index: Index | undefined | null, column: string, replace?: boolean | undefined | null): Promise<void>
-  update(onlyIf: string | undefined | null, columns: Array<[string, string]>): Promise<void>
-  query(): Query
-  vectorSearch(vector: Float32Array): VectorQuery
-  addColumns(transforms: Array<AddColumnsSql>): Promise<void>
-  alterColumns(alterations: Array<ColumnAlteration>): Promise<void>
-  dropColumns(columns: Array<string>): Promise<void>
-  version(): Promise<number>
-  checkout(version: number): Promise<void>
-  checkoutLatest(): Promise<void>
-  restore(): Promise<void>
-  listIndices(): Promise<Array<IndexConfig>>
-}

nodejs/lancedb/native.js

@@ -1,329 +0,0 @@
-/* tslint:disable */
-/* eslint-disable */
-/* prettier-ignore */
-
-/* auto-generated by NAPI-RS */
-
-const { existsSync, readFileSync } = require('fs')
-const { join } = require("path");
-
-const { platform, arch } = process;
-
-let nativeBinding = null;
-let localFileExisted = false;
-let loadError = null;
-
-function isMusl() {
-  // For Node 10
-  if (!process.report || typeof process.report.getReport !== "function") {
-    try {
-      const lddPath = require("child_process")
-        .execSync("which ldd")
-        .toString()
-        .trim();
-      return readFileSync(lddPath, "utf8").includes("musl");
-    } catch (e) {
-      return true;
-    }
-  } else {
-    const { glibcVersionRuntime } = process.report.getReport().header;
-    return !glibcVersionRuntime;
-  }
-}
-
-switch (platform) {
-  case "android":
-    switch (arch) {
-      case "arm64":
-        localFileExisted = existsSync(
-          join(__dirname, "lancedb-nodejs.android-arm64.node"),
-        );
-        try {
-          if (localFileExisted) {
-            nativeBinding = require("./lancedb-nodejs.android-arm64.node");
-          } else {
-            nativeBinding = require("lancedb-android-arm64");
-          }
-        } catch (e) {
-          loadError = e;
-        }
-        break;
-      case "arm":
-        localFileExisted = existsSync(
-          join(__dirname, "lancedb-nodejs.android-arm-eabi.node"),
-        );
-        try {
-          if (localFileExisted) {
-            nativeBinding = require("./lancedb-nodejs.android-arm-eabi.node");
-          } else {
-            nativeBinding = require("lancedb-android-arm-eabi");
-          }
-        } catch (e) {
-          loadError = e;
-        }
-        break;
-      default:
-        throw new Error(`Unsupported architecture on Android ${arch}`);
-    }
-    break;
-  case "win32":
-    switch (arch) {
-      case "x64":
-        localFileExisted = existsSync(
-          join(__dirname, "lancedb-nodejs.win32-x64-msvc.node"),
-        );
-        try {
-          if (localFileExisted) {
-            nativeBinding = require("./lancedb-nodejs.win32-x64-msvc.node");
-          } else {
-            nativeBinding = require("lancedb-win32-x64-msvc");
-          }
-        } catch (e) {
-          loadError = e;
-        }
-        break;
-      case "ia32":
-        localFileExisted = existsSync(
-          join(__dirname, "lancedb-nodejs.win32-ia32-msvc.node"),
-        );
-        try {
-          if (localFileExisted) {
-            nativeBinding = require("./lancedb-nodejs.win32-ia32-msvc.node");
-          } else {
-            nativeBinding = require("lancedb-win32-ia32-msvc");
-          }
-        } catch (e) {
-          loadError = e;
-        }
-        break;
-      case "arm64":
-        localFileExisted = existsSync(
-          join(__dirname, "lancedb-nodejs.win32-arm64-msvc.node"),
-        );
-        try {
-          if (localFileExisted) {
-            nativeBinding = require("./lancedb-nodejs.win32-arm64-msvc.node");
-          } else {
-            nativeBinding = require("lancedb-win32-arm64-msvc");
-          }
-        } catch (e) {
-          loadError = e;
-        }
-        break;
-      default:
-        throw new Error(`Unsupported architecture on Windows: ${arch}`);
-    }
-    break;
-  case "darwin":
-    localFileExisted = existsSync(
-      join(__dirname, "lancedb-nodejs.darwin-universal.node"),
-    );
-    try {
-      if (localFileExisted) {
-        nativeBinding = require("./lancedb-nodejs.darwin-universal.node");
-      } else {
-        nativeBinding = require("lancedb-darwin-universal");
-      }
-      break;
-    } catch {}
-    switch (arch) {
-      case "x64":
-        localFileExisted = existsSync(
-          join(__dirname, "lancedb-nodejs.darwin-x64.node"),
-        );
-        try {
-          if (localFileExisted) {
-            nativeBinding = require("./lancedb-nodejs.darwin-x64.node");
-          } else {
-            nativeBinding = require("lancedb-darwin-x64");
-          }
-        } catch (e) {
-          loadError = e;
-        }
-        break;
-      case "arm64":
-        localFileExisted = existsSync(
-          join(__dirname, "lancedb-nodejs.darwin-arm64.node"),
-        );
-        try {
-          if (localFileExisted) {
-            nativeBinding = require("./lancedb-nodejs.darwin-arm64.node");
-          } else {
-            nativeBinding = require("lancedb-darwin-arm64");
-          }
-        } catch (e) {
-          loadError = e;
-        }
-        break;
-      default:
-        throw new Error(`Unsupported architecture on macOS: ${arch}`);
-    }
-    break;
-  case "freebsd":
-    if (arch !== "x64") {
-      throw new Error(`Unsupported architecture on FreeBSD: ${arch}`);
-    }
-    localFileExisted = existsSync(
-      join(__dirname, "lancedb-nodejs.freebsd-x64.node"),
-    );
-    try {
-      if (localFileExisted) {
-        nativeBinding = require("./lancedb-nodejs.freebsd-x64.node");
-      } else {
-        nativeBinding = require("lancedb-freebsd-x64");
-      }
-    } catch (e) {
-      loadError = e;
-    }
-    break;
-  case "linux":
-    switch (arch) {
-      case "x64":
-        if (isMusl()) {
-          localFileExisted = existsSync(
-            join(__dirname, "lancedb-nodejs.linux-x64-musl.node"),
-          );
-          try {
-            if (localFileExisted) {
-              nativeBinding = require("./lancedb-nodejs.linux-x64-musl.node");
-            } else {
-              nativeBinding = require("lancedb-linux-x64-musl");
-            }
-          } catch (e) {
-            loadError = e;
-          }
-        } else {
-          localFileExisted = existsSync(
-            join(__dirname, "lancedb-nodejs.linux-x64-gnu.node"),
-          );
-          try {
-            if (localFileExisted) {
-              nativeBinding = require("./lancedb-nodejs.linux-x64-gnu.node");
-            } else {
-              nativeBinding = require("lancedb-linux-x64-gnu");
-            }
-          } catch (e) {
-            loadError = e;
-          }
-        }
-        break;
-      case "arm64":
-        if (isMusl()) {
-          localFileExisted = existsSync(
-            join(__dirname, "lancedb-nodejs.linux-arm64-musl.node"),
-          );
-          try {
-            if (localFileExisted) {
-              nativeBinding = require("./lancedb-nodejs.linux-arm64-musl.node");
-            } else {
-              nativeBinding = require("lancedb-linux-arm64-musl");
-            }
-          } catch (e) {
-            loadError = e;
-          }
-        } else {
-          localFileExisted = existsSync(
-            join(__dirname, "lancedb-nodejs.linux-arm64-gnu.node"),
-          );
-          try {
-            if (localFileExisted) {
-              nativeBinding = require("./lancedb-nodejs.linux-arm64-gnu.node");
-            } else {
-              nativeBinding = require("lancedb-linux-arm64-gnu");
-            }
-          } catch (e) {
-            loadError = e;
-          }
-        }
-        break;
-      case "arm":
-        localFileExisted = existsSync(
-          join(__dirname, "lancedb-nodejs.linux-arm-gnueabihf.node"),
-        );
-        try {
-          if (localFileExisted) {
-            nativeBinding = require("./lancedb-nodejs.linux-arm-gnueabihf.node");
-          } else {
-            nativeBinding = require("lancedb-linux-arm-gnueabihf");
-          }
-        } catch (e) {
-          loadError = e;
-        }
-        break;
-      case "riscv64":
-        if (isMusl()) {
-          localFileExisted = existsSync(
-            join(__dirname, "lancedb-nodejs.linux-riscv64-musl.node"),
-          );
-          try {
-            if (localFileExisted) {
-              nativeBinding = require("./lancedb-nodejs.linux-riscv64-musl.node");
-            } else {
-              nativeBinding = require("lancedb-linux-riscv64-musl");
-            }
-          } catch (e) {
-            loadError = e;
-          }
-        } else {
-          localFileExisted = existsSync(
-            join(__dirname, "lancedb-nodejs.linux-riscv64-gnu.node"),
-          );
-          try {
-            if (localFileExisted) {
-              nativeBinding = require("./lancedb-nodejs.linux-riscv64-gnu.node");
-            } else {
-              nativeBinding = require("lancedb-linux-riscv64-gnu");
-            }
-          } catch (e) {
-            loadError = e;
-          }
-        }
-        break;
-      case "s390x":
-        localFileExisted = existsSync(
-          join(__dirname, "lancedb-nodejs.linux-s390x-gnu.node"),
-        );
-        try {
-          if (localFileExisted) {
-            nativeBinding = require("./lancedb-nodejs.linux-s390x-gnu.node");
-          } else {
-            nativeBinding = require("lancedb-linux-s390x-gnu");
-          }
-        } catch (e) {
-          loadError = e;
-        }
-        break;
-      default:
-        throw new Error(`Unsupported architecture on Linux: ${arch}`);
-    }
-    break;
-  default:
-    throw new Error(`Unsupported OS: ${platform}, architecture: ${arch}`);
-}
-
-if (!nativeBinding) {
-  if (loadError) {
-    throw loadError;
-  }
-  throw new Error(`Failed to load native binding`);
-}
-
-const {
-  Connection,
-  Index,
-  RecordBatchIterator,
-  Query,
-  VectorQuery,
-  Table,
-  WriteMode,
-  connect,
-} = nativeBinding;
-
-module.exports.Connection = Connection;
-module.exports.Index = Index;
-module.exports.RecordBatchIterator = RecordBatchIterator;
-module.exports.Query = Query;
-module.exports.VectorQuery = VectorQuery;
-module.exports.Table = Table;
-module.exports.WriteMode = WriteMode;
-module.exports.connect = connect;

nodejs/lancedb/query.ts

@@ -20,7 +20,7 @@ import {
   VectorQuery as NativeVectorQuery,
 } from "./native";
 import { type IvfPqOptions } from "./indices";
-class RecordBatchIterator implements AsyncIterator<RecordBatch> {
+export class RecordBatchIterator implements AsyncIterator<RecordBatch> {
   private promisedInner?: Promise<NativeBatchIterator>;
   private inner?: NativeBatchIterator;
 

nodejs/npm/darwin-arm64/README.md

@@ -1,3 +1,3 @@
-# `lancedb-darwin-arm64`
+# `@lancedb/lancedb-darwin-arm64`
 
-This is the **aarch64-apple-darwin** binary for `lancedb`
+This is the **aarch64-apple-darwin** binary for `@lancedb/lancedb`

nodejs/npm/darwin-arm64/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "lancedb-darwin-arm64",
+  "name": "@lancedb/lancedb-darwin-arm64",
   "version": "0.4.3",
   "os": [
     "darwin"
@@ -11,7 +11,7 @@
   "files": [
     "lancedb.darwin-arm64.node"
   ],
-  "license": "MIT",
+  "license": "Apache 2.0",
   "engines": {
     "node": ">= 18"
   }

nodejs/npm/darwin-x64/README.md

@@ -1,3 +1,3 @@
-# `lancedb-darwin-x64`
+# `@lancedb/lancedb-darwin-x64`
 
-This is the **x86_64-apple-darwin** binary for `lancedb`
+This is the **x86_64-apple-darwin** binary for `@lancedb/lancedb`

nodejs/npm/darwin-x64/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "lancedb-darwin-x64",
+  "name": "@lancedb/lancedb-darwin-x64",
   "version": "0.4.3",
   "os": [
     "darwin"
@@ -11,7 +11,7 @@
   "files": [
     "lancedb.darwin-x64.node"
   ],
-  "license": "MIT",
+  "license": "Apache 2.0",
   "engines": {
     "node": ">= 18"
   }

nodejs/npm/linux-arm64-gnu/README.md

@@ -1,3 +1,3 @@
-# `lancedb-linux-arm64-gnu`
+# `@lancedb/lancedb-linux-arm64-gnu`
 
-This is the **aarch64-unknown-linux-gnu** binary for `lancedb`
+This is the **aarch64-unknown-linux-gnu** binary for `@lancedb/lancedb`

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "lancedb-linux-arm64-gnu",
+  "name": "@lancedb/lancedb-linux-arm64-gnu",
   "version": "0.4.3",
   "os": [
     "linux"
@@ -11,9 +11,9 @@
   "files": [
     "lancedb.linux-arm64-gnu.node"
   ],
-  "license": "MIT",
+  "license": "Apache 2.0",
   "engines": {
-    "node": ">= 10"
+    "node": ">= 18"
   },
   "libc": [
     "glibc"

nodejs/npm/linux-x64-gnu/README.md

@@ -1,3 +1,3 @@
-# `lancedb-linux-x64-gnu`
+# `@lancedb/lancedb-linux-x64-gnu`
 
-This is the **x86_64-unknown-linux-gnu** binary for `lancedb`
+This is the **x86_64-unknown-linux-gnu** binary for `@lancedb/lancedb`

nodejs/npm/linux-x64-gnu/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "lancedb-linux-x64-gnu",
+  "name": "@lancedb/lancedb-linux-x64-gnu",
   "version": "0.4.3",
   "os": [
     "linux"
@@ -11,9 +11,9 @@
   "files": [
     "lancedb.linux-x64-gnu.node"
   ],
-  "license": "MIT",
+  "license": "Apache 2.0",
   "engines": {
-    "node": ">= 10"
+    "node": ">= 18"
   },
   "libc": [
     "glibc"

nodejs/npm/win32-x64-msvc/README.md

@@ -0,0 +1,3 @@
+# `@lancedb/lancedb-win32-x64-msvc`
+
+This is the **x86_64-pc-windows-msvc** binary for `@lancedb/lancedb`

nodejs/npm/win32-x64-msvc/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "@lancedb/lancedb-win32-x64-msvc",
+  "version": "0.4.3",
+  "os": [
+    "win32"
+  ],
+  "cpu": [
+    "x64"
+  ],
+  "main": "lancedb.win32-x64-msvc.node",
+  "files": [
+    "lancedb.win32-x64-msvc.node"
+  ],
+  "license": "Apache 2.0",
+  "engines": {
+    "node": ">= 18"
+  }
+}

nodejs/package-lock.json

@@ -1,11 +1,11 @@
 {
-  "name": "lancedb",
+  "name": "@lancedb/lancedb",
   "version": "0.4.3",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
-      "name": "lancedb",
+      "name": "@lancedb/lancedb",
       "version": "0.4.3",
       "cpu": [
         "x64",
@@ -15,8 +15,12 @@
       "os": [
         "darwin",
         "linux",
-        "windows"
+        "win32"
       ],
+      "dependencies": {
+        "apache-arrow": "^15.0.0",
+        "openai": "^4.29.2"
+      },
       "devDependencies": {
         "@napi-rs/cli": "^2.18.0",
         "@types/jest": "^29.1.2",
@@ -29,6 +33,7 @@
         "eslint-plugin-jsdoc": "^48.2.1",
         "jest": "^29.7.0",
         "prettier": "^3.1.0",
+        "shx": "^0.3.4",
         "tmp": "^0.2.3",
         "ts-jest": "^29.1.2",
         "typedoc": "^0.25.7",
@@ -40,14 +45,11 @@
         "node": ">= 18"
       },
       "optionalDependencies": {
-        "lancedb-darwin-arm64": "0.4.3",
+        "@lancedb/lancedb-darwin-arm64": "0.4.3",
-        "lancedb-darwin-x64": "0.4.3",
+        "@lancedb/lancedb-darwin-x64": "0.4.3",
-        "lancedb-linux-arm64-gnu": "0.4.3",
+        "@lancedb/lancedb-linux-arm64-gnu": "0.4.3",
-        "lancedb-linux-x64-gnu": "0.4.3",
+        "@lancedb/lancedb-linux-x64-gnu": "0.4.3",
-        "openai": "^4.28.4"
+        "@lancedb/lancedb-win32-x64-msvc": "0.4.3"
-      },
-      "peerDependencies": {
-        "apache-arrow": "^15.0.0"
       }
     },
     "node_modules/@75lb/deep-merge": {
@@ -1317,6 +1319,66 @@
         "@jridgewell/sourcemap-codec": "^1.4.14"
       }
     },
+    "node_modules/@lancedb/lancedb-darwin-arm64": {
+      "version": "0.4.3",
+      "resolved": "https://registry.npmjs.org/@lancedb/lancedb-darwin-arm64/-/lancedb-darwin-arm64-0.4.3.tgz",
+      "integrity": "sha512-+kxuWUK9vtLBbjFMkIKeQ32kxK2tgvZRCQaU1I3RJ3+dLmDIVeIj+KJSlMelkKa2QC4JoyHQi9Ty1PdS2DojmQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@lancedb/lancedb-darwin-x64": {
+      "version": "0.4.3",
+      "resolved": "https://registry.npmjs.org/@lancedb/lancedb-darwin-x64/-/lancedb-darwin-x64-0.4.3.tgz",
+      "integrity": "sha512-JYvsSYxTOa/7OMojulz9h0gN2FwvypG/6l6dpLkViZ5LDvRcfVyDTzOLcOJkFn+db4TKeBOVyMWnnpDKaB+jLA==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@lancedb/lancedb-linux-x64-gnu": {
+      "version": "0.4.3",
+      "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-x64-gnu/-/lancedb-linux-x64-gnu-0.4.3.tgz",
+      "integrity": "sha512-jDANHchWNGmu1wfAyBk0apoFlLxtJ7FRc31pAQ3tKE4fwlgG7bUcaTX6s5C3vMNWXnyQLQtVuWZNXi2nVj879g==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@lancedb/lancedb-win32-x64-msvc": {
+      "version": "0.4.3",
+      "resolved": "https://registry.npmjs.org/@lancedb/lancedb-win32-x64-msvc/-/lancedb-win32-x64-msvc-0.4.3.tgz",
+      "integrity": "sha512-qADveXyv4YzllIbOOq8soqFfL7p7I35uhrD3PcTvj4Qxuo6q7pgQWQz2Mt3kGBpyPkH2yE4wWAGJhayShLRbiQ==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">= 18"
+      }
+    },
     "node_modules/@napi-rs/cli": {
       "version": "2.18.0",
       "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.18.0.tgz",
@@ -1396,7 +1458,6 @@
       "version": "0.5.6",
       "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.6.tgz",
       "integrity": "sha512-aYX01Ke9hunpoCexYAgQucEpARGQ5w/cqHFrIR+e9gdKb1QWTsVJuTJ2ozQzIAxLyRQe/m+2RqzkyOOGiMKRQA==",
-      "peer": true,
       "dependencies": {
         "tslib": "^2.4.0"
       }
@@ -1445,8 +1506,7 @@
     "node_modules/@types/command-line-args": {
       "version": "5.2.3",
       "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.3.tgz",
-      "integrity": "sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw==",
+      "integrity": "sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw=="
-      "peer": true
     },
     "node_modules/@types/command-line-usage": {
       "version": "5.0.2",
@@ -1514,7 +1574,6 @@
       "version": "2.6.11",
       "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.11.tgz",
       "integrity": "sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==",
-      "optional": true,
       "dependencies": {
         "@types/node": "*",
         "form-data": "^4.0.0"
@@ -1783,7 +1842,6 @@
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
       "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
-      "optional": true,
       "dependencies": {
         "event-target-shim": "^5.0.0"
       },
@@ -1816,7 +1874,6 @@
       "version": "4.5.0",
       "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz",
       "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==",
-      "optional": true,
       "dependencies": {
         "humanize-ms": "^1.2.1"
       },
@@ -1913,7 +1970,6 @@
       "version": "15.0.0",
       "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-15.0.0.tgz",
       "integrity": "sha512-e6aunxNKM+woQf137ny3tp/xbLjFJS2oGQxQhYGqW6dGeIwNV1jOeEAeR6sS2jwAI2qLO83gYIP2MBz02Gw5Xw==",
-      "peer": true,
       "dependencies": {
         "@swc/helpers": "^0.5.2",
         "@types/command-line-args": "^5.2.1",
@@ -2001,8 +2057,7 @@
     "node_modules/asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
-      "optional": true
     },
     "node_modules/babel-jest": {
       "version": "29.7.0",
@@ -2129,8 +2184,7 @@
     "node_modules/base-64": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
-      "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==",
+      "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA=="
-      "optional": true
     },
     "node_modules/brace-expansion": {
       "version": "1.1.11",
@@ -2296,7 +2350,6 @@
       "version": "0.0.2",
       "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
       "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==",
-      "optional": true,
       "engines": {
         "node": "*"
       }
@@ -2357,7 +2410,6 @@
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
       "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
-      "optional": true,
       "dependencies": {
         "delayed-stream": "~1.0.0"
       },
@@ -2469,7 +2521,6 @@
       "version": "0.0.2",
       "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
       "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==",
-      "optional": true,
       "engines": {
         "node": "*"
       }
@@ -2530,7 +2581,6 @@
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
       "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
-      "optional": true,
       "engines": {
         "node": ">=0.4.0"
       }
@@ -2557,7 +2607,6 @@
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz",
       "integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==",
-      "optional": true,
       "dependencies": {
         "base-64": "^0.1.0",
         "md5": "^2.3.0"
@@ -2862,7 +2911,6 @@
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
       "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
-      "optional": true,
       "engines": {
         "node": ">=6"
       }
@@ -3024,7 +3072,6 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
       "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
-      "optional": true,
       "dependencies": {
         "asynckit": "^0.4.0",
         "combined-stream": "^1.0.8",
@@ -3037,14 +3084,12 @@
     "node_modules/form-data-encoder": {
       "version": "1.7.2",
       "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz",
-      "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==",
+      "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="
-      "optional": true
     },
     "node_modules/formdata-node": {
       "version": "4.4.1",
       "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz",
       "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==",
-      "optional": true,
       "dependencies": {
         "node-domexception": "1.0.0",
         "web-streams-polyfill": "4.0.0-beta.3"
@@ -3057,7 +3102,6 @@
       "version": "4.0.0-beta.3",
       "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",
       "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==",
-      "optional": true,
       "engines": {
         "node": ">= 14"
       }
@@ -3272,7 +3316,6 @@
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
       "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
-      "optional": true,
       "dependencies": {
         "ms": "^2.0.0"
       }
@@ -3355,6 +3398,15 @@
       "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
       "dev": true
     },
+    "node_modules/interpret": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz",
+      "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==",
+      "dev": true,
+      "engines": {
+        "node": ">= 0.10"
+      }
+    },
     "node_modules/is-arrayish": {
       "version": "0.2.1",
       "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz",
@@ -3364,8 +3416,7 @@
     "node_modules/is-buffer": {
       "version": "1.1.6",
       "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
-      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
+      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
-      "optional": true
     },
     "node_modules/is-builtin-module": {
       "version": "3.2.1",
@@ -4458,7 +4509,6 @@
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
       "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
-      "optional": true,
       "dependencies": {
         "charenc": "0.0.2",
         "crypt": "0.0.2",
@@ -4497,7 +4547,6 @@
       "version": "1.52.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
       "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
-      "optional": true,
       "engines": {
         "node": ">= 0.6"
       }
@@ -4506,7 +4555,6 @@
       "version": "2.1.35",
       "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
       "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
-      "optional": true,
       "dependencies": {
         "mime-db": "1.52.0"
       },
@@ -4538,8 +4586,7 @@
     "node_modules/ms": {
       "version": "2.1.3",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
-      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
-      "optional": true
     },
     "node_modules/natural-compare": {
       "version": "1.4.0",
@@ -4567,7 +4614,6 @@
           "url": "https://paypal.me/jimmywarting"
         }
       ],
-      "optional": true,
       "engines": {
         "node": ">=10.5.0"
       }
@@ -4576,7 +4622,6 @@
       "version": "2.7.0",
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
       "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
-      "optional": true,
       "dependencies": {
         "whatwg-url": "^5.0.0"
       },
@@ -4623,10 +4668,9 @@
       }
     },
     "node_modules/openai": {
-      "version": "4.28.4",
+      "version": "4.29.2",
-      "resolved": "https://registry.npmjs.org/openai/-/openai-4.28.4.tgz",
+      "resolved": "https://registry.npmjs.org/openai/-/openai-4.29.2.tgz",
-      "integrity": "sha512-RNIwx4MT/F0zyizGcwS+bXKLzJ8QE9IOyigDG/ttnwB220d58bYjYFp0qjvGwEFBO6+pvFVIDABZPGDl46RFsg==",
+      "integrity": "sha512-cPkT6zjEcE4qU5OW/SoDDuXEsdOLrXlAORhzmaguj5xZSPlgKvLhi27sFWhLKj07Y6WKNWxcwIbzm512FzTBNQ==",
-      "optional": true,
       "dependencies": {
         "@types/node": "^18.11.18",
         "@types/node-fetch": "^2.6.4",
@@ -4643,10 +4687,9 @@
       }
     },
     "node_modules/openai/node_modules/@types/node": {
-      "version": "18.19.20",
+      "version": "18.19.26",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.20.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.26.tgz",
-      "integrity": "sha512-SKXZvI375jkpvAj8o+5U2518XQv76mAsixqfXiVyWyXZbVWQK25RurFovYpVIxVzul0rZoH58V/3SkEnm7s3qA==",
+      "integrity": "sha512-+wiMJsIwLOYCvUqSdKTrfkS8mpTp+MPINe6+Np4TAGFWWRWiBQ5kSq9nZGCSPkzx9mvT+uEukzpX4MOSCydcvw==",
-      "optional": true,
       "dependencies": {
         "undici-types": "~5.26.4"
       }
@@ -4996,6 +5039,18 @@
       "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==",
       "dev": true
     },
+    "node_modules/rechoir": {
+      "version": "0.6.2",
+      "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz",
+      "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==",
+      "dev": true,
+      "dependencies": {
+        "resolve": "^1.1.6"
+      },
+      "engines": {
+        "node": ">= 0.10"
+      }
+    },
     "node_modules/repeat-string": {
       "version": "1.6.1",
       "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz",
@@ -5145,6 +5200,23 @@
         "node": ">=8"
       }
     },
+    "node_modules/shelljs": {
+      "version": "0.8.5",
+      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz",
+      "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==",
+      "dev": true,
+      "dependencies": {
+        "glob": "^7.0.0",
+        "interpret": "^1.0.0",
+        "rechoir": "^0.6.2"
+      },
+      "bin": {
+        "shjs": "bin/shjs"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/shiki": {
       "version": "0.14.7",
       "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.14.7.tgz",
@@ -5157,6 +5229,22 @@
         "vscode-textmate": "^8.0.0"
       }
     },
+    "node_modules/shx": {
+      "version": "0.3.4",
+      "resolved": "https://registry.npmjs.org/shx/-/shx-0.3.4.tgz",
+      "integrity": "sha512-N6A9MLVqjxZYcVn8hLmtneQWIJtp8IKzMP4eMnx+nqkvXoqinUPCbUFLp2UcWTEIUONhlk0ewxr/jaVGlc+J+g==",
+      "dev": true,
+      "dependencies": {
+        "minimist": "^1.2.3",
+        "shelljs": "^0.8.5"
+      },
+      "bin": {
+        "shx": "lib/cli.js"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/signal-exit": {
       "version": "3.0.7",
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
@@ -5432,8 +5520,7 @@
     "node_modules/tr46": {
       "version": "0.0.3",
       "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
-      "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
+      "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
-      "optional": true
     },
     "node_modules/ts-api-utils": {
       "version": "1.0.3",
@@ -5929,7 +6016,6 @@
       "version": "3.3.3",
       "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz",
       "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==",
-      "optional": true,
       "engines": {
         "node": ">= 8"
       }
@@ -5937,14 +6023,12 @@
     "node_modules/webidl-conversions": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
-      "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
+      "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
-      "optional": true
     },
     "node_modules/whatwg-url": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
       "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
-      "optional": true,
       "dependencies": {
         "tr46": "~0.0.3",
         "webidl-conversions": "^3.0.0"

nodejs/package.json

@@ -1,17 +1,18 @@
 {
-  "name": "lancedb",
+  "name": "@lancedb/lancedb",
   "version": "0.4.3",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "napi": {
-    "name": "lancedb-nodejs",
+    "name": "lancedb",
     "triples": {
       "defaults": false,
       "additional": [
         "aarch64-apple-darwin",
         "aarch64-unknown-linux-gnu",
         "x86_64-apple-darwin",
-        "x86_64-unknown-linux-gnu"
+        "x86_64-unknown-linux-gnu",
+        "x86_64-pc-windows-msvc"
       ]
     }
   },
@@ -28,6 +29,7 @@
     "eslint-plugin-jsdoc": "^48.2.1",
     "jest": "^29.7.0",
     "prettier": "^3.1.0",
+    "shx": "^0.3.4",
     "tmp": "^0.2.3",
     "ts-jest": "^29.1.2",
     "typedoc": "^0.25.7",
@@ -48,13 +50,14 @@
   "os": [
     "darwin",
     "linux",
-    "windows"
+    "win32"
   ],
   "scripts": {
     "artifacts": "napi artifacts",
-    "build:native": "napi build --platform --release --js lancedb/native.js --dts lancedb/native.d.ts dist/",
     "build:debug": "napi build --platform --dts ../lancedb/native.d.ts --js ../lancedb/native.js dist/",
-    "build": "npm run build:debug && tsc -b",
+    "build:release": "napi build --platform --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js dist/",
+    "build": "npm run build:debug && tsc -b && shx cp lancedb/native.d.ts dist/native.d.ts",
+    "build-release": "npm run build:release && tsc -b && shx cp lancedb/native.d.ts dist/native.d.ts",
     "chkformat": "prettier . --check",
     "docs": "typedoc --plugin typedoc-plugin-markdown lancedb/index.ts",
     "lint": "eslint lancedb && eslint __test__",
@@ -64,13 +67,14 @@
     "version": "napi version"
   },
   "optionalDependencies": {
-    "lancedb-darwin-arm64": "0.4.3",
+    "@lancedb/lancedb-darwin-arm64": "0.4.3",
-    "lancedb-darwin-x64": "0.4.3",
+    "@lancedb/lancedb-darwin-x64": "0.4.3",
-    "lancedb-linux-arm64-gnu": "0.4.3",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.4.3",
-    "lancedb-linux-x64-gnu": "0.4.3",
+    "@lancedb/lancedb-linux-x64-gnu": "0.4.3",
-    "openai": "^4.28.4"
+    "@lancedb/lancedb-win32-x64-msvc": "0.4.3"
   },
-  "peerDependencies": {
+  "dependencies": {
+    "openai": "^4.29.2",
     "apache-arrow": "^15.0.0"
   }
 }

python/python/lancedb/db.py

@@ -31,7 +31,13 @@ from lancedb.utils.events import register_event
 from ._lancedb import connect as lancedb_connect
 from .pydantic import LanceModel
 from .table import AsyncTable, LanceTable, Table, _sanitize_data
-from .util import fs_from_uri, get_uri_location, get_uri_scheme, join_uri
+from .util import (
+    fs_from_uri,
+    get_uri_location,
+    get_uri_scheme,
+    join_uri,
+    validate_table_name,
+)
 
 if TYPE_CHECKING:
     from datetime import timedelta
@@ -387,6 +393,7 @@ class LanceDBConnection(DBConnection):
         """
         if mode.lower() not in ["create", "overwrite"]:
             raise ValueError("mode must be either 'create' or 'overwrite'")
+        validate_table_name(name)
 
         tbl = LanceTable.create(
             self,

python/python/lancedb/remote/db.py

@@ -26,6 +26,7 @@ from ..db import DBConnection
 from ..embeddings import EmbeddingFunctionConfig
 from ..pydantic import LanceModel
 from ..table import Table, _sanitize_data
+from ..util import validate_table_name
 from .arrow import to_ipc_binary
 from .client import ARROW_STREAM_CONTENT_TYPE, RestfulLanceDBClient
 from .errors import LanceDBClientError
@@ -223,6 +224,7 @@ class RemoteDBConnection(DBConnection):
         LanceTable(table4)
 
         """
+        validate_table_name(name)
         if data is None and schema is None:
             raise ValueError("Either data or schema must be provided.")
         if embedding_functions is not None:

python/python/lancedb/util.py

@@ -25,6 +25,8 @@ import numpy as np
 import pyarrow as pa
 import pyarrow.fs as pa_fs
 
+from ._lancedb import validate_table_name as native_validate_table_name
+
 
 def safe_import_adlfs():
     try:
@@ -286,3 +288,8 @@ def deprecated(func):
         return func(*args, **kwargs)
 
     return new_func
+
+
+def validate_table_name(name: str):
+    """Verify the table name is valid."""
+    native_validate_table_name(name)

python/python/tests/test_db.py

@@ -521,3 +521,15 @@ def test_prefilter_with_index(tmp_path):
         .to_arrow()
     )
     assert table.num_rows == 1
+
+
+def test_create_table_with_invalid_names(tmp_path):
+    db = lancedb.connect(uri=tmp_path)
+    data = [{"vector": np.random.rand(128), "item": "foo"} for i in range(10)]
+    with pytest.raises(ValueError):
+        db.create_table("foo/bar", data)
+    with pytest.raises(ValueError):
+        db.create_table("foo bar", data)
+    with pytest.raises(ValueError):
+        db.create_table("foo$$bar", data)
+    db.create_table("foo.bar", data)

python/src/lib.rs

@@ -42,6 +42,7 @@ pub fn _lancedb(_py: Python, m: &PyModule) -> PyResult<()> {
     m.add_class::<VectorQuery>()?;
     m.add_class::<RecordBatchStream>()?;
     m.add_function(wrap_pyfunction!(connect, m)?)?;
+    m.add_function(wrap_pyfunction!(util::validate_table_name, m)?)?;
     m.add("__version__", env!("CARGO_PKG_VERSION"))?;
     Ok(())
 }

python/src/util.rs

@@ -3,7 +3,7 @@ use std::sync::Mutex;
 use lancedb::DistanceType;
 use pyo3::{
     exceptions::{PyRuntimeError, PyValueError},
-    PyResult,
+    pyfunction, PyResult,
 };
 
 /// A wrapper around a rust builder
@@ -49,3 +49,9 @@ pub fn parse_distance_type(distance_type: impl AsRef<str>) -> PyResult<DistanceT
         ))),
     }
 }
+
+#[pyfunction]
+pub(crate) fn validate_table_name(table_name: &str) -> PyResult<()> {
+    lancedb::utils::validate_table_name(table_name)
+        .map_err(|e| PyValueError::new_err(e.to_string()))
+}

rust/ffi/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-node"
-version = "0.4.13"
+version = "0.4.14"
 description = "Serverless, low-latency vector database for AI applications"
 license.workspace = true
 edition.workspace = true

rust/lancedb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.4.13"
+version = "0.4.14"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
@@ -22,6 +22,7 @@ chrono = { workspace = true }
 object_store = { workspace = true }
 snafu = { workspace = true }
 half = { workspace = true }
+lazy_static.workspace = true
 lance = { workspace = true }
 lance-index = { workspace = true }
 lance-linalg = { workspace = true }
@@ -34,11 +35,10 @@ bytes = "1"
 futures.workspace = true
 num-traits.workspace = true
 url.workspace = true
+regex.workspace = true
 serde = { version = "^1" }
 serde_json = { version = "1" }
-
 # For remote feature
-
 reqwest = { version = "0.11.24", features = ["gzip", "json"], optional = true }
 
 [dev-dependencies]

rust/lancedb/src/connection.rs

@@ -31,6 +31,7 @@ use crate::arrow::IntoArrow;
 use crate::error::{CreateDirSnafu, Error, InvalidTableNameSnafu, Result};
 use crate::io::object_store::MirroringObjectStoreWrapper;
 use crate::table::{NativeTable, WriteOptions};
+use crate::utils::validate_table_name;
 use crate::Table;
 
 pub const LANCE_FILE_EXTENSION: &str = "lance";
@@ -675,13 +676,18 @@ impl Database {
 
     /// Get the URI of a table in the database.
     fn table_uri(&self, name: &str) -> Result<String> {
+        validate_table_name(name)?;
+
         let path = Path::new(&self.uri);
         let table_uri = path.join(format!("{}.{}", name, LANCE_FILE_EXTENSION));
 
         let mut uri = table_uri
             .as_path()
             .to_str()
-            .context(InvalidTableNameSnafu { name })?
+            .context(InvalidTableNameSnafu {
+                name,
+                reason: "Name is not valid URL",
+            })?
             .to_string();
 
         // If there are query string set on the connection, propagate to lance

rust/lancedb/src/error.rs

@@ -20,8 +20,8 @@ use snafu::Snafu;
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub(crate)))]
 pub enum Error {
-    #[snafu(display("Invalid table name: {name}"))]
+    #[snafu(display("Invalid table name (\"{name}\"): {reason}"))]
-    InvalidTableName { name: String },
+    InvalidTableName { name: String, reason: String },
     #[snafu(display("Invalid input, {message}"))]
     InvalidInput { message: String },
     #[snafu(display("Table '{name}' was not found"))]

rust/lancedb/src/lib.rs

@@ -230,9 +230,9 @@ pub enum DistanceType {
 impl From<DistanceType> for LanceDistanceType {
     fn from(value: DistanceType) -> Self {
         match value {
-            DistanceType::L2 => LanceDistanceType::L2,
+            DistanceType::L2 => Self::L2,
-            DistanceType::Cosine => LanceDistanceType::Cosine,
+            DistanceType::Cosine => Self::Cosine,
-            DistanceType::Dot => LanceDistanceType::Dot,
+            DistanceType::Dot => Self::Dot,
         }
     }
 }
@@ -240,9 +240,9 @@ impl From<DistanceType> for LanceDistanceType {
 impl From<LanceDistanceType> for DistanceType {
     fn from(value: LanceDistanceType) -> Self {
         match value {
-            LanceDistanceType::L2 => DistanceType::L2,
+            LanceDistanceType::L2 => Self::L2,
-            LanceDistanceType::Cosine => DistanceType::Cosine,
+            LanceDistanceType::Cosine => Self::Cosine,
-            LanceDistanceType::Dot => DistanceType::Dot,
+            LanceDistanceType::Dot => Self::Dot,
         }
     }
 }
@@ -251,7 +251,7 @@ impl<'a> TryFrom<&'a str> for DistanceType {
     type Error = <LanceDistanceType as TryFrom<&'a str>>::Error;
 
     fn try_from(value: &str) -> std::prelude::v1::Result<Self, Self::Error> {
-        LanceDistanceType::try_from(value).map(DistanceType::from)
+        LanceDistanceType::try_from(value).map(Self::from)
     }
 }
 

rust/lancedb/src/table.rs

@@ -854,6 +854,7 @@ impl NativeTable {
             .to_str()
             .ok_or(Error::InvalidTableName {
                 name: uri.to_string(),
+                reason: "Table name is not valid URL".to_string(),
             })?;
         Ok(name.to_string())
     }
@@ -1197,7 +1198,7 @@ impl NativeTable {
                 if dim != query_vector.len() as i32 {
                     return Err(Error::InvalidInput {
                         message: format!(
-                            "The dimension of the query vector does not match with the dimension of the vector column '{}': 
+                            "The dimension of the query vector does not match with the dimension of the vector column '{}':
                                 query dim={}, expected vector dim={}",
                             column,
                             query_vector.len(),

rust/lancedb/src/utils.rs

@@ -1,12 +1,30 @@
+// Copyright 2024 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::sync::Arc;
 
 use arrow_schema::Schema;
-
 use lance::dataset::{ReadParams, WriteParams};
 use lance::io::{ObjectStoreParams, WrappingObjectStore};
+use lazy_static::lazy_static;
 
 use crate::error::{Error, Result};
 
+lazy_static! {
+    static ref TABLE_NAME_REGEX: regex::Regex = regex::Regex::new(r"^[a-zA-Z0-9_\-\.]+$").unwrap();
+}
+
 pub trait PatchStoreParam {
     fn patch_with_store_wrapper(
         self,
@@ -64,6 +82,25 @@ impl PatchReadParam for ReadParams {
     }
 }
 
+/// Validate table name.
+pub fn validate_table_name(name: &str) -> Result<()> {
+    if name.is_empty() {
+        return Err(Error::InvalidTableName {
+            name: name.to_string(),
+            reason: "Table names cannot be empty strings".to_string(),
+        });
+    }
+    if !TABLE_NAME_REGEX.is_match(name) {
+        return Err(Error::InvalidTableName {
+            name: name.to_string(),
+            reason:
+                "Table names can only contain alphanumeric characters, underscores, hyphens, and periods"
+                    .to_string(),
+        });
+    }
+    Ok(())
+}
+
 /// Find one default column to create index.
 pub(crate) fn default_vector_column(schema: &Schema, dim: Option<i32>) -> Result<String> {
     // Try to find one fixed size list array column.
@@ -145,4 +182,20 @@ mod tests {
             .to_string()
             .contains("More than one"));
     }
+
+    #[test]
+    fn test_validate_table_name() {
+        assert!(validate_table_name("my_table").is_ok());
+        assert!(validate_table_name("my_table_1").is_ok());
+        assert!(validate_table_name("123mytable").is_ok());
+        assert!(validate_table_name("_12345table").is_ok());
+        assert!(validate_table_name("table.12345").is_ok());
+        assert!(validate_table_name("table.._dot_..12345").is_ok());
+
+        assert!(validate_table_name("").is_err());
+        assert!(validate_table_name("my_table!").is_err());
+        assert!(validate_table_name("my/table").is_err());
+        assert!(validate_table_name("my@table").is_err());
+        assert!(validate_table_name("name with space").is_err());
+    }
 }