cleanup

.bumpversion.cfg

@@ -1,12 +0,0 @@
-[bumpversion]
-current_version = 0.1.14
-commit = True
-message = Bump version: {current_version} → {new_version}
-tag = True
-tag_name = v{new_version}
-
-[bumpversion:file:node/package.json]
-
-[bumpversion:file:rust/ffi/node/Cargo.toml]
-
-[bumpversion:file:rust/vectordb/Cargo.toml]

.github/workflows/cargo-publish.yml

@@ -1,29 +0,0 @@
-name: Cargo Publish
-
-on:
-  release:
-    types: [ published ]
-
-env:
-  # This env var is used by Swatinem/rust-cache@v2 for the cache
-  # key, so we set it to make sure it is always consistent.
-  CARGO_TERM_COLOR: always
-
-jobs:
-  build:
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    steps:
-      - uses: actions/checkout@v3
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Publish the package
-        run: |
-          cargo publish -p vectordb --all-features --token ${{ secrets.CARGO_REGISTRY_TOKEN }}

.github/workflows/docs.yml

@@ -39,28 +39,6 @@ jobs:
         run: |
           python -m pip install -e .
           python -m pip install -r ../docs/requirements.txt
-      - name: Set up node
-        uses: actions/setup-node@v3
-        with:
-          node-version: ${{ matrix.node-version }}
-          cache: 'npm'
-          cache-dependency-path: node/package-lock.json
-      - uses: Swatinem/rust-cache@v2
-      - name: Install node dependencies
-        working-directory: node
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Build node
-        working-directory: node
-        run: |
-          npm ci
-          npm run build
-          npm run tsc
-      - name: Create markdown files
-        working-directory: node
-        run: |
-          npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
       - name: Build docs
         run: |
           PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
@@ -72,4 +50,4 @@ jobs:
           path: "docs/site"
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v1
+        uses: actions/deploy-pages@v1

.github/workflows/docs_test.yml

@@ -1,93 +0,0 @@
-name: Documentation Code Testing
-
-on:
-  push:
-    branches:
-      - main
-    paths:
-      - docs/**
-      - .github/workflows/docs_test.yml
-  pull_request:
-    paths:
-      - docs/**
-      - .github/workflows/docs_test.yml
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-env:
-  # Disable full debug symbol generation to speed up CI build and keep memory down
-  # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1"
-  RUST_BACKTRACE: "1"
-
-jobs:
-  test-python:
-    name: Test doc python code
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        python-minor-version: [ "11" ]
-        os: ["ubuntu-22.04"]
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.${{ matrix.python-minor-version }}
-        cache: "pip"
-        cache-dependency-path: "docs/test/requirements.txt"
-    - name: Build Python
-      working-directory: docs/test
-      run:
-        python -m pip install -r requirements.txt
-    - name: Create test files
-      run: |
-        cd docs/test
-        python md_testing.py
-    - name: Test
-      run: |
-        cd docs/test/python
-        for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
-  test-node:
-    name: Test doc nodejs code
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        node-version: [ "18" ]
-        os: ["ubuntu-22.04"]
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Set up Node
-      uses: actions/setup-node@v3
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: Install dependecies needed for ubuntu
-      if: ${{ matrix.os == 'ubuntu-22.04' }}
-      run: |
-        sudo apt install -y protobuf-compiler libssl-dev
-    - name: Install node dependencies
-      run: |
-        cd docs/test
-        npm install 
-    - name: Rust cache
-      uses: swatinem/rust-cache@v2
-    - name: Install LanceDB
-      run: |
-        cd docs/test/node_modules/vectordb 
-        npm ci
-        npm run build-release
-        npm run tsc
-    - name: Create test files
-      run: |
-        cd docs/test
-        node md_testing.js
-    - name: Test
-      run: |
-        cd docs/test/node
-        for d in *; do cd "$d"; echo "$d".js; node "$d".js; cd ..; done

.github/workflows/make-release-commit.yml

@@ -1,62 +0,0 @@
-name: Create release commit
-
-on:
-  workflow_dispatch:
-    inputs:
-      dry_run:
-        description: 'Dry run (create the local commit/tags but do not push it)'
-        required: true
-        default: "false"
-        type: choice
-        options:
-          - "true"
-          - "false"
-      part:
-        description: 'What kind of release is this?'
-        required: true
-        default: 'patch'
-        type: choice
-        options:
-          - patch
-          - minor
-          - major
-
-jobs:
-  bump-version:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out main
-      uses: actions/checkout@v3
-      with:
-        ref: main
-        persist-credentials: false
-        fetch-depth: 0
-        lfs: true
-    - name: Set git configs for bumpversion
-      shell: bash
-      run: |
-        git config user.name 'Lance Release'
-        git config user.email 'lance-dev@lancedb.com'
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-    - name: Bump version, create tag and commit
-      run: |
-        pip install bump2version
-        bumpversion --verbose ${{ inputs.part }}
-    - name: Update package-lock.json file
-      run: |
-        npm install
-        git add package-lock.json
-        # Add this change to the commit created by bumpversion
-        git commit --amend --no-edit
-      working-directory: node
-    - name: Push new version and tag
-      if: ${{ inputs.dry_run }} == "false"
-      uses: ad-m/github-push-action@master
-      with:
-        github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-        branch: main
-        tags: true
-

.github/workflows/node.yml

@@ -71,8 +71,6 @@ jobs:
         npm run build
         npm run pack-build
         npm install --no-save ./dist/vectordb-*.tgz
-        # Remove index.node to test with dependency installed
-        rm index.node
     - name: Test
       run: npm run test
   macos:
@@ -102,8 +100,6 @@ jobs:
         npm run build
         npm run pack-build
         npm install --no-save ./dist/vectordb-*.tgz
-        # Remove index.node to test with dependency installed
-        rm index.node
     - name: Test
       run: |
         npm run test

.github/workflows/npm-publish.yml

@@ -1,172 +0,0 @@
-name: NPM Publish
-
-on:
-  release:
-    types: [ published ]
-
-jobs:
-  node:
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    defaults:
-      run:
-        shell: bash
-        working-directory: node
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 20
-          cache: 'npm'
-          cache-dependency-path: node/package-lock.json
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Build
-        run: |
-          npm ci
-          npm run tsc
-          npm pack
-      - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: node-package
-          path: |
-            node/vectordb-*.tgz
-
-  node-macos:
-    runs-on: macos-12
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64-apple-darwin, aarch64-apple-darwin]
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-    - name: Install system dependencies
-      run: brew install protobuf
-    - name: Install npm dependencies
-      run: |
-        cd node
-        npm ci
-    - name: Install rustup target
-      if: ${{ matrix.target == 'aarch64-apple-darwin' }}
-      run: rustup target add aarch64-apple-darwin
-    - name: Build MacOS native node modules
-      run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
-    - name: Upload Darwin Artifacts
-      uses: actions/upload-artifact@v3
-      with:
-        name: darwin-native
-        path: |
-          node/dist/vectordb-darwin*.tgz
-
-  node-linux:
-    name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }})
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        libc:
-          - gnu
-          # TODO: re-enable musl once we have refactored to pre-built containers
-          # Right now we have to build node from source which is too expensive.
-          # - musl
-        arch:
-          - x86_64
-          # Building on aarch64 is too slow for now
-          # - aarch64
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-    - name: Change owner to root (for npm)
-      # The docker container is run as root, so we need the files to be owned by root
-      # Otherwise npm is a nightmare: https://github.com/npm/cli/issues/3773
-      run: sudo chown -R root:root .
-    - name: Set up QEMU
-      if: ${{ matrix.arch == 'aarch64' }}
-      uses: docker/setup-qemu-action@v2
-      with:
-        platforms: arm64
-    - name: Build Linux GNU native node modules
-      if: ${{ matrix.libc == 'gnu' }}
-      run: |
-        docker run \
-          -v $(pwd):/io -w /io \
-          rust:1.70-bookworm \
-          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu
-    - name: Build musl Linux native node modules
-      if: ${{ matrix.libc == 'musl' }}
-      run: |
-        docker run --platform linux/arm64/v8 \
-          -v $(pwd):/io -w /io \
-          quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
-          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
-    - name: Upload Linux Artifacts
-      uses: actions/upload-artifact@v3
-      with:
-        name: linux-native
-        path: |
-          node/dist/vectordb-linux*.tgz
-
-  node-windows:
-    runs-on: windows-2022
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64-pc-windows-msvc]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Install Protoc v21.12
-        working-directory: C:\
-        run: |
-          New-Item -Path 'C:\protoc' -ItemType Directory
-          Set-Location C:\protoc
-          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-          7z x protoc.zip
-          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-        shell: powershell
-      - name: Install npm dependencies
-        run: |
-          cd node
-          npm ci
-      - name: Build Windows native node modules
-        run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }}
-      - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: windows-native
-          path: |
-            node/dist/vectordb-win32*.tgz
-
-  release:
-    needs: [node, node-macos, node-linux]
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    steps:
-      - uses: actions/download-artifact@v3
-      - name: Display structure of downloaded files
-        run: ls -R
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 20
-          registry-url: 'https://registry.npmjs.org'
-      - name: Publish to NPM
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
-        run: |
-          mv */*.tgz .
-          for filename in *.tgz; do
-            npm publish $filename
-          done

.github/workflows/pypi-publish.yml

@@ -1,31 +0,0 @@
-name: PyPI Publish
-
-on:
-  release:
-    types: [ published ]
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the python-make-release action
-    if: startsWith(github.ref, 'refs/tags/python-v')
-    defaults:
-      run:
-        shell: bash
-        working-directory: python
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.8"
-      - name: Build distribution
-        run: |
-          ls -la
-          pip install wheel setuptools --upgrade
-          python setup.py sdist bdist_wheel
-      - name: Publish
-        uses: pypa/gh-action-pypi-publish@v1.8.5
-        with:
-          password: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
-          packages-dir: python/dist

.github/workflows/python-make-release-commit.yml

@@ -1,56 +0,0 @@
-name: Python - Create release commit
-
-on:
-  workflow_dispatch:
-    inputs:
-      dry_run:
-        description: 'Dry run (create the local commit/tags but do not push it)'
-        required: true
-        default: "false"
-        type: choice
-        options:
-          - "true"
-          - "false"
-      part:
-        description: 'What kind of release is this?'
-        required: true
-        default: 'patch'
-        type: choice
-        options:
-          - patch
-          - minor
-          - major
-
-jobs:
-  bump-version:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out main
-      uses: actions/checkout@v3
-      with:
-        ref: main
-        persist-credentials: false
-        fetch-depth: 0
-        lfs: true
-    - name: Set git configs for bumpversion
-      shell: bash
-      run: |
-        git config user.name 'Lance Release'
-        git config user.email 'lance-dev@lancedb.com'
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-    - name: Bump version, create tag and commit
-      working-directory: python
-      run: |
-        pip install bump2version
-        bumpversion --verbose ${{ inputs.part }}
-    - name: Push new version and tag
-      if: ${{ inputs.dry_run }} == "false"
-      uses: ad-m/github-push-action@master
-      with:
-        github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-        branch: main
-        tags: true
-

.github/workflows/python.yml

@@ -31,16 +31,9 @@ jobs:
     - name: Install lancedb
       run: |
         pip install -e .
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock black isort
-    - name: Black
-      run: black --check --diff --no-color --quiet .
-    - name: isort
-      run: isort --check --diff --quiet .
+        pip install pytest
     - name: Run tests
       run: pytest -x -v --durations=30 tests
-    - name: doctest
-      run: pytest --doctest-modules lancedb
   mac:
     timeout-minutes: 30
     runs-on: "macos-12"
@@ -56,13 +49,10 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.11"
+        python-version: "3.10"
     - name: Install lancedb
       run: |
         pip install -e .
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock black
-    - name: Black
-      run: black --check --diff --no-color --quiet .
+        pip install pytest
     - name: Run tests
-      run: pytest -x -v --durations=30 tests
+      run: pytest -x -v --durations=30 tests

.github/workflows/release.yml

@@ -0,0 +1,159 @@
+name: Prepare Release
+
+# Based on https://github.com/dherman/neon-prebuild-example/blob/eaa4d33d682e5eb7abbc3da7aed153a1b1acb1b3/.github/workflows/publish.yml
+
+on:
+  push:
+    tags:
+      - v*
+
+jobs:
+  draft-release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          prerelease: true # hardcoded on for now
+          generate_release_notes: true
+  
+  rust:
+    runs-on: ubuntu-latest
+    needs: draft-release
+    defaults:
+      run:
+        shell: bash
+        working-directory: rust/vectordb
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Package Rust
+        run: cargo package --all-features
+      - uses: softprops/action-gh-release@v1
+        with:
+          files: |
+            rust/target/vectordb-*.crate
+    
+  python:
+    runs-on: ubuntu-latest
+    needs: draft-release
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.10
+      - name: Build wheel
+        run: python setup.py sdist bdist_wheel
+      - uses: softprops/action-gh-release@v1
+        with:
+          files: |
+            python/dist/lancedb-*.tar.gz 
+            python/dist/lancedb-*.whl
+
+  node:
+    runs-on: ubuntu-latest
+    needs: draft-release
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          cache: 'npm'
+          cache-dependency-path: node/package-lock.json
+      - uses: Swatinem/rust-cache@v2
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Build
+        run: |
+          npm ci
+          npm run tsc
+          npm pack
+      - uses: softprops/action-gh-release@v1
+        with:
+          files: node/vectordb-*.tgz
+
+  node-macos:
+    runs-on: macos-12
+    needs: draft-release
+    strategy:
+      matrix:
+        target: [x86_64-apple-darwin, aarch64-apple-darwin]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Install system dependencies
+      run: brew install protobuf
+    - name: Build MacOS native node modules
+      run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
+    - uses: softprops/action-gh-release@v1
+      with:
+        files: node/dist/vectordb-darwin*.tgz
+
+  node-linux:
+    runs-on: ubuntu-latest
+    needs: draft-release
+    strategy:
+      matrix:
+        target:
+          - x86_64-unknown-linux-gnu:centos
+          - aarch64-unknown-linux-gnu:centos
+          - aarch64-unknown-linux-musl
+          - x86_64-unknown-linux-musl
+    steps:
+    - name: Install system dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y protobuf-compiler libssl-dev
+    - name: Build Linux native node modules
+      run: bash ci/build_linux_artifacts.sh ${{ matrix.target }}
+    - uses: softprops/action-gh-release@v1
+      with:
+        files: node/dist/vectordb-linux*.tgz
+
+  release:
+    needs: [python, node, node-macos, node-linux]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v3
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: |
+          python -m twine upload --non-interactive \
+            --skip-existing \
+            --repository testpypi python/dist/*
+      - name: Publish to NPM
+        run: |
+          for filename in node/dist/*.tgz; do
+            npm publish --dry-run $filename
+          done
+      - name: Publish to crates.io
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+        run: |
+          cargo publish --dry-run --no-verify rust/target/vectordb-*.crate
+      # - uses: softprops/action-gh-release@v1
+      #   with:
+      #     draft: false

.github/workflows/rust.yml

@@ -1,89 +0,0 @@
-name: Rust
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths:
-      - Cargo.toml
-      - rust/**
-      - .github/workflows/rust.yml
-
-env:
-  # This env var is used by Swatinem/rust-cache@v2 for the cache
-  # key, so we set it to make sure it is always consistent.
-  CARGO_TERM_COLOR: always
-  # Disable full debug symbol generation to speed up CI build and keep memory down
-  # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1"
-  RUST_BACKTRACE: "1"
-
-jobs:
-  linux:
-    timeout-minutes: 30
-    runs-on: ubuntu-22.04
-    defaults:
-      run:
-        shell: bash
-        working-directory: rust
-    steps:
-    - uses: actions/checkout@v3
-      with:
-          fetch-depth: 0
-          lfs: true
-    - uses: Swatinem/rust-cache@v2
-      with:
-          workspaces: rust
-    - name: Install dependencies
-      run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-    - name: Build
-      run: cargo build --all-features
-    - name: Run tests
-      run: cargo test --all-features
-  macos:
-    runs-on: macos-12
-    timeout-minutes: 30
-    defaults:
-      run:
-        shell: bash
-        working-directory: rust
-    steps:
-      - uses: actions/checkout@v3
-        with:
-            fetch-depth: 0
-            lfs: true
-      - name: CPU features
-        run: sysctl -a | grep cpu
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
-      - name: Install dependencies
-        run: brew install protobuf
-      - name: Build
-        run: cargo build --all-features
-      - name: Run tests
-        run: cargo test --all-features
-  windows:
-    runs-on: windows-2022
-    steps:
-      - uses: actions/checkout@v3
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
-      - name: Install Protoc v21.12
-        working-directory: C:\
-        run: |
-          New-Item -Path 'C:\protoc' -ItemType Directory
-          Set-Location C:\protoc
-          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-          7z x protoc.zip
-          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-        shell: powershell
-      - name: Run tests
-        run: |
-          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
-          cargo build
-          cargo test

.gitignore

@@ -3,7 +3,6 @@
 *.egg-info
 **/__pycache__
 .DS_Store
-venv
 
 .vscode
 
@@ -18,7 +17,7 @@ site
 python/build
 python/dist
 
-**/.ipynb_checkpoints
+notebooks/.ipynb_checkpoints
 
 **/.hypothesis
 
@@ -33,4 +32,3 @@ node/examples/**/dist
 ## Rust
 target
 
-Cargo.lock

.pre-commit-config.yaml

@@ -8,14 +8,4 @@ repos:
 -   repo: https://github.com/psf/black
     rev: 22.12.0
     hooks:
-    -   id: black
--   repo: https://github.com/astral-sh/ruff-pre-commit
-    # Ruff version.
-    rev: v0.0.277
-    hooks:
-    - id: ruff
--   repo: https://github.com/pycqa/isort
-    rev: 5.12.0
-    hooks:
-    - id: isort
-      name: isort (python)
+    -   id: black

Cargo.toml

@@ -4,13 +4,3 @@ members = [
     "rust/ffi/node"
 ]
 resolver = "2"
-
-[workspace.dependencies]
-lance = "=0.5.8"
-arrow-array = "42.0"
-arrow-data = "42.0"
-arrow-schema = "42.0"
-arrow-ipc = "42.0"
-half = { "version" = "2.2.1", default-features = false }
-object_store = "0.6.1"
-

Cross.toml

@@ -0,0 +1,14 @@
+# These make sure our builds are compatible with old glibc versions.
+[target.x86_64-unknown-linux-gnu]
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH protobuf-compiler",
+]
+image = "ghcr.io/cross-rs/x86_64-unknown-linux-gnu:main-centos"
+
+[target.aarch64-unknown-linux-gnu]
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH protobuf-compiler",
+]
+image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main-centos"

README.md

@@ -10,10 +10,6 @@
 <a href="https://discord.gg/zMM32dvNtd">Discord</a> •
 <a href="https://twitter.com/lancedb">Twitter</a>
 
-</p>
-
-<img max-width="750px" alt="LanceDB Multimodal Search" src="https://github.com/lancedb/lancedb/assets/917119/09c5afc5-7816-4687-bae4-f2ca194426ec">
-
 </p>
 </div>
 
@@ -27,15 +23,13 @@ The key features of LanceDB include:
 
 * Store, query and filter vectors, metadata and multi-modal data (text, images, videos, point clouds, and more).
 
-* Support for vector similarity search, full-text search and SQL.
-
 * Native Python and Javascript/Typescript support.
 
 * Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure.
 
 * Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lanecdb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
 
-LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
+LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/eto-ai/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
 
 ## Quick Start
 
@@ -65,7 +59,7 @@ pip install lancedb
 ```python
 import lancedb
 
-uri = "data/sample-lancedb"
+uri = "/tmp/lancedb"
 db = lancedb.connect(uri)
 table = db.create_table("my_table",
                          data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
@@ -75,4 +69,4 @@ result = table.search([100, 100]).limit(2).to_df()
 
 ## Blogs, Tutorials & Videos
 * 📈 <a href="https://blog.eto.ai/benchmarking-random-access-in-lance-ed690757a826">2000x better performance with Lance over Parquet</a>
-* 🤖 <a href="https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb">Build a question and answer bot with LanceDB</a>
+* 🤖 <a href="https://github.com/lancedb/lancedb/blob/main/notebooks/youtube_transcript_search.ipynb">Build a question and answer bot with LanceDB</a>

ci/build_linux_artifacts.sh

@@ -1,4 +1,3 @@
-#!/bin/bash
 # Builds the Linux artifacts (node binaries).
 # Usage: ./build_linux_artifacts.sh [target]
 # Targets supported: 
@@ -7,66 +6,35 @@
 # - aarch64-unknown-linux-musl
 # - x86_64-unknown-linux-musl
 
-# TODO: refactor this into a Docker container we can pull
+# On MacOS, need to run in a linux container:
+# docker run -v $(pwd):/io -w /io 
+
+# Must run rustup toolchain install stable-x86_64-unknown-linux-gnu --force-non-host
 
 set -e
 
-setup_dependencies() {
-    echo "Installing system dependencies..."
-    if [[ $1 == *musl ]]; then
-        # musllinux
-        apk add openssl-dev
-    else
-        # rust / debian
-        apt update
-        apt install -y libssl-dev protobuf-compiler
-    fi
-}
-
-install_node() {
-    echo "Installing node..."
-    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
-    source "$HOME"/.bashrc
-
-    if [[ $1 == *musl ]]; then
-        # This node version is 15, we need 16 or higher:
-        # apk add nodejs-current npm 
-        # So instead we install from source (nvm doesn't provide binaries for musl):
-        nvm install -s --no-progress 17
-    else
-        nvm install --no-progress 17 # latest that supports glibc 2.17
-    fi
-}
-
-build_node_binary() {
-    echo "Building node library for $1..."
+build_node_binaries() {
     pushd node
-
-    npm ci
     
-    if [[ $1 == *musl ]]; then
-        # This is needed for cargo to allow build cdylibs with musl
-        export RUSTFLAGS="-C target-feature=-crt-static"
-    fi
-
-    # Cargo can run out of memory while pulling dependencies, especially when running
-    # in QEMU. This is a workaround for that.
-    export CARGO_NET_GIT_FETCH_WITH_CLI=true
-
-    # We don't pass in target, since the native target here already matches
-    # We need to pass OPENSSL_LIB_DIR and OPENSSL_INCLUDE_DIR for static build to work https://github.com/sfackler/rust-openssl/issues/877
-    OPENSSL_STATIC=1 OPENSSL_LIB_DIR=/usr/lib/x86_64-linux-gnu OPENSSL_INCLUDE_DIR=/usr/include/openssl/ npm run build-release
-    npm run pack-build
-
+    for target in $1
+    do
+        echo "Building node library for $target"
+        # cross doesn't yet pass this down to Docker, so we do it ourselves.
+        if [[ $target == x86_64* ]]; then
+            export CROSS_CONTAINER_OPTS="--platform linux/amd64"
+        else
+            export CROSS_CONTAINER_OPTS="--platform linux/arm64/v8"
+        fi
+        npm run cross-release -- --target $target
+        npm run pack-build -- --target $target
+    done
     popd
 }
 
-TARGET=${1:-x86_64-unknown-linux-gnu}
-# Others:
-# aarch64-unknown-linux-gnu
-# x86_64-unknown-linux-musl
-# aarch64-unknown-linux-musl
-
-setup_dependencies $TARGET
-install_node $TARGET
-build_node_binary $TARGET
+if [ -n "$1" ]; then
+    targets=$1
+else
+    # targets="x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu aarch64-unknown-linux-musl x86_64-unknown-linux-musl"
+    targets="aarch64-unknown-linux-gnu"
+fi
+build_node_binaries $targets

ci/build_macos_artifacts.sh

@@ -1,21 +1,16 @@
 # Builds the macOS artifacts (node binaries).
-# Usage: ./ci/build_macos_artifacts.sh [target]
+# Usage: ./build_macos_artifacts.sh [target]
 # Targets supported: x86_64-apple-darwin aarch64-apple-darwin
 
-prebuild_rust() {
-    # Building here for the sake of easier debugging.
-    pushd rust/ffi/node
-    echo "Building rust library for $1"
-    export RUST_BACKTRACE=1
-    cargo build --release --target $1
-    popd
-}
-
 build_node_binaries() {
     pushd node
-    echo "Building node library for $1"
-    npm run build-release -- --target $1
-    npm run pack-build -- --target $1
+    
+    for target in $1
+    do
+        echo "Building node library for $target"
+        npm run build-release -- --target $target
+        npm run pack-build -- --target $target
+    done
     popd
 }
 
@@ -24,10 +19,4 @@ if [ -n "$1" ]; then
 else
     targets="x86_64-apple-darwin aarch64-apple-darwin"
 fi
-
-echo "Building artifacts for targets: $targets"
-for target in $targets
-    do
-    prebuild_rust $target
-    build_node_binaries $target
-done
+build_node_binaries $targets

ci/build_windows_artifacts.ps1

@@ -1,41 +0,0 @@
-# Builds the Windows artifacts (node binaries).
-# Usage:  .\ci\build_windows_artifacts.ps1 [target]
-# Targets supported:
-# - x86_64-pc-windows-msvc
-# - i686-pc-windows-msvc
-
-function Prebuild-Rust {
-    param (
-        [string]$target
-    )
-
-    # Building here for the sake of easier debugging.
-    Push-Location -Path "rust/ffi/node"
-    Write-Host "Building rust library for $target"
-    $env:RUST_BACKTRACE=1
-    cargo build --release --target $target
-    Pop-Location
-}
-
-function Build-NodeBinaries {
-    param (
-        [string]$target
-    )
-
-    Push-Location -Path "node"
-    Write-Host "Building node library for $target"
-    npm run build-release -- --target $target
-    npm run pack-build -- --target $target
-    Pop-Location
-}
-
-$targets = $args[0]
-if (-not $targets) {
-    $targets = "x86_64-pc-windows-msvc"
-}
-
-Write-Host "Building artifacts for targets: $targets"
-foreach ($target in $targets) {
-    Prebuild-Rust $target
-    Build-NodeBinaries $target
-}

ci/ubuntu_build.dockerfile

@@ -0,0 +1,31 @@
+# On MacOS, need to run in a linux container:
+# cat ci/ubuntu_build.dockerfile | docker build -t lancedb-node-build -
+# docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd):/io -w /io lancedb-node-build bash ci/build_linux_artifacts.sh
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/Moscow
+
+RUN apt update && apt install -y protobuf-compiler libssl-dev build-essential curl \
+    software-properties-common npm docker.io
+
+# Install rust
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install cross
+# https://github.com/cross-rs/cross/issues/1257#issuecomment-1544553706
+RUN cargo install cross --git https://github.com/cross-rs/cross
+
+# Install additional build targets
+RUN rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu aarch64-unknown-linux-musl x86_64-unknown-linux-musl
+
+# Install node
+RUN npm install npm@latest -g && \
+    npm install n -g && \
+    n latest
+
+# set CROSS_CONTAINER_IN_CONTAINER to inform `cross` that it is executed from within a container
+ENV CROSS_CONTAINER_IN_CONTAINER=true
+ENV CROSS_CONTAINER_ENGINE_NO_BUILDKIT=1

docs/mkdocs.yml

@@ -1,84 +1,32 @@
-site_name: LanceDB Docs
-repo_url: https://github.com/lancedb/lancedb
-repo_name: lancedb/lancedb
+site_name: LanceDB Documentation
 docs_dir: src
 
 theme:
   name: "material"
-  logo: assets/logo.png
-  favicon: assets/logo.png
   features:
     - content.code.copy
-    - content.tabs.link
-  icon:
-    repo: fontawesome/brands/github
-  custom_dir: overrides
 
 plugins:
 - search
-- autorefs
 - mkdocstrings:
     handlers:
       python:
         paths: [../python]
-        selection:
-          docstring_style: numpy
-        rendering:
-          heading_level: 4
-          show_source: false
-          show_symbol_type_in_heading: true
-          show_signature_annotations: true
-          show_root_heading: true
-          members_order: source
-        import:
-          # for cross references
-          - https://arrow.apache.org/docs/objects.inv
-          - https://pandas.pydata.org/docs/objects.inv
 - mkdocs-jupyter
 
+nav:
+- Home: index.md
+- Basics: basic.md
+- Embeddings: embedding.md
+- Indexing: ann_indexes.md
+- Integrations: integrations.md
+- Python API: python.md
+
 markdown_extensions:
-- admonition
-- footnotes
-- pymdownx.superfences
-- pymdownx.details
 - pymdownx.highlight:
     anchor_linenums: true
     line_spans: __span
     pygments_lang_class: true
 - pymdownx.inlinehilite
 - pymdownx.snippets
-- pymdownx.superfences
-- pymdownx.tabbed:
-    alternate_style: true
-- md_in_html
-
-nav:
-- Home: index.md
-- Basics: basic.md
-- Embeddings: embedding.md
-- Python full-text search: fts.md
-- Python integrations:
-  - Pandas and PyArrow: python/arrow.md
-  - DuckDB: python/duckdb.md
-  - LangChain 🦜️🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
-  - LlamaIndex 🦙: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
-  - Pydantic: python/pydantic.md
-- Python examples:
-  - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
-  - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
-  - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
-  - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
-  - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
-- Javascript examples:
-  - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
-  - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
-- References:
-  - Vector Search: search.md
-  - SQL filters: sql.md
-  - Indexing: ann_indexes.md
-- API references:
-  - Python API: python/python.md
-  - Javascript API: javascript/modules.md
-
-extra_css:
-  - styles/global.css
+- pymdownx.superfences

docs/overrides/partials/header.html

@@ -1,176 +0,0 @@
-<!--
-  Copyright (c) 2016-2023 Martin Donath <martin.donath@squidfunk.com>
-
-  Permission is hereby granted, free of charge, to any person obtaining a copy
-  of this software and associated documentation files (the "Software"), to
-  deal in the Software without restriction, including without limitation the
-  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-  sell copies of the Software, and to permit persons to whom the Software is
-  furnished to do so, subject to the following conditions:
-
-  The above copyright notice and this permission notice shall be included in
-  all copies or substantial portions of the Software.
-
-  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-  FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
-  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-  IN THE SOFTWARE.
--->
-
-{% set class = "md-header" %}
-{% if "navigation.tabs.sticky" in features %}
-  {% set class = class ~ " md-header--shadow md-header--lifted" %}
-{% elif "navigation.tabs" not in features %}
-  {% set class = class ~ " md-header--shadow" %}
-{% endif %}
-
-<!-- Header -->
-<header class="{{ class }}" data-md-component="header">
-  <nav
-    class="md-header__inner md-grid"
-    aria-label="{{ lang.t('header') }}"
-  >
-
-    <!-- Link to home -->
-    <a
-      href="{{ config.extra.homepage | d(nav.homepage.url, true) | url }}"
-      title="{{ config.site_name | e }}"
-      class="md-header__button md-logo"
-      aria-label="{{ config.site_name }}"
-      data-md-component="logo"
-    >
-      {% include "partials/logo.html" %}
-    </a>
-
-    <!-- Button to open drawer -->
-    <label class="md-header__button md-icon" for="__drawer">
-      {% include ".icons/material/menu" ~ ".svg" %}
-    </label>
-
-    <!-- Header title -->
-    <div class="md-header__title" style="width: auto !important;" data-md-component="header-title">
-      <div class="md-header__ellipsis">
-        <div class="md-header__topic">
-          <span class="md-ellipsis">
-            {{ config.site_name }}
-          </span>
-        </div>
-        <div class="md-header__topic" data-md-component="header-topic">
-          <span class="md-ellipsis">
-            {% if page.meta and page.meta.title %}
-              {{ page.meta.title }}
-            {% else %}
-              {{ page.title }}
-            {% endif %}
-          </span>
-        </div>
-      </div>
-    </div>
-
-    <!-- Color palette -->
-    {% if config.theme.palette %}
-      {% if not config.theme.palette is mapping %}
-        <form class="md-header__option" data-md-component="palette">
-          {% for option in config.theme.palette %}
-            {% set scheme  = option.scheme  | d("default", true) %}
-            {% set primary = option.primary | d("indigo", true) %}
-            {% set accent  = option.accent  | d("indigo", true) %}
-            <input
-              class="md-option"
-              data-md-color-media="{{ option.media }}"
-              data-md-color-scheme="{{ scheme | replace(' ', '-') }}"
-              data-md-color-primary="{{ primary | replace(' ', '-') }}"
-              data-md-color-accent="{{ accent | replace(' ', '-') }}"
-              {% if option.toggle %}
-                aria-label="{{ option.toggle.name }}"
-              {% else  %}
-                aria-hidden="true"
-              {% endif %}
-              type="radio"
-              name="__palette"
-              id="__palette_{{ loop.index }}"
-            />
-            {% if option.toggle %}
-              <label
-                class="md-header__button md-icon"
-                title="{{ option.toggle.name }}"
-                for="__palette_{{ loop.index0 or loop.length }}"
-                hidden
-              >
-                {% include ".icons/" ~ option.toggle.icon ~ ".svg" %}
-              </label>
-            {% endif %}
-          {% endfor %}
-        </form>
-      {% endif %}
-    {% endif %}
-
-    <!-- Site language selector -->
-    {% if config.extra.alternate %}
-      <div class="md-header__option">
-        <div class="md-select">
-          {% set icon = config.theme.icon.alternate or "material/translate" %}
-          <button
-            class="md-header__button md-icon"
-            aria-label="{{ lang.t('select.language') }}"
-          >
-            {% include ".icons/" ~ icon ~ ".svg" %}
-          </button>
-          <div class="md-select__inner">
-            <ul class="md-select__list">
-              {% for alt in config.extra.alternate %}
-                <li class="md-select__item">
-                  <a
-                    href="{{ alt.link | url }}"
-                    hreflang="{{ alt.lang }}"
-                    class="md-select__link"
-                  >
-                    {{ alt.name }}
-                  </a>
-                </li>
-              {% endfor %}
-            </ul>
-          </div>
-        </div>
-      </div>
-    {% endif %}
-
-    <!-- Button to open search modal -->
-    {% if "material/search" in config.plugins %}
-      <label class="md-header__button md-icon" for="__search">
-        {% include ".icons/material/magnify.svg" %}
-      </label>
-
-      <!-- Search interface -->
-      {% include "partials/search.html" %}
-    {% endif %}
-
-    <div style="margin-left: 10px; margin-right: 5px;">
-        <a href="https://discord.com/invite/zMM32dvNtd" target="_blank" rel="noopener noreferrer">
-            <svg fill="#FFFFFF" xmlns="http://www.w3.org/2000/svg"  viewBox="0 0 50 50" width="25px" height="25px"><path d="M 41.625 10.769531 C 37.644531 7.566406 31.347656 7.023438 31.078125 7.003906 C 30.660156 6.96875 30.261719 7.203125 30.089844 7.589844 C 30.074219 7.613281 29.9375 7.929688 29.785156 8.421875 C 32.417969 8.867188 35.652344 9.761719 38.578125 11.578125 C 39.046875 11.867188 39.191406 12.484375 38.902344 12.953125 C 38.710938 13.261719 38.386719 13.429688 38.050781 13.429688 C 37.871094 13.429688 37.6875 13.378906 37.523438 13.277344 C 32.492188 10.15625 26.210938 10 25 10 C 23.789063 10 17.503906 10.15625 12.476563 13.277344 C 12.007813 13.570313 11.390625 13.425781 11.101563 12.957031 C 10.808594 12.484375 10.953125 11.871094 11.421875 11.578125 C 14.347656 9.765625 17.582031 8.867188 20.214844 8.425781 C 20.0625 7.929688 19.925781 7.617188 19.914063 7.589844 C 19.738281 7.203125 19.34375 6.960938 18.921875 7.003906 C 18.652344 7.023438 12.355469 7.566406 8.320313 10.8125 C 6.214844 12.761719 2 24.152344 2 34 C 2 34.175781 2.046875 34.34375 2.132813 34.496094 C 5.039063 39.605469 12.972656 40.941406 14.78125 41 C 14.789063 41 14.800781 41 14.8125 41 C 15.132813 41 15.433594 40.847656 15.621094 40.589844 L 17.449219 38.074219 C 12.515625 36.800781 9.996094 34.636719 9.851563 34.507813 C 9.4375 34.144531 9.398438 33.511719 9.765625 33.097656 C 10.128906 32.683594 10.761719 32.644531 11.175781 33.007813 C 11.234375 33.0625 15.875 37 25 37 C 34.140625 37 38.78125 33.046875 38.828125 33.007813 C 39.242188 32.648438 39.871094 32.683594 40.238281 33.101563 C 40.601563 33.515625 40.5625 34.144531 40.148438 34.507813 C 40.003906 34.636719 37.484375 36.800781 32.550781 38.074219 L 34.378906 40.589844 C 34.566406 40.847656 34.867188 41 35.1875 41 C 35.199219 41 35.210938 41 35.21875 41 C 37.027344 40.941406 44.960938 39.605469 47.867188 34.496094 C 47.953125 34.34375 48 34.175781 48 34 C 48 24.152344 43.785156 12.761719 41.625 10.769531 Z M 18.5 30 C 16.566406 30 15 28.210938 15 26 C 15 23.789063 16.566406 22 18.5 22 C 20.433594 22 22 23.789063 22 26 C 22 28.210938 20.433594 30 18.5 30 Z M 31.5 30 C 29.566406 30 28 28.210938 28 26 C 28 23.789063 29.566406 22 31.5 22 C 33.433594 22 35 23.789063 35 26 C 35 28.210938 33.433594 30 31.5 30 Z"/></svg>
-        </a>
-    </div>
-    <div style="margin-left: 5px; margin-right: 5px;">
-        <a href="https://twitter.com/lancedb" target="_blank" rel="noopener noreferrer">
-            <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0,0,256,256" width="25px" height="25px" fill-rule="nonzero"><g fill-opacity="0" fill="#ffffff" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><path d="M0,256v-256h256v256z" id="bgRectangle"></path></g><g fill="#ffffff" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(4,4)"><path d="M57,17.114c-1.32,1.973 -2.991,3.707 -4.916,5.097c0.018,0.423 0.028,0.847 0.028,1.274c0,13.013 -9.902,28.018 -28.016,28.018c-5.562,0 -12.81,-1.948 -15.095,-4.423c0.772,0.092 1.556,0.138 2.35,0.138c4.615,0 8.861,-1.575 12.23,-4.216c-4.309,-0.079 -7.946,-2.928 -9.199,-6.84c1.96,0.308 4.447,-0.17 4.447,-0.17c0,0 -7.7,-1.322 -7.899,-9.779c2.226,1.291 4.46,1.231 4.46,1.231c0,0 -4.441,-2.734 -4.379,-8.195c0.037,-3.221 1.331,-4.953 1.331,-4.953c8.414,10.361 20.298,10.29 20.298,10.29c0,0 -0.255,-1.471 -0.255,-2.243c0,-5.437 4.408,-9.847 9.847,-9.847c2.832,0 5.391,1.196 7.187,3.111c2.245,-0.443 4.353,-1.263 6.255,-2.391c-0.859,3.44 -4.329,5.448 -4.329,5.448c0,0 2.969,-0.329 5.655,-1.55z"></path></g></g></svg>
-        </a>
-    </div>
-
-    <!-- Repository information -->
-    {% if config.repo_url %}
-      <div class="md-header__source" style="margin-left: -5px !important;">
-        {% include "partials/source.html" %}
-      </div>
-    {% endif %}
-  </nav>
-
-  <!-- Navigation tabs (sticky) -->
-  {% if "navigation.tabs.sticky" in features %}
-    {% if "navigation.tabs" in features %}
-      {% include "partials/tabs.html" %}
-    {% endif %}
-  {% endif %}
-</header>

docs/src/ann_indexes.md

@@ -1,7 +1,7 @@
 # ANN (Approximate Nearest Neighbor) Indexes
 
 You can create an index over your vector data to make search faster.
-Vector indexes are faster but less accurate than exhaustive search (KNN or Flat Search).
+Vector indexes are faster but less accurate than exhaustive search.
 LanceDB provides many parameters to fine-tune the index's size, the speed of queries, and the accuracy of results.
 
 Currently, LanceDB does *not* automatically create the ANN index.
@@ -10,64 +10,36 @@ If you can live with <100ms latency, skipping index creation is a simpler workfl
 
 In the future we will look to automatically create and configure the ANN index.
 
-## Types of Index
+## Creating an ANN Index
 
-Lance can support multiple index types, the most widely used one is `IVF_PQ`.
+Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
 
-* `IVF_PQ`: use **Inverted File Index (IVF)** to first divide the dataset into `N` partitions,
-   and then use **Product Quantization** to compress vectors in each partition.
-* `DISKANN` (**Experimental**): organize the vector as a on-disk graph, where the vertices approximately
-   represent the nearest neighbors of each vector.
+```python
+import lancedb
+import numpy as np
+uri = "~/.lancedb"
+db = lancedb.connect(uri)
 
-## Creating an IVF_PQ Index
+# Create 10,000 sample vectors
+data = [{"vector": row, "item": f"item {i}"}
+   for i, row in enumerate(np.random.random((10_000, 768)).astype('float32'))]
 
-Lance supports `IVF_PQ` index type by default.
+# Add the vectors to a table
+tbl = db.create_table("my_vectors", data=data)
 
-=== "Python"
-     Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
+# Create and train the index - you need to have enough data in the table for an effective training step
+tbl.create_index(num_partitions=256, num_sub_vectors=96)
+```
 
-     ```python
-     import lancedb
-     import numpy as np
-     uri = "data/sample-lancedb"
-     db = lancedb.connect(uri)
-
-     # Create 10,000 sample vectors
-     data = [{"vector": row, "item": f"item {i}"}
-        for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]
-
-     # Add the vectors to a table
-     tbl = db.create_table("my_vectors", data=data)
-
-     # Create and train the index - you need to have enough data in the table for an effective training step
-     tbl.create_index(num_partitions=256, num_sub_vectors=96)
-     ```
-
-=== "Javascript"
-     ```javascript
-     const vectordb = require('vectordb')
-     const db = await vectordb.connect('data/sample-lancedb')
-
-     let data = []
-     for (let i = 0; i < 10_000; i++) {
-         data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
-     }
-     const table = await db.createTable('my_vectors', data)
-     await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 256, num_sub_vectors: 96 })
-     ```
-
-- **metric** (default: "L2"): The distance metric to use. By default it uses euclidean distance "`L2`".
-We also support "cosine" and "dot" distance as well.
-- **num_partitions** (default: 256): The number of partitions of the index.
-- **num_sub_vectors** (default: 96): The number of sub-vectors (M) that will be created during Product Quantization (PQ).
-For D dimensional vector, it will be divided into `M` of `D/M` sub-vectors, each of which is presented by
-a single PQ code.
-
-<figure markdown>
-  ![IVF PQ](./assets/ivf_pq.png)
-  <figcaption>IVF_PQ index with <code>num_partitions=2, num_sub_vectors=4</code></figcaption>
-</figure>
+Since `create_index` has a training step, it can take a few minutes to finish for large tables. You can control the index
+creation by providing the following parameters:
 
+- **metric** (default: "L2"): The distance metric to use. By default we use euclidean distance. We also support cosine distance.
+- **num_partitions** (default: 256): The number of partitions of the index. The number of partitions should be configured so each partition has 3-5K vectors. For example, a table
+with ~1M vectors should use 256 partitions. You can specify arbitrary number of partitions but powers of 2 is most conventional.
+A higher number leads to faster queries, but it makes index generation slower.
+- **num_sub_vectors** (default: 96): The number of subvectors (M) that will be created during Product Quantization (PQ). A larger number makes
+search more accurate, but also makes the index larger and slower to build.
 
 ## Querying an ANN Index
 
@@ -81,33 +53,22 @@ There are a couple of parameters that can be used to fine-tune the search:
   e.g., for 1M vectors divided up into 256 partitions, nprobes should be set to ~20-40.<br/>
   Note: nprobes is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
 - **refine_factor** (default: None): Refine the results by reading extra elements and re-ranking them in memory.<br/>
-  A higher number makes search more accurate but also slower. If you find the recall is less than ideal, try refine_factor=10 to start.<br/>
+  A higher number makes search more accurate but also slower. If you find the recall is less than idea, try refine_factor=10 to start.<br/>
   e.g., for 1M vectors divided into 256 partitions, if you're looking for top 20, then refine_factor=200 reranks the whole partition.<br/>
   Note: refine_factor is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
 
-=== "Python"
-     ```python
-     tbl.search(np.random.random((1536))) \
-         .limit(2) \
-         .nprobes(20) \
-         .refine_factor(10) \
-         .to_df()
-     ```
-     ```
-                                              vector       item       score
-     0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
-     1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
-     ```
 
-=== "Javascript"
-     ```javascript
-     const results_1 = await table
-         .search(Array(1536).fill(1.2))
-         .limit(2)
-         .nprobes(20)
-         .refineFactor(10)
-         .execute()
-     ```
+```python
+tbl.search(np.random.random((768))) \
+    .limit(2) \
+    .nprobes(20) \
+    .refine_factor(10) \
+    .to_df()
+
+                                              vector       item       score
+0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
+1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
+```
 
 The search will return the data requested in addition to the score of each item.
 
@@ -117,66 +78,18 @@ The search will return the data requested in addition to the score of each item.
 
 You can further filter the elements returned by a search using a where clause.
 
-=== "Python"
-     ```python
-     tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_df()
-     ```
-
-=== "Javascript"
-     ```javascript
-     const results_2 = await table
-         .search(Array(1536).fill(1.2))
-         .where("id != '1141'")
-         .execute()
-     ```
+```python
+tbl.search(np.random.random((768))).where("item != 'item 1141'").to_df()
+```
 
 ### Projections (select clause)
 
 You can select the columns returned by the query using a select clause.
 
-=== "Python"
-     ```python
-     tbl.search(np.random.random((1536))).select(["vector"]).to_df()
-     ```
-     ```
-     vector      score
-     0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
-     1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
-     ...
-     ```
-
-=== "Javascript"
-     ```javascript
-     const results_3 = await table
-         .search(Array(1536).fill(1.2))
-         .select(["id"])
-         .execute()
-     ```
-
-## FAQ
-
-### When is it necessary to create an ANN vector index.
-
-`LanceDB` has manually tuned SIMD code for computing vector distances.
-In our benchmarks, computing 100K pairs of 1K dimension vectors only take less than 20ms.
-For small dataset (<100K rows) or the applications which can accept 100ms latency, vector indices are usually not necessary.
-
-For large-scale or higher dimension vectors, it is beneficial to create vector index.
-
-### How big is my index, and how many memory will it take.
-
-In LanceDB, all vector indices are disk-based, meaning that when responding to a vector query, only the relevant pages from the index file are loaded from disk and cached in memory. Additionally, each sub-vector is usually encoded into 1 byte PQ code.
-
-For example, with a 1024-dimension dataset, if we choose `num_sub_vectors=64`, each sub-vector has `1024 / 64 = 16` float32 numbers.
-Product quantization can lead to approximately `16 * sizeof(float32) / 1 = 64` times of space reduction.
-
-### How to choose `num_partitions` and `num_sub_vectors` for `IVF_PQ` index.
-
-`num_partitions` is used to decide how many partitions the first level `IVF` index uses.
-Higher number of partitions could lead to more efficient I/O during queries and better accuracy, but it takes much more time to train.
-On `SIFT-1M` dataset, our benchmark shows that keeping each partition 1K-4K rows lead to a good latency / recall.
-
-`num_sub_vectors` decides how many Product Quantization code to generate on each vector. Because
-Product Quantization is a lossy compression of the original vector, the more `num_sub_vectors` usually results to
-less space distortion, and thus yield better accuracy. However, similarly, more `num_sub_vectors` causes heavier I/O and
-more PQ computation, thus, higher latency. `dimension / num_sub_vectors` should be aligned with 8 for better SIMD efficiency.
+```python
+tbl.search(np.random.random((768))).select(["vector"]).to_df()
+                                              vector      score
+0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
+1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
+...
+```

docs/src/basic.md

@@ -1,171 +1,74 @@
 # Basic LanceDB Functionality
 
-We'll cover the basics of using LanceDB on your local machine in this section.
-
-??? info "LanceDB runs embedded on your backend application, so there is no need to run a separate server."
-
-      <img src="../assets/lancedb_embedded_explanation.png" width="650px" />
-
-## Installation
-
-=== "Python"
-      ```shell
-      pip install lancedb
-      ```
-
-=== "Javascript"
-      ```shell
-      npm install vectordb
-      ```
-
 ## How to connect to a database
 
-=== "Python"
-      ```python
-      import lancedb
-      uri = "data/sample-lancedb"
-      db = lancedb.connect(uri)
-      ```
+In local mode, LanceDB stores data in a directory on your local machine. To connect to a local database, you can use the following code:
+```python
+import lancedb
+uri = "~/.lancedb"
+db = lancedb.connect(uri)
+```
 
-      LanceDB will create the directory if it doesn't exist (including parent directories).
+LanceDB will create the directory if it doesn't exist (including parent directories).
 
-      If you need a reminder of the uri, use the `db.uri` property.
-
-=== "Javascript"
-      ```javascript
-      const lancedb = require("vectordb");
-
-      const uri = "data/sample-lancedb";
-      const db = await lancedb.connect(uri);
-      ```
-      
-      LanceDB will create the directory if it doesn't exist (including parent directories).
-
-      If you need a reminder of the uri, you can call `db.uri()`.
+If you need a reminder of the uri, use the `db.uri` property.
 
 ## How to create a table
 
-=== "Python"
-      ```python
-      tbl = db.create_table("my_table",
-                        data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                              {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      ```
+To create a table, you can use the following code:
+```python
+tbl = db.create_table("my_table",
+                      data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                            {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+```
 
-      If the table already exists, LanceDB will raise an error by default.
-      If you want to overwrite the table, you can pass in `mode="overwrite"`
-      to the `create_table` method.
+Under the hood, LanceDB is converting the input data into an Apache Arrow table
+and persisting it to disk in [Lance format](github.com/eto-ai/lance).
 
-      You can also pass in a pandas DataFrame directly:
-      ```python
-      import pandas as pd
-      df = pd.DataFrame([{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      tbl = db.create_table("table_from_df", data=df)
-      ```
+If the table already exists, LanceDB will raise an error by default.
+If you want to overwrite the table, you can pass in `mode="overwrite"`
+to the `create_table` method.
 
-=== "Javascript"
-      ```javascript
-      const tb = await db.createTable("my_table",
-                        data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                              {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      ```
-      
-!!! warning
-
-      If the table already exists, LanceDB will raise an error by default.
-      If you want to overwrite the table, you can pass in `mode="overwrite"`
-      to the `createTable` function.
-
-??? info "Under the hood, LanceDB is converting the input data into an Apache Arrow table and persisting it to disk in [Lance format](https://www.github.com/lancedb/lance)."
+You can also pass in a pandas DataFrame directly:
+```python
+import pandas as pd
+df = pd.DataFrame([{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                   {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+tbl = db.create_table("table_from_df", data=df)
+```
 
 ## How to open an existing table
 
 Once created, you can open a table using the following code:
+```python
+tbl = db.open_table("my_table")
+```
 
-=== "Python"
-      ```python
-      tbl = db.open_table("my_table")
-      ```
+If you forget the name of your table, you can always get a listing of all table names:
 
-      If you forget the name of your table, you can always get a listing of all table names:
-
-      ```python
-      print(db.table_names())
-      ```
-
-=== "Javascript"
-      ```javascript
-      const tbl = await db.openTable("my_table");
-      ```
-
-      If you forget the name of your table, you can always get a listing of all table names:
-
-      ```javascript
-      console.log(await db.tableNames());
-      ```
+```python
+db.table_names()
+```
 
 ## How to add data to a table
 
 After a table has been created, you can always add more data to it using
 
-=== "Python"
-      ```python
-      df = pd.DataFrame([{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
-                        {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}])
-      tbl.add(df)
-      ```
-
-=== "Javascript"
-      ```javascript
-      await tbl.add([{vector: [1.3, 1.4], item: "fizz", price: 100.0},
-              {vector: [9.5, 56.2], item: "buzz", price: 200.0}])
-      ```
-
-## How to delete rows from a table
-
-Use the `delete()` method on tables to delete rows from a table. To choose
-which rows to delete, provide a filter that matches on the metadata columns.
-This can delete any number of rows that match the filter.
-
-=== "Python"
-      ```python
-      tbl.delete('item = "fizz"')
-      ```
-
-=== "Javascript"
-      ```javascript
-      await tbl.delete('item = "fizz"')
-      ```
-
-The deletion predicate is a SQL expression that supports the same expressions
-as the `where()` clause on a search. They can be as simple or complex as needed.
-To see what expressions are supported, see the [SQL filters](sql.md) section.
-
-
-=== "Python"
-
-      Read more: [lancedb.table.Table.delete][]
-
-=== "Javascript"
-
-      Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
+```python
+df = pd.DataFrame([{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
+                   {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}])
+tbl.add(df)
+```
 
 ## How to search for (approximate) nearest neighbors
 
 Once you've embedded the query, you can find its nearest neighbors using the following code:
 
-=== "Python"
-      ```python
-      tbl.search([100, 100]).limit(2).to_df()
-      ```
+```python
+tbl.search([100, 100]).limit(2).to_df()
+```
 
-      This returns a pandas DataFrame with the results.
-
-=== "Javascript"
-      ```javascript
-      const query = await tbl.search([100, 100]).limit(2).execute();
-      ```
+This returns a pandas DataFrame with the results.
 
 ## What's next
 

docs/src/embedding.md

@@ -25,88 +25,55 @@ def embed_func(batch):
     return [model.encode(sentence) for sentence in batch]
 ```
 
-Please note that currently HuggingFace is only supported in the Python SDK.
-
 ### OpenAI example
 
 You can also use an external API like OpenAI to generate embeddings
 
-=== "Python"
-      ```python
-        import openai
-        import os
+```python
+import openai
+import os
 
-        # Configuring the environment variable OPENAI_API_KEY
-        if "OPENAI_API_KEY" not in os.environ:
-        # OR set the key here as a variable
-        openai.api_key = "sk-..."
+# Configuring the environment variable OPENAI_API_KEY
+if "OPENAI_API_KEY" not in os.environ:
+    # OR set the key here as a variable
+    openai.api_key = "sk-..."
 
-        # verify that the API key is working
-        assert len(openai.Model.list()["data"]) > 0
+# verify that the API key is working
+assert len(openai.Model.list()["data"]) > 0
 
-        def embed_func(c):
-            rs = openai.Embedding.create(input=c, engine="text-embedding-ada-002")
-            return [record["embedding"] for record in rs["data"]]
-      ```
-
-=== "Javascript"
-      ```javascript
-        const lancedb = require("vectordb");
-
-        // You need to provide an OpenAI API key
-        const apiKey = "sk-..."
-        // The embedding function will create embeddings for the 'text' column
-        const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
-      ```
+def embed_func(c):
+    rs = openai.Embedding.create(input=c, engine="text-embedding-ada-002")
+    return [record["embedding"] for record in rs["data"]]
+```
 
 ## Applying an embedding function
 
-=== "Python"
-    Using an embedding function, you can apply it to raw data
-    to generate embeddings for each row.
+Using an embedding function, you can apply it to raw data
+to generate embeddings for each row.
 
-    Say if you have a pandas DataFrame with a `text` column that you want to be embedded,
-    you can use the [with_embeddings](https://lancedb.github.io/lancedb/python/#lancedb.embeddings.with_embeddings)
-    function to generate embeddings and add create a combined pyarrow table:
+Say if you have a pandas DataFrame with a `text` column that you want to be embedded,
+you can use the [with_embeddings](https://lancedb.github.io/lancedb/python/#lancedb.embeddings.with_embeddings)
+function to generate embeddings and add create a combined pyarrow table:
 
+```python
+import pandas as pd
+from lancedb.embeddings import with_embeddings
 
-     ```python
-     import pandas as pd
-     from lancedb.embeddings import with_embeddings
+df = pd.DataFrame([{"text": "pepperoni"},
+                   {"text": "pineapple"}])
+data = with_embeddings(embed_func, df)
 
-     df = pd.DataFrame([{"text": "pepperoni"},
-                        {"text": "pineapple"}])
-     data = with_embeddings(embed_func, df)
+# The output is used to create / append to a table
+# db.create_table("my_table", data=data)
+```
 
-     # The output is used to create / append to a table
-     # db.create_table("my_table", data=data)
-     ```
+If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
 
-     If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
-
-     By default, LanceDB calls the function with batches of 1000 rows. This can be configured
-     using the `batch_size` parameter to `with_embeddings`.
-
-     LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
-     API call is reliable.
-
-=== "Javascript"
-     Using an embedding function, you can apply it to raw data
-     to generate embeddings for each row.
-
-     You can just pass the embedding function created previously and LanceDB will automatically generate
-     embededings for your data.
-
-      ```javascript
-      const db = await lancedb.connect("data/sample-lancedb");
-      const data = [
-        { text: 'pepperoni'  },
-        { text: 'pineapple' }
-      ]
-
-      const table = await db.createTable('vectors', data, embedding)
-      ```
+By default, LanceDB calls the function with batches of 1000 rows. This can be configured
+using the `batch_size` parameter to `with_embeddings`.
 
+LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
+API call is reliable.
 
 ## Searching with an embedding function
 
@@ -114,25 +81,13 @@ At inference time, you also need the same embedding function to embed your query
 It's important that you use the same model / function otherwise the embedding vectors don't
 belong in the same latent space and your results will be nonsensical.
 
-=== "Python"
-     ```python
-     query = "What's the best pizza topping?"
-     query_vector = embed_func([query])[0]
-     tbl.search(query_vector).limit(10).to_df()
-     ```
-
-     The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
-
-=== "Javascript"
-     ```javascript
-      const results = await table
-        .search("What's the best pizza topping?")
-        .limit(10)
-        .execute()
-     ```
-
-     The above snippet returns an array of records with the 10 closest vectors to the query.
+```python
+query = "What's the best pizza topping?"
+query_vector = embed_func([query])[0]
+tbl.search(query_vector).limit(10).to_df()
+```
 
+The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
 
 ## Roadmap
 

docs/src/examples/langchain.md

@@ -4,4 +4,4 @@
 
 <img id="splash" width="400" alt="langchain" src="https://user-images.githubusercontent.com/917119/236580868-61a246a9-e587-4c2b-8ae5-6fe5f7b7e81e.png">
 
-This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/code_qa_bot.ipynb)
+This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/notebooks/code_qa_bot.ipynb)

docs/src/examples/modal_langchain.py

@@ -1,117 +0,0 @@
-import pickle
-import re
-import sys
-import zipfile
-from pathlib import Path
-
-import requests
-from langchain.chains import RetrievalQA
-from langchain.document_loaders import UnstructuredHTMLLoader
-from langchain.embeddings import OpenAIEmbeddings
-from langchain.llms import OpenAI
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.vectorstores import LanceDB
-from modal import Image, Secret, Stub, web_endpoint
-
-import lancedb
-
-lancedb_image = Image.debian_slim().pip_install(
-    "lancedb", "langchain", "openai", "pandas", "tiktoken", "unstructured", "tabulate"
-)
-
-stub = Stub(
-    name="example-langchain-lancedb",
-    image=lancedb_image,
-    secrets=[Secret.from_name("my-openai-secret")],
-)
-
-docsearch = None
-docs_path = Path("docs.pkl")
-db_path = Path("lancedb")
-
-
-def get_document_title(document):
-    m = str(document.metadata["source"])
-    title = re.findall("pandas.documentation(.*).html", m)
-    if title[0] is not None:
-        return title[0]
-    return ""
-
-
-def download_docs():
-    pandas_docs = requests.get(
-        "https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip"
-    )
-    with open(Path("pandas.documentation.zip"), "wb") as f:
-        f.write(pandas_docs.content)
-
-    file = zipfile.ZipFile(Path("pandas.documentation.zip"))
-    file.extractall(path=Path("pandas_docs"))
-
-
-def store_docs():
-    docs = []
-
-    if not docs_path.exists():
-        for p in Path("pandas_docs/pandas.documentation").rglob("*.html"):
-            if p.is_dir():
-                continue
-            loader = UnstructuredHTMLLoader(p)
-            raw_document = loader.load()
-
-            m = {}
-            m["title"] = get_document_title(raw_document[0])
-            m["version"] = "2.0rc0"
-            raw_document[0].metadata = raw_document[0].metadata | m
-            raw_document[0].metadata["source"] = str(raw_document[0].metadata["source"])
-            docs = docs + raw_document
-
-        with docs_path.open("wb") as fh:
-            pickle.dump(docs, fh)
-    else:
-        with docs_path.open("rb") as fh:
-            docs = pickle.load(fh)
-
-    return docs
-
-
-def qanda_langchain(query):
-    download_docs()
-    docs = store_docs()
-
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200,)
-    documents = text_splitter.split_documents(docs)
-    embeddings = OpenAIEmbeddings()
-
-    db = lancedb.connect(db_path)
-    table = db.create_table(
-        "pandas_docs",
-        data=[
-            {
-                "vector": embeddings.embed_query("Hello World"),
-                "text": "Hello World",
-                "id": "1",
-            }
-        ],
-        mode="overwrite",
-    )
-    docsearch = LanceDB.from_documents(documents, embeddings, connection=table)
-    qa = RetrievalQA.from_chain_type(
-        llm=OpenAI(), chain_type="stuff", retriever=docsearch.as_retriever()
-    )
-    return qa.run(query)
-
-
-@stub.function()
-@web_endpoint(method="GET")
-def web(query: str):
-    answer = qanda_langchain(query)
-    return {
-        "answer": answer,
-    }
-
-
-@stub.function()
-def cli(query: str):
-    answer = qanda_langchain(query)
-    print(answer)

docs/src/examples/multimodal_search.md

@@ -1,7 +0,0 @@
-# Image multimodal search
-
-## Search through an image dataset using natural language, full text and SQL
-
-<img id="splash" width="400" alt="multimodal search" src="https://github.com/lancedb/lancedb/assets/917119/993a7c9f-be01-449d-942e-1ce1d4ed63af">
-
-This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/multimodal_search.ipynb)

docs/src/examples/nodejs.md

@@ -0,0 +1,99 @@
+# YouTube transcript QA bot with NodeJS
+
+## use LanceDB's Javascript API and OpenAI to build a QA bot for YouTube transcripts
+
+<img id="splash" width="400" alt="nodejs" src="https://github.com/lancedb/lancedb/assets/917119/3a140e75-bf8e-438a-a1e4-af14a72bcf98">
+
+This Q&A bot will allow you to search through youtube transcripts using natural language! We'll introduce how you can use LanceDB's Javascript API to store and manage your data easily.
+
+For this example we're using a HuggingFace dataset that contains YouTube transcriptions: `jamescalam/youtube-transcriptions`, to make it easier, we've converted it to a LanceDB `db` already, which you can download and put in a working directory:
+
+```wget -c https://eto-public.s3.us-west-2.amazonaws.com/lancedb_demo.tar.gz -O - | tar -xz -C .```
+
+Now, we'll create a simple app that can:
+1. Take a text based query and search for contexts in our corpus, using embeddings generated from the OpenAI Embedding API.
+2. Create a prompt with the contexts, and call the OpenAI Completion API to answer the text based query.
+
+Dependencies and setup of OpenAI API:
+
+```javascript
+const lancedb = require("vectordb");
+const { Configuration, OpenAIApi } = require("openai");
+
+const configuration = new Configuration({
+    apiKey: process.env.OPENAI_API_KEY,
+    });
+const openai = new OpenAIApi(configuration);
+```
+
+First, let's set our question and the context amount. The context amount will be used to query similar documents in our corpus.
+
+```javascript
+const QUESTION = "who was the 12th person on the moon and when did they land?";
+const CONTEXT_AMOUNT = 3;
+```
+
+Now, let's generate an embedding from this question:
+
+```javascript
+const embeddingResponse = await openai.createEmbedding({
+    model: "text-embedding-ada-002",
+    input: QUESTION,
+});
+
+const embedding = embeddingResponse.data["data"][0]["embedding"];
+```
+
+Once we have the embedding, we can connect to LanceDB (using the database we downloaded earlier), and search through the chatbot table.
+We'll extract 3 similar documents found.
+
+```javascript
+const db = await lancedb.connect('./lancedb');
+const tbl = await db.openTable('chatbot');
+const query = tbl.search(embedding);
+query.limit = CONTEXT_AMOUNT;
+const context = await query.execute();
+```
+
+Let's combine the context together so we can pass it into our prompt:
+
+```javascript
+for (let i = 1; i < context.length; i++) {
+    context[0]["text"] += " " + context[i]["text"];
+}
+```
+
+Lastly, let's construct the prompt. You could play around with this to create more accurate/better prompts to yield results.
+
+```javascript
+const prompt = "Answer the question based on the context below.\n\n" +
+    "Context:\n" +
+    `${context[0]["text"]}\n` +
+    `\n\nQuestion: ${QUESTION}\nAnswer:`;
+```
+
+We pass the prompt, along with the context, to the completion API.
+
+```javascript
+const completion = await openai.createCompletion({
+    model: "text-davinci-003",
+    prompt,
+    temperature: 0,
+    max_tokens: 400,
+    top_p: 1,
+    frequency_penalty: 0,
+    presence_penalty: 0,
+});
+```
+
+And that's it!
+
+```javascript
+console.log(completion.data.choices[0].text);
+```
+
+The response is (which is non deterministic):
+
+```
+The 12th person on the moon was Harrison Schmitt and he landed on December 11, 1972.
+```

docs/src/examples/serverless_qa_bot_with_modal_and_langchain.md

@@ -1,166 +0,0 @@
-# Serverless QA Bot with Modal and LangChain
-
-## use LanceDB's LangChain integration with Modal to run a serverless app
-
-<img id="splash" width="400" alt="modal" src="https://github.com/lancedb/lancedb/assets/917119/7d80a40f-60d7-48a6-972f-dab05000eccf">
-
-We're going to build a QA bot for your documentation using LanceDB's LangChain integration and use Modal for deployment.
-
-Modal is an end-to-end compute platform for model inference, batch jobs, task queues, web apps and more. It's a great way to deploy your LanceDB models and apps.
-
-To get started, ensure that you have created an account and logged into [Modal](https://modal.com/). To follow along, the full source code is available on Github [here](https://github.com/lancedb/lancedb/blob/main/docs/src/examples/modal_langchain.py).
-
-### Setting up Modal
-
-We'll start by specifying our dependencies and creating a new Modal `Stub`:
-
-```python
-lancedb_image = Image.debian_slim().pip_install(
-    "lancedb",
-    "langchain",
-    "openai",
-    "pandas",
-    "tiktoken",
-    "unstructured",
-    "tabulate"
-)
-
-stub = Stub(
-    name="example-langchain-lancedb",
-    image=lancedb_image,
-    secrets=[Secret.from_name("my-openai-secret")],
-)
-```
-
-We're using Modal's Secrets injection to secure our OpenAI key. To set your own, you can access the Modal UI and enter your key.
-
-### Setting up caches for LanceDB and LangChain
-
-Next, we can setup some globals to cache our LanceDB database, as well as our LangChain docsource:
-
-```python
-docsearch = None
-docs_path = Path("docs.pkl")
-db_path = Path("lancedb")
-```
-
-### Downloading our dataset
-
-We're going use a pregenerated dataset, which stores HTML files of the Pandas 2.0 documentation. 
-You could switch this out for your own dataset.
-
-```python
-def download_docs():
-    pandas_docs = requests.get("https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip")
-    with open(Path("pandas.documentation.zip"), "wb") as f:
-        f.write(pandas_docs.content)
-
-    file = zipfile.ZipFile(Path("pandas.documentation.zip"))
-    file.extractall(path=Path("pandas_docs"))
-```
-
-### Pre-processing the dataset and generating metadata
-
-Once we've downloaded it, we want to parse and pre-process them using LangChain, and then vectorize them and store it in LanceDB.
-Let's first create a function that uses LangChains `UnstructuredHTMLLoader` to parse them.
-We can then add our own metadata to it and store it alongside the data, we'll later be able to use this for filtering metadata.
-
-```python
-def store_docs():
-    docs = []
-
-    if not docs_path.exists():
-        for p in Path("pandas_docs/pandas.documentation").rglob("*.html"):
-            if p.is_dir():
-                continue
-            loader = UnstructuredHTMLLoader(p)
-            raw_document = loader.load()
-
-            m = {}
-            m["title"] = get_document_title(raw_document[0])
-            m["version"] = "2.0rc0"
-            raw_document[0].metadata = raw_document[0].metadata | m
-            raw_document[0].metadata["source"] = str(raw_document[0].metadata["source"])
-            docs = docs + raw_document
-
-        with docs_path.open("wb") as fh:
-            pickle.dump(docs, fh)
-    else:
-        with docs_path.open("rb") as fh:
-            docs = pickle.load(fh)
-
-    return docs
-```
-
-### Simple LangChain chain for a QA bot
-
-Now we can create a simple LangChain chain for our QA bot. We'll use the `RecursiveCharacterTextSplitter` to split our documents into chunks, and then use the `OpenAIEmbeddings` to vectorize them.
-
-Lastly, we'll create a LanceDB table and store the vectorized documents in it, then create a `RetrievalQA` model from the chain and return it.
-
-```python
-def qanda_langchain(query):
-    download_docs()
-    docs = store_docs()
-
-    text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=1000,
-        chunk_overlap=200,
-    )
-    documents = text_splitter.split_documents(docs)
-    embeddings = OpenAIEmbeddings()
-
-    db = lancedb.connect(db_path) 
-    table = db.create_table("pandas_docs", data=[
-        {"vector": embeddings.embed_query("Hello World"), "text": "Hello World", "id": "1"}
-    ], mode="overwrite")
-    docsearch = LanceDB.from_documents(documents, embeddings, connection=table)
-    qa = RetrievalQA.from_chain_type(llm=OpenAI(), chain_type="stuff", retriever=docsearch.as_retriever())
-    return qa.run(query)
-```
-
-### Creating our Modal entry points
-
-Now we can create our Modal entry points for our CLI and web endpoint:
-
-```python
-@stub.function()
-@web_endpoint(method="GET")
-def web(query: str):
-    answer = qanda_langchain(query)
-    return {
-        "answer": answer,
-    }
-    
-@stub.function()
-def cli(query: str):
-    answer = qanda_langchain(query)
-    print(answer)
-```
-
-# Testing it out!
-
-Testing the CLI:
-
-```bash
-modal run modal_langchain.py --query "What are the major differences in pandas 2.0?"
-```
-
-Testing the web endpoint:
-
-```bash
-modal serve modal_langchain.py
-```
-
-In the CLI, Modal will provide you a web endpoint. Copy this endpoint URI for the next step.
-Once this is served, then we can hit it with `curl`. 
-
-Note, the first time this runs, it will take a few minutes to download the dataset and vectorize it.
-An actual production example would pre-cache/load the dataset and vectorized documents prior
-
-```bash
-curl --get --data-urlencode "query=What are the major differences in pandas 2.0?" https://your-modal-endpoint-app.modal.run
-
-{"answer":" The major differences in pandas 2.0 include the ability to use any numpy numeric dtype in a Index, installing optional dependencies with pip extras, and enhancements, bug fixes, and performance improvements."}
-```
-

docs/src/examples/transformerjs_embedding_search_nodejs.md

@@ -1,121 +0,0 @@
-# Vector embedding search using TransformersJS
-
-## Embed and query data from LacneDB using TransformersJS
-
-<img id="splash" width="400" alt="transformersjs" src="https://github.com/lancedb/lancedb/assets/43097991/88a31e30-3d6f-4eef-9216-4b7c688f1b4f">
-
-This example shows how to use the [transformers.js](https://github.com/xenova/transformers.js) library to perform vector embedding search using LanceDB's Javascript API.
-
-
-### Setting up
-First, install the dependencies:
-```bash
-npm install vectordb
-npm i @xenova/transformers
-```
-
-We will also be using the [all-MiniLM-L6-v2](https://huggingface.co/Xenova/all-MiniLM-L6-v2) model to make it compatible with Transformers.js
-
-Within our `index.js` file we will import the necessary libraries and define our model and database:
-
-```javascript
-const lancedb = require('vectordb')
-const { pipeline } = await import('@xenova/transformers')
-const pipe = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
-```
-
-### Creating the embedding function
-
-Next, we will create a function that will take in a string and return the vector embedding of that string. We will use the `pipe` function we defined earlier to get the vector embedding of the string.
-
-```javascript
-// Define the function. `sourceColumn` is required for LanceDB to know
-// which column to use as input.
-const embed_fun = {}
-embed_fun.sourceColumn = 'text'
-embed_fun.embed = async function (batch) {
-    let result = []
-    // Given a batch of strings, we will use the `pipe` function to get
-    // the vector embedding of each string.
-    for (let text of batch) {
-        // 'mean' pooling and normalizing allows the embeddings to share the
-        // same length.
-        const res = await pipe(text, { pooling: 'mean', normalize: true })
-        result.push(Array.from(res['data']))
-    }
-    return (result)
-}
-```
-
-### Creating the database
-
-Now, we will create the LanceDB database and add the embedding function we defined earlier.
-
-```javascript
-// Link a folder and create a table with data
-const db = await lancedb.connect('data/sample-lancedb')
-
-// You can also import any other data, but make sure that you have a column
-// for the embedding function to use.
-const data = [
-    { id: 1, text: 'Cherry', type: 'fruit' },
-    { id: 2, text: 'Carrot', type: 'vegetable' },
-    { id: 3, text: 'Potato', type: 'vegetable' },
-    { id: 4, text: 'Apple', type: 'fruit' },
-    { id: 5, text: 'Banana', type: 'fruit' }
-]
-
-// Create the table with the embedding function
-const table = await db.createTable('food_table', data, "create", embed_fun)
-```
-
-### Performing the search
-
-Now, we can perform the search using the `search` function. LanceDB automatically uses the embedding function we defined earlier to get the vector embedding of the query string.
-
-```javascript
-// Query the table
-const results = await table
-    .search("a sweet fruit to eat")
-    .metricType("cosine")
-    .limit(2)
-    .execute()
-console.log(results.map(r => r.text))
-```
-```bash
-[ 'Banana', 'Cherry' ]
-```
-
-Output of `results`:
-```bash
-[
-  {
-    vector: Float32Array(384) [
-      -0.057455405592918396,
-      0.03617725893855095,
-      -0.0367760956287384,
-      ... 381 more items
-    ],
-    id: 5,
-    text: 'Banana',
-    type: 'fruit',
-    score: 0.4919965863227844
-  },
-  {
-    vector: Float32Array(384) [
-      0.0009714411571621895,
-      0.008223623037338257,
-      0.009571489877998829,
-      ... 381 more items
-    ],
-    id: 1,
-    text: 'Cherry',
-    type: 'fruit',
-    score: 0.5540297031402588
-  }
-]
-```
-
-### Wrapping it up
-
-In this example, we showed how to use the `transformers.js` library to perform vector embedding search using LanceDB's Javascript API. You can find the full code for this example on [Github](https://github.com/lancedb/lancedb/blob/main/node/examples/js-transformers/index.js)!

docs/src/examples/youtube_transcript_bot_with_nodejs.md

@@ -1,139 +0,0 @@
-# YouTube transcript QA bot with NodeJS
-
-## use LanceDB's Javascript API and OpenAI to build a QA bot for YouTube transcripts
-
-<img id="splash" width="400" alt="nodejs" src="https://github.com/lancedb/lancedb/assets/917119/3a140e75-bf8e-438a-a1e4-af14a72bcf98">
-
-This Q&A bot will allow you to search through youtube transcripts using natural language! We'll introduce how to use LanceDB's Javascript API to store and manage your data easily.
-
-```bash
-npm install vectordb
-```
-
-## Download the data
-
-For this example, we're using a sample of a HuggingFace dataset that contains YouTube transcriptions: `jamescalam/youtube-transcriptions`. Download and extract this file under the `data` folder:
-
-```bash
-wget -c https://eto-public.s3.us-west-2.amazonaws.com/datasets/youtube_transcript/youtube-transcriptions_sample.jsonl
-```
-
-## Prepare Context
-
-Each item in the dataset contains just a short chunk of text. We'll need to merge a bunch of these chunks together on a rolling basis. For this demo, we'll look back 20 records to create a more complete context for each sentence.
-
-First, we need to read and parse the input file.
-
-```javascript
-const lines = (await fs.readFile(INPUT_FILE_NAME, 'utf-8'))
-  .toString()
-  .split('\n')
-  .filter(line => line.length > 0)
-  .map(line => JSON.parse(line))
-
-const data = contextualize(lines, 20, 'video_id')
-```
-
-The contextualize function groups the transcripts by video_id and then creates the expanded context for each item.
-
-```javascript
-function contextualize (rows, contextSize, groupColumn) {
-  const grouped = []
-  rows.forEach(row => {
-    if (!grouped[row[groupColumn]]) {
-      grouped[row[groupColumn]] = []
-    }
-    grouped[row[groupColumn]].push(row)
-  })
-
-  const data = []
-  Object.keys(grouped).forEach(key => {
-    for (let i = 0; i < grouped[key].length; i++) {
-      const start = i - contextSize > 0 ? i - contextSize : 0
-      grouped[key][i].context = grouped[key].slice(start, i + 1).map(r => r.text).join(' ')
-    }
-    data.push(...grouped[key])
-  })
-  return data
-}
-```
-
-## Create the LanceDB Table
-
-To load our data into LanceDB, we need to create embedding (vectors) for each item. For this example, we will use the OpenAI embedding functions, which have a native integration with LanceDB.
-
-```javascript
-// You need to provide an OpenAI API key, here we read it from the OPENAI_API_KEY environment variable
-const apiKey = process.env.OPENAI_API_KEY
-// The embedding function will create embeddings for the 'context' column
-const embedFunction = new lancedb.OpenAIEmbeddingFunction('context', apiKey)
-// Connects to LanceDB
-const db = await lancedb.connect('data/youtube-lancedb')
-const tbl = await db.createTable('vectors', data, embedFunction)
-```
-
-## Create and answer the prompt
-
-We will accept questions in natural language and use our corpus stored in LanceDB to answer them. First, we need to set up the OpenAI client:
-
-```javascript
-const configuration = new Configuration({ apiKey })
-const openai = new OpenAIApi(configuration)
-```
-
-Then we can prompt questions and use LanceDB to retrieve the three most relevant transcripts for this prompt.
-
-```javascript
-const query = await rl.question('Prompt: ')
-const results = await tbl
-  .search(query)
-  .select(['title', 'text', 'context'])
-  .limit(3)
-  .execute()
-```
-
-The query and the transcripts' context are appended together in a single prompt:
-
-```javascript
-function createPrompt (query, context) {
-    let prompt =
-        'Answer the question based on the context below.\n\n' +
-        'Context:\n'
-
-    // need to make sure our prompt is not larger than max size
-    prompt = prompt + context.map(c => c.context).join('\n\n---\n\n').substring(0, 3750)
-    prompt = prompt + `\n\nQuestion: ${query}\nAnswer:`
-    return prompt
-}
-```
-
-We can now use the OpenAI Completion API to process our custom prompt and give us an answer.
-
-```javascript
-const response = await openai.createCompletion({
-  model: 'text-davinci-003',
-  prompt: createPrompt(query, results),
-  max_tokens: 400,
-  temperature: 0,
-  top_p: 1,
-  frequency_penalty: 0,
-  presence_penalty: 0
-})
-console.log(response.data.choices[0].text)
-```
-
-## Let's put it all together now
-
-Now we can provide queries and have them answered based on your local LanceDB data.
-
-```bash
-Prompt: who was the 12th person on the moon and when did they land?
- The 12th person on the moon was Harrison Schmitt and he landed on December 11, 1972.
-Prompt: Which training method should I use for sentence transformers when I only have pairs of related sentences?
- NLI with multiple negative ranking loss.
-```
-
-## That's a wrap
-
-In this example, you learned how to use LanceDB to store and query embedding representations of your local data. The complete example code is on [GitHub](https://github.com/lancedb/lancedb/tree/main/node/examples), and you can also download the LanceDB dataset using [this link](https://eto-public.s3.us-west-2.amazonaws.com/datasets/youtube_transcript/youtube-lancedb.zip).
-

docs/src/examples/youtube_transcript_search.md

@@ -4,4 +4,4 @@
 
 <img id="splash" width="400" alt="youtube transcript search" src="https://user-images.githubusercontent.com/917119/236965568-def7394d-171c-45f2-939d-8edfeaadd88c.png">
 
-This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb)
+This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/notebooks/youtube_transcript_search.ipynb)

docs/src/fts.md

@@ -1,65 +0,0 @@
-# [EXPERIMENTAL] Full text search
-
-LanceDB now provides experimental support for full text search.
-This is currently Python only. We plan to push the integration down to Rust in the future
-to make this available for JS as well.
-
-## Installation
-
-To use full text search, you must install optional dependency tantivy-py:
-
-# tantivy 0.19.2
-pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-
-
-## Quickstart
-
-Assume:
-1. `table` is a LanceDB Table
-2. `text` is the name of the Table column that we want to index
-
-For example,
-
-```python
-import lancedb
-
-uri = "data/sample-lancedb"
-db = lancedb.connect(uri)
-
-table = db.create_table("my_table",
-            data=[{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
-                  {"vector": [5.9, 26.5], "text": "There are several kittens playing"}])
-
-```
-
-To create the index:
-
-```python
-table.create_fts_index("text")
-```
-
-To search:
-
-```python
-df = table.search("puppy").limit(10).select(["text"]).to_df()
-```
-
-LanceDB automatically looks for an FTS index if the input is str.
-
-## Multiple text columns
-
-If you have multiple columns to index, pass them all as a list to `create_fts_index`:
-
-```python
-table.create_fts_index(["text1", "text2"])
-```
-
-Note that the search API call does not change - you can search over all indexed columns at once.
-
-## Current limitations
-
-1. Currently we do not yet support incremental writes.
-If you add data after fts index creation, it won't be reflected
-in search results until you do a full reindex.
-
-2. We currently only support local filesystem paths for the fts index.

docs/src/index.md

@@ -1,6 +1,6 @@
 # Welcome to LanceDB's Documentation
 
-LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrevial, filtering and management of embeddings.
+LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrivial, filtering and management of embeddings.
 
 The key features of LanceDB include:
 
@@ -8,65 +8,42 @@ The key features of LanceDB include:
 
 * Store, query and filter vectors, metadata and multi-modal data (text, images, videos, point clouds, and more).
 
-* Support for vector similarity search, full-text search and SQL.
-
-* Native Python and Javascript/Typescript support.
+* Native Python and Javascript/Typescript support (coming soon).
 
 * Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure.
 
-* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
+* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lanecdb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
 
-LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
+LanceDB's core is written in Rust 🦀 and is built using Lance, an open-source columnar format designed for performant ML workloads.
 
-## Quick Start
 
-=== "Python"
-      ```shell
-      pip install lancedb
-      ```
+## Installation
 
-      ```python
-      import lancedb
+```shell
+pip install lancedb
+```
 
-      uri = "data/sample-lancedb"
-      db = lancedb.connect(uri)
-      table = db.create_table("my_table",
-                              data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                                    {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      result = table.search([100, 100]).limit(2).to_df()
-      ```
+## Quickstart
 
-=== "Javascript"
-      ```shell
-      npm install vectordb
-      ```
+```python
+import lancedb
 
-      ```javascript
-      const lancedb = require("vectordb");
+db = lancedb.connect(".")
+table = db.create_table("my_table",
+                         data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                               {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+result = table.search([100, 100]).limit(2).to_df()
+```
 
-      const uri = "data/sample-lancedb";
-      const db = await lancedb.connect(uri);
-      const table = await db.createTable("my_table",
-            [{ id: 1, vector: [3.1, 4.1], item: "foo", price: 10.0 },
-            { id: 2, vector: [5.9, 26.5], item: "bar", price: 20.0 }])
-      const results = await table.search([100, 100]).limit(2).execute();
-      ```
+## Complete Demos
 
-## Complete Demos (Python)
-- [YouTube Transcript Search](notebooks/youtube_transcript_search.ipynb)
-- [Documentation QA Bot using LangChain](notebooks/code_qa_bot.ipynb)
-- [Multimodal search using CLIP](notebooks/multimodal_search.ipynb)
-- [Serverless QA Bot with S3 and Lambda](examples/serverless_lancedb_with_s3_and_lambda.md)
-- [Serverless QA Bot with Modal](examples/serverless_qa_bot_with_modal_and_langchain.md)
+We will be adding completed demo apps built using LanceDB.
+- [YouTube Transcript Search](../notebooks/youtube_transcript_search.ipynb)
 
-## Complete Demos (JavaScript)
-- [YouTube Transcript Search](examples/youtube_transcript_bot_with_nodejs.md)
 
 ## Documentation Quick Links
 * [`Basic Operations`](basic.md) - basic functionality of LanceDB.
 * [`Embedding Functions`](embedding.md) - functions for working with embeddings.
 * [`Indexing`](ann_indexes.md) - create vector indexes to speed up queries.
-* [`Full text search`](fts.md) - [EXPERIMENTAL] full-text search API
-* [`Ecosystem Integrations`](python/integration.md) - integrating LanceDB with python data tooling ecosystem.
-* [`Python API Reference`](python/python.md) - detailed documentation for the LanceDB Python SDK.
-* [`Node API Reference`](javascript/modules.md) - detailed documentation for the LanceDB Python SDK.
+* [`Ecosystem Integrations`](integrations.md) - integrating LanceDB with python data tooling ecosystem.
+* [`API Reference`](python.md) - detailed documentation for the LanceDB Python SDK.

docs/src/integrations.md

@@ -0,0 +1,111 @@
+# Integrations
+
+Built on top of Apache Arrow, `LanceDB` is easy to integrate with the Python ecosystem, including Pandas, PyArrow and DuckDB.
+
+## Pandas and PyArrow
+
+First, we need to connect to a `LanceDB` database.
+
+``` py
+
+import lancedb
+
+db = lancedb.connect("/tmp/lancedb")
+```
+
+And write a `Pandas DataFrame` to LanceDB directly.
+
+```py
+import pandas as pd
+
+data = pd.DataFrame({
+    "vector": [[3.1, 4.1], [5.9, 26.5]],
+    "item": ["foo", "bar"],
+    "price": [10.0, 20.0]
+})
+table = db.create_table("pd_table", data=data)
+
+# Optionally, create a IVF_PQ index
+table.create_index(num_partitions=256, num_sub_vectors=96)
+```
+
+You will find detailed instructions of creating dataset and index in [Basic Operations](basic.md) and [Indexing](indexing.md)
+sections.
+
+
+We can now perform similarity searches via `LanceDB`.
+
+```py
+# Open the table previously created.
+table = db.open_table("pd_table")
+
+query_vector = [100, 100]
+# Pandas DataFrame
+df = table.search(query_vector).limit(1).to_df()
+print(df)
+```
+
+```
+    vector     item  price        score
+0  [5.9, 26.5]  bar   20.0  14257.05957
+```
+
+If you have a simple filter, it's faster to provide a where clause to `LanceDB`'s search query.
+If you have more complex criteria, you can always apply the filter to the resulting pandas `DataFrame` from the search query.
+
+```python
+
+# Apply the filter via LanceDB
+results = table.search([100, 100]).where("price < 15").to_df()
+assert len(results) == 1
+assert results["item"].iloc[0] == "foo"
+
+# Apply the filter via Pandas
+df = results = table.search([100, 100]).to_df()
+results = df[df.price < 15]
+assert len(results) == 1
+assert results["item"].iloc[0] == "foo"
+```
+
+## DuckDB
+
+`LanceDB` works with `DuckDB` via [PyArrow integration](https://duckdb.org/docs/guides/python/sql_on_arrow).
+
+Let us start with installing `duckdb` and `lancedb`.
+
+```shell
+pip install duckdb lancedb
+```
+
+We will re-use the dataset created previously
+
+```python
+import lancedb
+
+db = lancedb.connect("/tmp/lancedb")
+table = db.open_table("pd_table")
+arrow_table = table.to_arrow()
+```
+
+`DuckDB` can directly query the `arrow_table`:
+
+```python
+In [15]: duckdb.query("SELECT * FROM t")
+Out[15]:
+┌─────────────┬─────────┬────────┐
+│   vector    │  item   │ price  │
+│   float[]   │ varchar │ double │
+├─────────────┼─────────┼────────┤
+│ [3.1, 4.1]  │ foo     │   10.0 │
+│ [5.9, 26.5] │ bar     │   20.0 │
+└─────────────┴─────────┴────────┘
+
+In [16]: duckdb.query("SELECT mean(price) FROM t")
+Out[16]:
+┌─────────────┐
+│ mean(price) │
+│   double    │
+├─────────────┤
+│        15.0 │
+└─────────────┘
+```

docs/src/javascript/.nojekyll

@@ -1 +0,0 @@
-TypeDoc added this file to prevent GitHub Pages from using Jekyll. You can turn off this behavior by setting the `githubPages` option to false.

docs/src/javascript/README.md

@@ -1,73 +0,0 @@
-vectordb / [Exports](modules.md)
-
-# LanceDB
-
-A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb).
-
-## Installation
-
-```bash
-npm install vectordb
-```
-
-This will download the appropriate native library for your platform. We currently
-support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
-yet support Windows or musl-based Linux (such as Alpine Linux).
-
-## Usage
-
-### Basic Example
-
-```javascript
-const lancedb = require('vectordb');
-const db = await lancedb.connect('data/sample-lancedb');
-const table = await db.createTable("my_table",
-      [{ id: 1, vector: [0.1, 1.0], item: "foo", price: 10.0 },
-      { id: 2, vector: [3.9, 0.5], item: "bar", price: 20.0 }])
-const results = await table.search([0.1, 0.3]).limit(20).execute();
-console.log(results);
-```
-
-The [examples](./examples) folder contains complete examples.
-
-## Development
-
-To build everything fresh:
-
-```bash
-npm install
-npm run tsc
-npm run build
-```
-
-Then you should be able to run the tests with:
-
-```bash
-npm test
-```
-
-### Rebuilding Rust library
-
-```bash
-npm run build
-```
-
-### Rebuilding Typescript
-
-```bash
-npm run tsc
-```
-
-### Fix lints
-
-To run the linter and have it automatically fix all errors
-
-```bash
-npm run lint -- --fix
-```
-
-To build documentation
-
-```bash
-npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
-```

docs/src/javascript/classes/LocalConnection.md

@@ -1,350 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / LocalConnection
-
-# Class: LocalConnection
-
-A connection to a LanceDB database.
-
-## Implements
-
-- [`Connection`](../interfaces/Connection.md)
-
-## Table of contents
-
-### Constructors
-
-- [constructor](LocalConnection.md#constructor)
-
-### Properties
-
-- [\_db](LocalConnection.md#_db)
-- [\_options](LocalConnection.md#_options)
-
-### Accessors
-
-- [uri](LocalConnection.md#uri)
-
-### Methods
-
-- [createTable](LocalConnection.md#createtable)
-- [createTableArrow](LocalConnection.md#createtablearrow)
-- [dropTable](LocalConnection.md#droptable)
-- [openTable](LocalConnection.md#opentable)
-- [tableNames](LocalConnection.md#tablenames)
-
-## Constructors
-
-### constructor
-
-• **new LocalConnection**(`db`, `options`)
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `db` | `any` |
-| `options` | [`ConnectionOptions`](../interfaces/ConnectionOptions.md) |
-
-#### Defined in
-
-[index.ts:184](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L184)
-
-## Properties
-
-### \_db
-
-• `Private` `Readonly` **\_db**: `any`
-
-#### Defined in
-
-[index.ts:182](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L182)
-
-___
-
-### \_options
-
-• `Private` `Readonly` **\_options**: [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
-
-#### Defined in
-
-[index.ts:181](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L181)
-
-## Accessors
-
-### uri
-
-• `get` **uri**(): `string`
-
-#### Returns
-
-`string`
-
-#### Implementation of
-
-[Connection](../interfaces/Connection.md).[uri](../interfaces/Connection.md#uri)
-
-#### Defined in
-
-[index.ts:189](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L189)
-
-## Methods
-
-### createTable
-
-▸ **createTable**(`name`, `data`, `mode?`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-Creates a new Table and initialize it with new data.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
-| `mode?` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-#### Implementation of
-
-[Connection](../interfaces/Connection.md).[createTable](../interfaces/Connection.md#createtable)
-
-#### Defined in
-
-[index.ts:230](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L230)
-
-▸ **createTable**(`name`, `data`, `mode`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` |
-| `data` | `Record`<`string`, `unknown`\>[] |
-| `mode` | [`WriteMode`](../enums/WriteMode.md) |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-#### Implementation of
-
-Connection.createTable
-
-#### Defined in
-
-[index.ts:231](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L231)
-
-▸ **createTable**<`T`\>(`name`, `data`, `mode`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-Creates a new Table and initialize it with new data.
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
-| `mode` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-#### Implementation of
-
-Connection.createTable
-
-#### Defined in
-
-[index.ts:241](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L241)
-
-▸ **createTable**<`T`\>(`name`, `data`, `mode`, `embeddings?`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` |
-| `data` | `Record`<`string`, `unknown`\>[] |
-| `mode` | [`WriteMode`](../enums/WriteMode.md) |
-| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-#### Implementation of
-
-Connection.createTable
-
-#### Defined in
-
-[index.ts:242](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L242)
-
-___
-
-### createTableArrow
-
-▸ **createTableArrow**(`name`, `table`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` |
-| `table` | `Table`<`any`\> |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-#### Implementation of
-
-[Connection](../interfaces/Connection.md).[createTableArrow](../interfaces/Connection.md#createtablearrow)
-
-#### Defined in
-
-[index.ts:266](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L266)
-
-___
-
-### dropTable
-
-▸ **dropTable**(`name`): `Promise`<`void`\>
-
-Drop an existing table.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table to drop. |
-
-#### Returns
-
-`Promise`<`void`\>
-
-#### Implementation of
-
-[Connection](../interfaces/Connection.md).[dropTable](../interfaces/Connection.md#droptable)
-
-#### Defined in
-
-[index.ts:276](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L276)
-
-___
-
-### openTable
-
-▸ **openTable**(`name`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-Open a table in the database.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-#### Implementation of
-
-[Connection](../interfaces/Connection.md).[openTable](../interfaces/Connection.md#opentable)
-
-#### Defined in
-
-[index.ts:205](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L205)
-
-▸ **openTable**<`T`\>(`name`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-Open a table in the database.
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-#### Implementation of
-
-Connection.openTable
-
-#### Defined in
-
-[index.ts:212](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L212)
-
-▸ **openTable**<`T`\>(`name`, `embeddings?`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` |
-| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-#### Implementation of
-
-Connection.openTable
-
-#### Defined in
-
-[index.ts:213](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L213)
-
-___
-
-### tableNames
-
-▸ **tableNames**(): `Promise`<`string`[]\>
-
-Get the names of all tables in the database.
-
-#### Returns
-
-`Promise`<`string`[]\>
-
-#### Implementation of
-
-[Connection](../interfaces/Connection.md).[tableNames](../interfaces/Connection.md#tablenames)
-
-#### Defined in
-
-[index.ts:196](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L196)

docs/src/javascript/classes/LocalTable.md

@@ -1,302 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / LocalTable
-
-# Class: LocalTable<T\>
-
-A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
-
-## Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-## Implements
-
-- [`Table`](../interfaces/Table.md)<`T`\>
-
-## Table of contents
-
-### Constructors
-
-- [constructor](LocalTable.md#constructor)
-
-### Properties
-
-- [\_embeddings](LocalTable.md#_embeddings)
-- [\_name](LocalTable.md#_name)
-- [\_options](LocalTable.md#_options)
-- [\_tbl](LocalTable.md#_tbl)
-
-### Accessors
-
-- [name](LocalTable.md#name)
-
-### Methods
-
-- [add](LocalTable.md#add)
-- [countRows](LocalTable.md#countrows)
-- [createIndex](LocalTable.md#createindex)
-- [delete](LocalTable.md#delete)
-- [overwrite](LocalTable.md#overwrite)
-- [search](LocalTable.md#search)
-
-## Constructors
-
-### constructor
-
-• **new LocalTable**<`T`\>(`tbl`, `name`, `options`)
-
-#### Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `tbl` | `any` |
-| `name` | `string` |
-| `options` | [`ConnectionOptions`](../interfaces/ConnectionOptions.md) |
-
-#### Defined in
-
-[index.ts:287](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L287)
-
-• **new LocalTable**<`T`\>(`tbl`, `name`, `options`, `embeddings`)
-
-#### Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `tbl` | `any` |  |
-| `name` | `string` |  |
-| `options` | [`ConnectionOptions`](../interfaces/ConnectionOptions.md) |  |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use when interacting with this table |
-
-#### Defined in
-
-[index.ts:294](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L294)
-
-## Properties
-
-### \_embeddings
-
-• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
-
-#### Defined in
-
-[index.ts:284](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L284)
-
-___
-
-### \_name
-
-• `Private` `Readonly` **\_name**: `string`
-
-#### Defined in
-
-[index.ts:283](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L283)
-
-___
-
-### \_options
-
-• `Private` `Readonly` **\_options**: [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
-
-#### Defined in
-
-[index.ts:285](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L285)
-
-___
-
-### \_tbl
-
-• `Private` `Readonly` **\_tbl**: `any`
-
-#### Defined in
-
-[index.ts:282](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L282)
-
-## Accessors
-
-### name
-
-• `get` **name**(): `string`
-
-#### Returns
-
-`string`
-
-#### Implementation of
-
-[Table](../interfaces/Table.md).[name](../interfaces/Table.md#name)
-
-#### Defined in
-
-[index.ts:302](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L302)
-
-## Methods
-
-### add
-
-▸ **add**(`data`): `Promise`<`number`\>
-
-Insert records into this Table.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
-
-#### Returns
-
-`Promise`<`number`\>
-
-The number of rows added to the table
-
-#### Implementation of
-
-[Table](../interfaces/Table.md).[add](../interfaces/Table.md#add)
-
-#### Defined in
-
-[index.ts:320](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L320)
-
-___
-
-### countRows
-
-▸ **countRows**(): `Promise`<`number`\>
-
-Returns the number of rows in this table.
-
-#### Returns
-
-`Promise`<`number`\>
-
-#### Implementation of
-
-[Table](../interfaces/Table.md).[countRows](../interfaces/Table.md#countrows)
-
-#### Defined in
-
-[index.ts:362](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L362)
-
-___
-
-### createIndex
-
-▸ **createIndex**(`indexParams`): `Promise`<`any`\>
-
-Create an ANN index on this Table vector index.
-
-**`See`**
-
-VectorIndexParams.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `indexParams` | [`IvfPQIndexConfig`](../interfaces/IvfPQIndexConfig.md) | The parameters of this Index, |
-
-#### Returns
-
-`Promise`<`any`\>
-
-#### Implementation of
-
-[Table](../interfaces/Table.md).[createIndex](../interfaces/Table.md#createindex)
-
-#### Defined in
-
-[index.ts:355](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L355)
-
-___
-
-### delete
-
-▸ **delete**(`filter`): `Promise`<`void`\>
-
-Delete rows from this table.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `filter` | `string` | A filter in the same format used by a sql WHERE clause. |
-
-#### Returns
-
-`Promise`<`void`\>
-
-#### Implementation of
-
-[Table](../interfaces/Table.md).[delete](../interfaces/Table.md#delete)
-
-#### Defined in
-
-[index.ts:371](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L371)
-
-___
-
-### overwrite
-
-▸ **overwrite**(`data`): `Promise`<`number`\>
-
-Insert records into this Table, replacing its contents.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
-
-#### Returns
-
-`Promise`<`number`\>
-
-The number of rows added to the table
-
-#### Implementation of
-
-[Table](../interfaces/Table.md).[overwrite](../interfaces/Table.md#overwrite)
-
-#### Defined in
-
-[index.ts:338](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L338)
-
-___
-
-### search
-
-▸ **search**(`query`): [`Query`](Query.md)<`T`\>
-
-Creates a search query to find the nearest neighbors of the given search term
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `query` | `T` | The query search term |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Implementation of
-
-[Table](../interfaces/Table.md).[search](../interfaces/Table.md#search)
-
-#### Defined in
-
-[index.ts:310](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L310)

docs/src/javascript/classes/OpenAIEmbeddingFunction.md

@@ -1,105 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / OpenAIEmbeddingFunction
-
-# Class: OpenAIEmbeddingFunction
-
-An embedding function that automatically creates vector representation for a given column.
-
-## Implements
-
-- [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`string`\>
-
-## Table of contents
-
-### Constructors
-
-- [constructor](OpenAIEmbeddingFunction.md#constructor)
-
-### Properties
-
-- [\_modelName](OpenAIEmbeddingFunction.md#_modelname)
-- [\_openai](OpenAIEmbeddingFunction.md#_openai)
-- [sourceColumn](OpenAIEmbeddingFunction.md#sourcecolumn)
-
-### Methods
-
-- [embed](OpenAIEmbeddingFunction.md#embed)
-
-## Constructors
-
-### constructor
-
-• **new OpenAIEmbeddingFunction**(`sourceColumn`, `openAIKey`, `modelName?`)
-
-#### Parameters
-
-| Name | Type | Default value |
-| :------ | :------ | :------ |
-| `sourceColumn` | `string` | `undefined` |
-| `openAIKey` | `string` | `undefined` |
-| `modelName` | `string` | `'text-embedding-ada-002'` |
-
-#### Defined in
-
-[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L21)
-
-## Properties
-
-### \_modelName
-
-• `Private` `Readonly` **\_modelName**: `string`
-
-#### Defined in
-
-[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L19)
-
-___
-
-### \_openai
-
-• `Private` `Readonly` **\_openai**: `any`
-
-#### Defined in
-
-[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L18)
-
-___
-
-### sourceColumn
-
-• **sourceColumn**: `string`
-
-The name of the column that will be used as input for the Embedding Function.
-
-#### Implementation of
-
-[EmbeddingFunction](../interfaces/EmbeddingFunction.md).[sourceColumn](../interfaces/EmbeddingFunction.md#sourcecolumn)
-
-#### Defined in
-
-[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L50)
-
-## Methods
-
-### embed
-
-▸ **embed**(`data`): `Promise`<`number`[][]\>
-
-Creates a vector representation for the given values.
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `data` | `string`[] |
-
-#### Returns
-
-`Promise`<`number`[][]\>
-
-#### Implementation of
-
-[EmbeddingFunction](../interfaces/EmbeddingFunction.md).[embed](../interfaces/EmbeddingFunction.md#embed)
-
-#### Defined in
-
-[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L38)

docs/src/javascript/classes/Query.md

@@ -1,349 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / Query
-
-# Class: Query<T\>
-
-A builder for nearest neighbor queries for LanceDB.
-
-## Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-## Table of contents
-
-### Constructors
-
-- [constructor](Query.md#constructor)
-
-### Properties
-
-- [\_embeddings](Query.md#_embeddings)
-- [\_filter](Query.md#_filter)
-- [\_limit](Query.md#_limit)
-- [\_metricType](Query.md#_metrictype)
-- [\_nprobes](Query.md#_nprobes)
-- [\_query](Query.md#_query)
-- [\_queryVector](Query.md#_queryvector)
-- [\_refineFactor](Query.md#_refinefactor)
-- [\_select](Query.md#_select)
-- [\_tbl](Query.md#_tbl)
-- [where](Query.md#where)
-
-### Methods
-
-- [execute](Query.md#execute)
-- [filter](Query.md#filter)
-- [limit](Query.md#limit)
-- [metricType](Query.md#metrictype)
-- [nprobes](Query.md#nprobes)
-- [refineFactor](Query.md#refinefactor)
-- [select](Query.md#select)
-
-## Constructors
-
-### constructor
-
-• **new Query**<`T`\>(`tbl`, `query`, `embeddings?`)
-
-#### Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `tbl` | `any` |
-| `query` | `T` |
-| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
-
-#### Defined in
-
-[index.ts:448](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L448)
-
-## Properties
-
-### \_embeddings
-
-• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
-
-#### Defined in
-
-[index.ts:446](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L446)
-
-___
-
-### \_filter
-
-• `Private` `Optional` **\_filter**: `string`
-
-#### Defined in
-
-[index.ts:444](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L444)
-
-___
-
-### \_limit
-
-• `Private` **\_limit**: `number`
-
-#### Defined in
-
-[index.ts:440](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L440)
-
-___
-
-### \_metricType
-
-• `Private` `Optional` **\_metricType**: [`MetricType`](../enums/MetricType.md)
-
-#### Defined in
-
-[index.ts:445](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L445)
-
-___
-
-### \_nprobes
-
-• `Private` **\_nprobes**: `number`
-
-#### Defined in
-
-[index.ts:442](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L442)
-
-___
-
-### \_query
-
-• `Private` `Readonly` **\_query**: `T`
-
-#### Defined in
-
-[index.ts:438](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L438)
-
-___
-
-### \_queryVector
-
-• `Private` `Optional` **\_queryVector**: `number`[]
-
-#### Defined in
-
-[index.ts:439](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L439)
-
-___
-
-### \_refineFactor
-
-• `Private` `Optional` **\_refineFactor**: `number`
-
-#### Defined in
-
-[index.ts:441](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L441)
-
-___
-
-### \_select
-
-• `Private` `Optional` **\_select**: `string`[]
-
-#### Defined in
-
-[index.ts:443](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L443)
-
-___
-
-### \_tbl
-
-• `Private` `Readonly` **\_tbl**: `any`
-
-#### Defined in
-
-[index.ts:437](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L437)
-
-___
-
-### where
-
-• **where**: (`value`: `string`) => [`Query`](Query.md)<`T`\>
-
-#### Type declaration
-
-▸ (`value`): [`Query`](Query.md)<`T`\>
-
-A filter statement to be applied to this query.
-
-##### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | `string` | A filter in the same format used by a sql WHERE clause. |
-
-##### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:496](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L496)
-
-## Methods
-
-### execute
-
-▸ **execute**<`T`\>(): `Promise`<`T`[]\>
-
-Execute the query and return the results as an Array of Objects
-
-#### Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `Record`<`string`, `unknown`\> |
-
-#### Returns
-
-`Promise`<`T`[]\>
-
-#### Defined in
-
-[index.ts:519](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L519)
-
-___
-
-### filter
-
-▸ **filter**(`value`): [`Query`](Query.md)<`T`\>
-
-A filter statement to be applied to this query.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | `string` | A filter in the same format used by a sql WHERE clause. |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:491](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L491)
-
-___
-
-### limit
-
-▸ **limit**(`value`): [`Query`](Query.md)<`T`\>
-
-Sets the number of results that will be returned
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | `number` | number of results |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:464](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L464)
-
-___
-
-### metricType
-
-▸ **metricType**(`value`): [`Query`](Query.md)<`T`\>
-
-The MetricType used for this Query.
-
-**`See`**
-
-MetricType for the different options
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | [`MetricType`](../enums/MetricType.md) | The metric to the. |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:511](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L511)
-
-___
-
-### nprobes
-
-▸ **nprobes**(`value`): [`Query`](Query.md)<`T`\>
-
-The number of probes used. A higher number makes search more accurate but also slower.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | `number` | The number of probes used. |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:482](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L482)
-
-___
-
-### refineFactor
-
-▸ **refineFactor**(`value`): [`Query`](Query.md)<`T`\>
-
-Refine the results by reading extra elements and re-ranking them in memory.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | `number` | refine factor to use in this query. |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:473](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L473)
-
-___
-
-### select
-
-▸ **select**(`value`): [`Query`](Query.md)<`T`\>
-
-Return only the specified columns.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | `string`[] | Only select the specified columns. If not specified, all columns will be returned. |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:502](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L502)

docs/src/javascript/enums/MetricType.md

@@ -1,49 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / MetricType
-
-# Enumeration: MetricType
-
-Distance metrics type.
-
-## Table of contents
-
-### Enumeration Members
-
-- [Cosine](MetricType.md#cosine)
-- [Dot](MetricType.md#dot)
-- [L2](MetricType.md#l2)
-
-## Enumeration Members
-
-### Cosine
-
-• **Cosine** = ``"cosine"``
-
-Cosine distance
-
-#### Defined in
-
-[index.ts:567](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L567)
-
-___
-
-### Dot
-
-• **Dot** = ``"dot"``
-
-Dot product
-
-#### Defined in
-
-[index.ts:572](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L572)
-
-___
-
-### L2
-
-• **L2** = ``"l2"``
-
-Euclidean distance
-
-#### Defined in
-
-[index.ts:562](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L562)

docs/src/javascript/enums/WriteMode.md

@@ -1,49 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / WriteMode
-
-# Enumeration: WriteMode
-
-Write mode for writing a table.
-
-## Table of contents
-
-### Enumeration Members
-
-- [Append](WriteMode.md#append)
-- [Create](WriteMode.md#create)
-- [Overwrite](WriteMode.md#overwrite)
-
-## Enumeration Members
-
-### Append
-
-• **Append** = ``"append"``
-
-Append new data to the table.
-
-#### Defined in
-
-[index.ts:552](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L552)
-
-___
-
-### Create
-
-• **Create** = ``"create"``
-
-Create a new [Table](../interfaces/Table.md).
-
-#### Defined in
-
-[index.ts:548](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L548)
-
-___
-
-### Overwrite
-
-• **Overwrite** = ``"overwrite"``
-
-Overwrite the existing [Table](../interfaces/Table.md) if presented.
-
-#### Defined in
-
-[index.ts:550](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L550)

docs/src/javascript/interfaces/AwsCredentials.md

@@ -1,41 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / AwsCredentials
-
-# Interface: AwsCredentials
-
-## Table of contents
-
-### Properties
-
-- [accessKeyId](AwsCredentials.md#accesskeyid)
-- [secretKey](AwsCredentials.md#secretkey)
-- [sessionToken](AwsCredentials.md#sessiontoken)
-
-## Properties
-
-### accessKeyId
-
-• **accessKeyId**: `string`
-
-#### Defined in
-
-[index.ts:31](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L31)
-
-___
-
-### secretKey
-
-• **secretKey**: `string`
-
-#### Defined in
-
-[index.ts:33](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L33)
-
-___
-
-### sessionToken
-
-• `Optional` **sessionToken**: `string`
-
-#### Defined in
-
-[index.ts:35](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L35)

docs/src/javascript/interfaces/Connection.md

@@ -1,152 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / Connection
-
-# Interface: Connection
-
-A LanceDB Connection that allows you to open tables and create new ones.
-
-Connection could be local against filesystem or remote against a server.
-
-## Implemented by
-
-- [`LocalConnection`](../classes/LocalConnection.md)
-
-## Table of contents
-
-### Properties
-
-- [uri](Connection.md#uri)
-
-### Methods
-
-- [createTable](Connection.md#createtable)
-- [createTableArrow](Connection.md#createtablearrow)
-- [dropTable](Connection.md#droptable)
-- [openTable](Connection.md#opentable)
-- [tableNames](Connection.md#tablenames)
-
-## Properties
-
-### uri
-
-• **uri**: `string`
-
-#### Defined in
-
-[index.ts:70](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L70)
-
-## Methods
-
-### createTable
-
-▸ **createTable**<`T`\>(`name`, `data`, `mode?`, `embeddings?`): `Promise`<[`Table`](Table.md)<`T`\>\>
-
-Creates a new Table and initialize it with new data.
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
-| `mode?` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
-| `embeddings?` | [`EmbeddingFunction`](EmbeddingFunction.md)<`T`\> | An embedding function to use on this table |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`T`\>\>
-
-#### Defined in
-
-[index.ts:90](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L90)
-
-___
-
-### createTableArrow
-
-▸ **createTableArrow**(`name`, `table`): `Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` |
-| `table` | `Table`<`any`\> |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-#### Defined in
-
-[index.ts:92](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L92)
-
-___
-
-### dropTable
-
-▸ **dropTable**(`name`): `Promise`<`void`\>
-
-Drop an existing table.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table to drop. |
-
-#### Returns
-
-`Promise`<`void`\>
-
-#### Defined in
-
-[index.ts:98](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L98)
-
-___
-
-### openTable
-
-▸ **openTable**<`T`\>(`name`, `embeddings?`): `Promise`<[`Table`](Table.md)<`T`\>\>
-
-Open a table in the database.
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `embeddings?` | [`EmbeddingFunction`](EmbeddingFunction.md)<`T`\> | An embedding function to use on this table |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`T`\>\>
-
-#### Defined in
-
-[index.ts:80](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L80)
-
-___
-
-### tableNames
-
-▸ **tableNames**(): `Promise`<`string`[]\>
-
-#### Returns
-
-`Promise`<`string`[]\>
-
-#### Defined in
-
-[index.ts:72](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L72)

docs/src/javascript/interfaces/ConnectionOptions.md

@@ -1,30 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / ConnectionOptions
-
-# Interface: ConnectionOptions
-
-## Table of contents
-
-### Properties
-
-- [awsCredentials](ConnectionOptions.md#awscredentials)
-- [uri](ConnectionOptions.md#uri)
-
-## Properties
-
-### awsCredentials
-
-• `Optional` **awsCredentials**: [`AwsCredentials`](AwsCredentials.md)
-
-#### Defined in
-
-[index.ts:40](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L40)
-
-___
-
-### uri
-
-• **uri**: `string`
-
-#### Defined in
-
-[index.ts:39](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L39)

docs/src/javascript/interfaces/EmbeddingFunction.md

@@ -1,60 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / EmbeddingFunction
-
-# Interface: EmbeddingFunction<T\>
-
-An embedding function that automatically creates vector representation for a given column.
-
-## Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-## Implemented by
-
-- [`OpenAIEmbeddingFunction`](../classes/OpenAIEmbeddingFunction.md)
-
-## Table of contents
-
-### Properties
-
-- [embed](EmbeddingFunction.md#embed)
-- [sourceColumn](EmbeddingFunction.md#sourcecolumn)
-
-## Properties
-
-### embed
-
-• **embed**: (`data`: `T`[]) => `Promise`<`number`[][]\>
-
-#### Type declaration
-
-▸ (`data`): `Promise`<`number`[][]\>
-
-Creates a vector representation for the given values.
-
-##### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `data` | `T`[] |
-
-##### Returns
-
-`Promise`<`number`[][]\>
-
-#### Defined in
-
-[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/embedding_function.ts#L27)
-
-___
-
-### sourceColumn
-
-• **sourceColumn**: `string`
-
-The name of the column that will be used as input for the Embedding Function.
-
-#### Defined in
-
-[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/embedding_function.ts#L22)

docs/src/javascript/interfaces/IvfPQIndexConfig.md

@@ -1,149 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / IvfPQIndexConfig
-
-# Interface: IvfPQIndexConfig
-
-## Table of contents
-
-### Properties
-
-- [column](IvfPQIndexConfig.md#column)
-- [index\_name](IvfPQIndexConfig.md#index_name)
-- [max\_iters](IvfPQIndexConfig.md#max_iters)
-- [max\_opq\_iters](IvfPQIndexConfig.md#max_opq_iters)
-- [metric\_type](IvfPQIndexConfig.md#metric_type)
-- [num\_bits](IvfPQIndexConfig.md#num_bits)
-- [num\_partitions](IvfPQIndexConfig.md#num_partitions)
-- [num\_sub\_vectors](IvfPQIndexConfig.md#num_sub_vectors)
-- [replace](IvfPQIndexConfig.md#replace)
-- [type](IvfPQIndexConfig.md#type)
-- [use\_opq](IvfPQIndexConfig.md#use_opq)
-
-## Properties
-
-### column
-
-• `Optional` **column**: `string`
-
-The column to be indexed
-
-#### Defined in
-
-[index.ts:382](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L382)
-
-___
-
-### index\_name
-
-• `Optional` **index\_name**: `string`
-
-A unique name for the index
-
-#### Defined in
-
-[index.ts:387](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L387)
-
-___
-
-### max\_iters
-
-• `Optional` **max\_iters**: `number`
-
-The max number of iterations for kmeans training.
-
-#### Defined in
-
-[index.ts:402](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L402)
-
-___
-
-### max\_opq\_iters
-
-• `Optional` **max\_opq\_iters**: `number`
-
-Max number of iterations to train OPQ, if `use_opq` is true.
-
-#### Defined in
-
-[index.ts:421](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L421)
-
-___
-
-### metric\_type
-
-• `Optional` **metric\_type**: [`MetricType`](../enums/MetricType.md)
-
-Metric type, L2 or Cosine
-
-#### Defined in
-
-[index.ts:392](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L392)
-
-___
-
-### num\_bits
-
-• `Optional` **num\_bits**: `number`
-
-The number of bits to present one PQ centroid.
-
-#### Defined in
-
-[index.ts:416](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L416)
-
-___
-
-### num\_partitions
-
-• `Optional` **num\_partitions**: `number`
-
-The number of partitions this index
-
-#### Defined in
-
-[index.ts:397](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L397)
-
-___
-
-### num\_sub\_vectors
-
-• `Optional` **num\_sub\_vectors**: `number`
-
-Number of subvectors to build PQ code
-
-#### Defined in
-
-[index.ts:412](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L412)
-
-___
-
-### replace
-
-• `Optional` **replace**: `boolean`
-
-Replace an existing index with the same name if it exists.
-
-#### Defined in
-
-[index.ts:426](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L426)
-
-___
-
-### type
-
-• **type**: ``"ivf_pq"``
-
-#### Defined in
-
-[index.ts:428](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L428)
-
-___
-
-### use\_opq
-
-• `Optional` **use\_opq**: `boolean`
-
-Train as optimized product quantization.
-
-#### Defined in
-
-[index.ts:407](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L407)

docs/src/javascript/interfaces/Table.md

@@ -1,221 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / Table
-
-# Interface: Table<T\>
-
-A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
-
-## Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-## Implemented by
-
-- [`LocalTable`](../classes/LocalTable.md)
-
-## Table of contents
-
-### Properties
-
-- [add](Table.md#add)
-- [countRows](Table.md#countrows)
-- [createIndex](Table.md#createindex)
-- [delete](Table.md#delete)
-- [name](Table.md#name)
-- [overwrite](Table.md#overwrite)
-- [search](Table.md#search)
-
-## Properties
-
-### add
-
-• **add**: (`data`: `Record`<`string`, `unknown`\>[]) => `Promise`<`number`\>
-
-#### Type declaration
-
-▸ (`data`): `Promise`<`number`\>
-
-Insert records into this Table.
-
-##### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
-
-##### Returns
-
-`Promise`<`number`\>
-
-The number of rows added to the table
-
-#### Defined in
-
-[index.ts:120](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L120)
-
-___
-
-### countRows
-
-• **countRows**: () => `Promise`<`number`\>
-
-#### Type declaration
-
-▸ (): `Promise`<`number`\>
-
-Returns the number of rows in this table.
-
-##### Returns
-
-`Promise`<`number`\>
-
-#### Defined in
-
-[index.ts:140](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L140)
-
-___
-
-### createIndex
-
-• **createIndex**: (`indexParams`: [`IvfPQIndexConfig`](IvfPQIndexConfig.md)) => `Promise`<`any`\>
-
-#### Type declaration
-
-▸ (`indexParams`): `Promise`<`any`\>
-
-Create an ANN index on this Table vector index.
-
-**`See`**
-
-VectorIndexParams.
-
-##### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `indexParams` | [`IvfPQIndexConfig`](IvfPQIndexConfig.md) | The parameters of this Index, |
-
-##### Returns
-
-`Promise`<`any`\>
-
-#### Defined in
-
-[index.ts:135](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L135)
-
-___
-
-### delete
-
-• **delete**: (`filter`: `string`) => `Promise`<`void`\>
-
-#### Type declaration
-
-▸ (`filter`): `Promise`<`void`\>
-
-Delete rows from this table.
-
-This can be used to delete a single row, many rows, all rows, or
-sometimes no rows (if your predicate matches nothing).
-
-**`Examples`**
-
-```ts
-const con = await lancedb.connect("./.lancedb")
-const data = [
-   {id: 1, vector: [1, 2]},
-   {id: 2, vector: [3, 4]},
-   {id: 3, vector: [5, 6]},
-];
-const tbl = await con.createTable("my_table", data)
-await tbl.delete("id = 2")
-await tbl.countRows() // Returns 2
-```
-
-If you have a list of values to delete, you can combine them into a
-stringified list and use the `IN` operator:
-
-```ts
-const to_remove = [1, 5];
-await tbl.delete(`id IN (${to_remove.join(",")})`)
-await tbl.countRows() // Returns 1
-```
-
-##### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `filter` | `string` | A filter in the same format used by a sql WHERE clause. The filter must not be empty. |
-
-##### Returns
-
-`Promise`<`void`\>
-
-#### Defined in
-
-[index.ts:174](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L174)
-
-___
-
-### name
-
-• **name**: `string`
-
-#### Defined in
-
-[index.ts:106](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L106)
-
-___
-
-### overwrite
-
-• **overwrite**: (`data`: `Record`<`string`, `unknown`\>[]) => `Promise`<`number`\>
-
-#### Type declaration
-
-▸ (`data`): `Promise`<`number`\>
-
-Insert records into this Table, replacing its contents.
-
-##### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
-
-##### Returns
-
-`Promise`<`number`\>
-
-The number of rows added to the table
-
-#### Defined in
-
-[index.ts:128](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L128)
-
-___
-
-### search
-
-• **search**: (`query`: `T`) => [`Query`](../classes/Query.md)<`T`\>
-
-#### Type declaration
-
-▸ (`query`): [`Query`](../classes/Query.md)<`T`\>
-
-Creates a search query to find the nearest neighbors of the given search term
-
-##### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `query` | `T` | The query search term |
-
-##### Returns
-
-[`Query`](../classes/Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:112](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L112)

docs/src/javascript/modules.md

@@ -1,82 +0,0 @@
-[vectordb](README.md) / Exports
-
-# vectordb
-
-## Table of contents
-
-### Enumerations
-
-- [MetricType](enums/MetricType.md)
-- [WriteMode](enums/WriteMode.md)
-
-### Classes
-
-- [LocalConnection](classes/LocalConnection.md)
-- [LocalTable](classes/LocalTable.md)
-- [OpenAIEmbeddingFunction](classes/OpenAIEmbeddingFunction.md)
-- [Query](classes/Query.md)
-
-### Interfaces
-
-- [AwsCredentials](interfaces/AwsCredentials.md)
-- [Connection](interfaces/Connection.md)
-- [ConnectionOptions](interfaces/ConnectionOptions.md)
-- [EmbeddingFunction](interfaces/EmbeddingFunction.md)
-- [IvfPQIndexConfig](interfaces/IvfPQIndexConfig.md)
-- [Table](interfaces/Table.md)
-
-### Type Aliases
-
-- [VectorIndexParams](modules.md#vectorindexparams)
-
-### Functions
-
-- [connect](modules.md#connect)
-
-## Type Aliases
-
-### VectorIndexParams
-
-Ƭ **VectorIndexParams**: [`IvfPQIndexConfig`](interfaces/IvfPQIndexConfig.md)
-
-#### Defined in
-
-[index.ts:431](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L431)
-
-## Functions
-
-### connect
-
-▸ **connect**(`uri`): `Promise`<[`Connection`](interfaces/Connection.md)\>
-
-Connect to a LanceDB instance at the given URI
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `uri` | `string` | The uri of the database. |
-
-#### Returns
-
-`Promise`<[`Connection`](interfaces/Connection.md)\>
-
-#### Defined in
-
-[index.ts:47](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L47)
-
-▸ **connect**(`opts`): `Promise`<[`Connection`](interfaces/Connection.md)\>
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `opts` | `Partial`<[`ConnectionOptions`](interfaces/ConnectionOptions.md)\> |
-
-#### Returns
-
-`Promise`<[`Connection`](interfaces/Connection.md)\>
-
-#### Defined in
-
-[index.ts:48](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L48)

docs/src/notebooks/diffusiondb/datagen.py

@@ -1,109 +0,0 @@
-#!/usr/bin/env python
-#
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-"""Dataset hf://poloclub/diffusiondb
-"""
-
-import io
-from argparse import ArgumentParser
-from multiprocessing import Pool
-
-import lance
-import pyarrow as pa
-from datasets import load_dataset
-from PIL import Image
-from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast
-
-import lancedb
-
-MODEL_ID = "openai/clip-vit-base-patch32"
-
-device = "cuda"
-
-tokenizer = CLIPTokenizerFast.from_pretrained(MODEL_ID)
-model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32").to(device)
-processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
-
-schema = pa.schema(
-    [
-        pa.field("prompt", pa.string()),
-        pa.field("seed", pa.uint32()),
-        pa.field("step", pa.uint16()),
-        pa.field("cfg", pa.float32()),
-        pa.field("sampler", pa.string()),
-        pa.field("width", pa.uint16()),
-        pa.field("height", pa.uint16()),
-        pa.field("timestamp", pa.timestamp("s")),
-        pa.field("image_nsfw", pa.float32()),
-        pa.field("prompt_nsfw", pa.float32()),
-        pa.field("vector", pa.list_(pa.float32(), 512)),
-        pa.field("image", pa.binary()),
-    ]
-)
-
-
-def pil_to_bytes(img) -> list[bytes]:
-    buf = io.BytesIO()
-    img.save(buf, format="PNG")
-    return buf.getvalue()
-
-
-def generate_clip_embeddings(batch) -> pa.RecordBatch:
-    image = processor(text=None, images=batch["image"], return_tensors="pt")[
-        "pixel_values"
-    ].to(device)
-    img_emb = model.get_image_features(image)
-    batch["vector"] = img_emb.cpu().tolist()
-
-    with Pool() as p:
-        batch["image_bytes"] = p.map(pil_to_bytes, batch["image"])
-    return batch
-
-
-def datagen(args):
-    """Generate DiffusionDB dataset, and use CLIP model to generate image embeddings."""
-    dataset = load_dataset("poloclub/diffusiondb", args.subset)
-    data = []
-    for b in dataset.map(
-        generate_clip_embeddings, batched=True, batch_size=256, remove_columns=["image"]
-    )["train"]:
-        b["image"] = b["image_bytes"]
-        del b["image_bytes"]
-        data.append(b)
-    tbl = pa.Table.from_pylist(data, schema=schema)
-    return tbl
-
-
-def main():
-    parser = ArgumentParser()
-    parser.add_argument(
-        "-o", "--output", metavar="DIR", help="Output lance directory", required=True
-    )
-    parser.add_argument(
-        "-s",
-        "--subset",
-        choices=["2m_all", "2m_first_10k", "2m_first_100k"],
-        default="2m_first_10k",
-        help="subset of the hg dataset",
-    )
-
-    args = parser.parse_args()
-
-    batches = datagen(args)
-    lance.write_dataset(batches, args.output)
-
-
-if __name__ == "__main__":
-    main()

docs/src/notebooks/diffusiondb/requirements.txt

@@ -1,9 +0,0 @@
-datasets
-Pillow
-lancedb
-isort
-black
-transformers
---index-url https://download.pytorch.org/whl/cu118
-torch
-torchvision

docs/src/notebooks/multimodal_search.ipynb

@@ -1,269 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
-      "\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
-     ]
-    }
-   ],
-   "source": [
-    "!pip install --quiet -U lancedb\n",
-    "!pip install --quiet gradio transformers torch torchvision"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import io\n",
-    "import PIL\n",
-    "import duckdb\n",
-    "import lancedb"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## First run setup: Download data and pre-process"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 30,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "<lance.dataset.LanceDataset at 0x3045db590>"
-      ]
-     },
-     "execution_count": 30,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "# remove null prompts\n",
-    "import lance\n",
-    "import pyarrow.compute as pc\n",
-    "\n",
-    "# download s3://eto-public/datasets/diffusiondb/small_10k.lance to this uri\n",
-    "data = lance.dataset(\"~/datasets/rawdata.lance\").to_table()\n",
-    "\n",
-    "# First data processing and full-text-search index\n",
-    "db = lancedb.connect(\"~/datasets/demo\")\n",
-    "tbl = db.create_table(\"diffusiondb\", data.filter(~pc.field(\"prompt\").is_null()))\n",
-    "tbl = tbl.create_fts_index([\"prompt\"])"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Create / Open LanceDB Table"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "db = lancedb.connect(\"~/datasets/demo\")\n",
-    "tbl = db.open_table(\"diffusiondb\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Create CLIP embedding function for the text"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast\n",
-    "\n",
-    "MODEL_ID = \"openai/clip-vit-base-patch32\"\n",
-    "\n",
-    "tokenizer = CLIPTokenizerFast.from_pretrained(MODEL_ID)\n",
-    "model = CLIPModel.from_pretrained(MODEL_ID)\n",
-    "processor = CLIPProcessor.from_pretrained(MODEL_ID)\n",
-    "\n",
-    "def embed_func(query):\n",
-    "    inputs = tokenizer([query], padding=True, return_tensors=\"pt\")\n",
-    "    text_features = model.get_text_features(**inputs)\n",
-    "    return text_features.detach().numpy()[0]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Search functions for Gradio"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def find_image_vectors(query):\n",
-    "    emb = embed_func(query)\n",
-    "    code = (\n",
-    "        \"import lancedb\\n\"\n",
-    "        \"db = lancedb.connect('~/datasets/demo')\\n\"\n",
-    "        \"tbl = db.open_table('diffusiondb')\\n\\n\"\n",
-    "        f\"embedding = embed_func('{query}')\\n\"\n",
-    "        \"tbl.search(embedding).limit(9).to_df()\"\n",
-    "    )\n",
-    "    return (_extract(tbl.search(emb).limit(9).to_df()), code)\n",
-    "\n",
-    "def find_image_keywords(query):\n",
-    "    code = (\n",
-    "        \"import lancedb\\n\"\n",
-    "        \"db = lancedb.connect('~/datasets/demo')\\n\"\n",
-    "        \"tbl = db.open_table('diffusiondb')\\n\\n\"\n",
-    "        f\"tbl.search('{query}').limit(9).to_df()\"\n",
-    "    )\n",
-    "    return (_extract(tbl.search(query).limit(9).to_df()), code)\n",
-    "\n",
-    "def find_image_sql(query):\n",
-    "    code = (\n",
-    "        \"import lancedb\\n\"\n",
-    "        \"import duckdb\\n\"\n",
-    "        \"db = lancedb.connect('~/datasets/demo')\\n\"\n",
-    "        \"tbl = db.open_table('diffusiondb')\\n\\n\"\n",
-    "        \"diffusiondb = tbl.to_lance()\\n\"\n",
-    "        f\"duckdb.sql('{query}').to_df()\"\n",
-    "    )    \n",
-    "    diffusiondb = tbl.to_lance()\n",
-    "    return (_extract(duckdb.sql(query).to_df()), code)\n",
-    "\n",
-    "def _extract(df):\n",
-    "    image_col = \"image\"\n",
-    "    return [(PIL.Image.open(io.BytesIO(row[image_col])), row[\"prompt\"]) for _, row in df.iterrows()]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Setup Gradio interface"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 28,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Running on local URL:  http://127.0.0.1:7881\n",
-      "\n",
-      "To create a public link, set `share=True` in `launch()`.\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<div><iframe src=\"http://127.0.0.1:7881/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
-      ],
-      "text/plain": [
-       "<IPython.core.display.HTML object>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/plain": []
-     },
-     "execution_count": 28,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import gradio as gr\n",
-    "\n",
-    "\n",
-    "with gr.Blocks() as demo:\n",
-    "    with gr.Row():\n",
-    "        with gr.Tab(\"Embeddings\"):\n",
-    "            vector_query = gr.Textbox(value=\"portraits of a person\", show_label=False)\n",
-    "            b1 = gr.Button(\"Submit\")\n",
-    "        with gr.Tab(\"Keywords\"):\n",
-    "            keyword_query = gr.Textbox(value=\"ninja turtle\", show_label=False)\n",
-    "            b2 = gr.Button(\"Submit\")\n",
-    "        with gr.Tab(\"SQL\"):\n",
-    "            sql_query = gr.Textbox(value=\"SELECT * from diffusiondb WHERE image_nsfw >= 2 LIMIT 9\", show_label=False)\n",
-    "            b3 = gr.Button(\"Submit\")\n",
-    "    with gr.Row():\n",
-    "        code = gr.Code(label=\"Code\", language=\"python\")\n",
-    "    with gr.Row():\n",
-    "        gallery = gr.Gallery(\n",
-    "                label=\"Found images\", show_label=False, elem_id=\"gallery\"\n",
-    "            ).style(columns=[3], rows=[3], object_fit=\"contain\", height=\"auto\")   \n",
-    "        \n",
-    "    b1.click(find_image_vectors, inputs=vector_query, outputs=[gallery, code])\n",
-    "    b2.click(find_image_keywords, inputs=keyword_query, outputs=[gallery, code])\n",
-    "    b3.click(find_image_sql, inputs=sql_query, outputs=[gallery, code])\n",
-    "    \n",
-    "demo.launch()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.3"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 1
-}

docs/src/python.md

@@ -0,0 +1,14 @@
+# LanceDB Python API Reference
+
+## Installation
+
+```shell
+pip install lancedb
+```
+
+## ::: lancedb
+## ::: lancedb.db
+## ::: lancedb.table
+## ::: lancedb.query
+## ::: lancedb.embeddings
+## ::: lancedb.context

docs/src/python/arrow.md

@@ -1,101 +0,0 @@
-# Pandas and PyArrow
-
-
-Built on top of [Apache Arrow](https://arrow.apache.org/),
-`LanceDB` is easy to integrate with the Python ecosystem, including [Pandas](https://pandas.pydata.org/)
-and PyArrow.
-
-## Create dataset
-
-First, we need to connect to a `LanceDB` database.
-
-```py
-
-import lancedb
-
-db = lancedb.connect("data/sample-lancedb")
-```
-
-Afterwards, we write a `Pandas DataFrame` to LanceDB directly.
-
-```py
-import pandas as pd
-
-data = pd.DataFrame({
-    "vector": [[3.1, 4.1], [5.9, 26.5]],
-    "item": ["foo", "bar"],
-    "price": [10.0, 20.0]
-})
-table = db.create_table("pd_table", data=data)
-```
-
-Similar to [`pyarrow.write_dataset()`](https://arrow.apache.org/docs/python/generated/pyarrow.dataset.write_dataset.html),
-[db.create_table()](../python/#lancedb.db.DBConnection.create_table) accepts a wide-range of forms of data.
-
-For example, if you have a dataset that is larger than memory size, you can create table with `Iterator[pyarrow.RecordBatch]`,
-to lazily generate data:
-
-```py
-
-from typing import Iterable
-import pyarrow as pa
-import lancedb
-
-def make_batches() -> Iterable[pa.RecordBatch]:
-    for i in range(5):
-        yield pa.RecordBatch.from_arrays(
-            [
-                pa.array([[3.1, 4.1], [5.9, 26.5]]),
-                pa.array(["foo", "bar"]),
-                pa.array([10.0, 20.0]),
-            ],
-            ["vector", "item", "price"])
-
-schema=pa.schema([
-    pa.field("vector", pa.list_(pa.float32())),
-    pa.field("item", pa.utf8()),
-    pa.field("price", pa.float32()),
-])
-
-table = db.create_table("iterable_table", data=make_batches(), schema=schema)
-```
-
-You will find detailed instructions of creating dataset in
-[Basic Operations](../basic.md) and [API](../python/#lancedb.db.DBConnection.create_table)
-sections.
-
-## Vector Search
-
-We can now perform similarity search via `LanceDB` Python API.
-
-```py
-# Open the table previously created.
-table = db.open_table("pd_table")
-
-query_vector = [100, 100]
-# Pandas DataFrame
-df = table.search(query_vector).limit(1).to_df()
-print(df)
-```
-
-```
-    vector     item  price        score
-0  [5.9, 26.5]  bar   20.0  14257.05957
-```
-
-If you have a simple filter, it's faster to provide a `where clause` to `LanceDB`'s search query.
-If you have more complex criteria, you can always apply the filter to the resulting Pandas `DataFrame`.
-
-```python
-
-# Apply the filter via LanceDB
-results = table.search([100, 100]).where("price < 15").to_df()
-assert len(results) == 1
-assert results["item"].iloc[0] == "foo"
-
-# Apply the filter via Pandas
-df = results = table.search([100, 100]).to_df()
-results = df[df.price < 15]
-assert len(results) == 1
-assert results["item"].iloc[0] == "foo"
-```

docs/src/python/duckdb.md

@@ -1,56 +0,0 @@
-# DuckDB
-
-`LanceDB` works with `DuckDB` via [PyArrow integration](https://duckdb.org/docs/guides/python/sql_on_arrow).
-
-Let us start with installing `duckdb` and `lancedb`.
-
-```shell
-pip install duckdb lancedb
-```
-
-We will re-use [the dataset created previously](./arrow.md):
-
-```python
-import pandas as pd
-import lancedb
-
-db = lancedb.connect("data/sample-lancedb")
-data = pd.DataFrame({
-    "vector": [[3.1, 4.1], [5.9, 26.5]],
-    "item": ["foo", "bar"],
-    "price": [10.0, 20.0]
-})
-table = db.create_table("pd_table", data=data)
-arrow_table = table.to_arrow()
-```
-
-`DuckDB` can directly query the `arrow_table`:
-
-```python
-import duckdb
-
-duckdb.query("SELECT * FROM arrow_table")
-```
-
-```
-┌─────────────┬─────────┬────────┐
-│   vector    │  item   │ price  │
-│   float[]   │ varchar │ double │
-├─────────────┼─────────┼────────┤
-│ [3.1, 4.1]  │ foo     │   10.0 │
-│ [5.9, 26.5] │ bar     │   20.0 │
-└─────────────┴─────────┴────────┘
-```
-
-```py
-duckdb.query("SELECT mean(price) FROM arrow_table")
-```
-
-```
-┌─────────────┐
-│ mean(price) │
-│   double    │
-├─────────────┤
-│        15.0 │
-└─────────────┘
-```

docs/src/python/integration.md

@@ -1,7 +0,0 @@
-# Integration
-
-Built on top of [Apache Arrow](https://arrow.apache.org/),
-`LanceDB` is very easy to be integrate with Python ecosystems.
-
-* [Pandas and Arrow Integration](./arrow.md)
-* [DuckDB Integration](./duckdb.md)

docs/src/python/pydantic.md

@@ -1,35 +0,0 @@
-# Pydantic
-
-[Pydantic](https://docs.pydantic.dev/latest/) is a data validation library in Python.
-
-## Schema
-
-LanceDB supports to create Apache Arrow Schema from a
-[Pydantic BaseModel](https://docs.pydantic.dev/latest/api/main/#pydantic.main.BaseModel)
-via [pydantic_to_schema()](python.md##lancedb.pydantic.pydantic_to_schema) method.
-
-::: lancedb.pydantic.pydantic_to_schema
-
-## Vector Field
-
-LanceDB provides a [`vector(dim)`](python.md#lancedb.pydantic.vector) method to define a
-vector Field in a Pydantic Model.
-
-::: lancedb.pydantic.vector
-
-## Type Conversion
-
-LanceDB automatically convert Pydantic fields to
-[Apache Arrow DataType](https://arrow.apache.org/docs/python/generated/pyarrow.DataType.html#pyarrow.DataType).
-
-Current supported type conversions:
-
-| Pydantic Field Type | PyArrow Data Type |
-| ------------------- | ----------------- |
-| `int`               | `pyarrow.int64`   |
-| `float`              | `pyarrow.float64`  |
-| `bool`              | `pyarrow.bool`    |
-| `str`               | `pyarrow.utf8()`    |
-| `list`              | `pyarrow.List`    |
-| `BaseModel`         | `pyarrow.Struct`    |
-| `vector(n)`         | `pyarrow.FixedSizeList(float32, n)` |

docs/src/python/python.md

@@ -1,59 +0,0 @@
-# LanceDB Python API Reference
-
-## Installation
-
-```shell
-pip install lancedb
-```
-
-## Connection
-
-::: lancedb.connect
-
-::: lancedb.db.DBConnection
-
-## Table
-
-::: lancedb.table.Table
-
-## Querying
-
-::: lancedb.query.Query
-
-::: lancedb.query.LanceQueryBuilder
-
-::: lancedb.query.LanceFtsQueryBuilder
-
-## Embeddings
-
-::: lancedb.embeddings.with_embeddings
-
-::: lancedb.embeddings.EmbeddingFunction
-
-## Context
-
-::: lancedb.context.contextualize
-
-::: lancedb.context.Contextualizer
-
-## Full text search
-
-::: lancedb.fts.create_index
-
-::: lancedb.fts.populate_index
-
-::: lancedb.fts.search_index
-
-## Utilities
-
-::: lancedb.vector
-
-## Integrations
-
-### Pydantic
-
-::: lancedb.pydantic.pydantic_to_schema
-
-::: lancedb.pydantic.vector
-
-

docs/src/search.md

@@ -1,121 +0,0 @@
-# Vector Search
-
-`Vector Search` finds the nearest vectors from the database.
-In a recommendation system or search engine, you can find similar products from
-the one you searched.
-In LLM and other AI applications,
-each data point can be [presented by the embeddings generated from some models](embedding.md),
-it returns the most relevant features.
-
-A search in high-dimensional vector space, is to find `K-Nearest-Neighbors (KNN)` of the query vector.
-
-## Metric
-
-In LanceDB, a `Metric` is the way to describe the distance between a pair of vectors.
-Currently, we support the following metrics:
-
-| Metric      | Description                          |
-| ----------- | ------------------------------------ |
-| `L2`        | [Euclidean / L2 distance](https://en.wikipedia.org/wiki/Euclidean_distance) |
-| `Cosine`    | [Cosine Similarity](https://en.wikipedia.org/wiki/Cosine_similarity)|
-| `Dot`       | [Dot Production](https://en.wikipedia.org/wiki/Dot_product) |
-
-
-## Search
-
-### Flat Search
-
-If LanceDB does not create a vector index, LanceDB would need to scan (`Flat Search`) the entire vector column
-and compute the distance for each vector in order to find the closest matches.
-
-
-<!-- Setup Code
-```python
-import lancedb
-import numpy as np
-uri = "data/sample-lancedb"
-db = lancedb.connect(uri)
-
-data = [{"vector": row, "item": f"item {i}"}
-     for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]
-
-db.create_table("my_vectors", data=data)
-```
--->
-<!-- Setup Code
-```javascript
-const vectordb_setup = require('vectordb')
-const db_setup = await vectordb_setup.connect('data/sample-lancedb')
-
-let data = []
-for (let i = 0; i < 10_000; i++) {
-     data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
-}
-await db_setup.createTable('my_vectors', data)
-```
--->
-=== "Python"
-
-
-    ```python
-    import lancedb
-    import numpy as np
-
-    db = lancedb.connect("data/sample-lancedb")
-
-    tbl = db.open_table("my_vectors")
-
-    df = tbl.search(np.random.random((1536))) \
-        .limit(10) \
-        .to_df()
-    ```
-
-=== "JavaScript"
-
-    ```javascript
-    const vectordb = require('vectordb')
-    const db = await vectordb.connect('data/sample-lancedb')
-
-    const tbl = await db.openTable("my_vectors")
-
-    const results_1 = await tbl.search(Array(1536).fill(1.2))
-        .limit(10)
-        .execute()
-    ```
-
-By default, `l2` will be used as `Metric` type. You can customize the metric type
-as well.
-
-=== "Python"
-
-    ```python
-    df = tbl.search(np.random.random((1536))) \
-        .metric("cosine") \
-        .limit(10) \
-        .to_df()
-    ```
-
-
-=== "JavaScript"
-
-    ```javascript
-    const results_2 = await tbl.search(Array(1536).fill(1.2))
-        .metricType("cosine")
-        .limit(10)
-        .execute()
-    ```
-
-
-### Approximate Nearest Neighbor (ANN) Search with Vector Index.
-
-To accelerate vector retrievals, it is common to build vector indices.
-A vector index is a data structure specifically designed to efficiently organize and
-search vector data based on their similarity or distance metrics.
-By constructing a vector index, you can reduce the search space and avoid the need
-for brute-force scanning of the entire vector column.
-
-However, fast vector search using indices often entails making a trade-off with accuracy to some extent.
-This is why it is often called **Approximate Nearest Neighbors (ANN)** search, while the Flat Search (KNN)
-always returns 100% recall.
-
-See [ANN Index](ann_indexes.md) for more details.

docs/src/sql.md

@@ -1,120 +0,0 @@
-# SQL filters
-
-LanceDB embraces the utilization of standard SQL expressions as predicates for hybrid
-filters. It can be used during hybrid vector search and deletion operations.
-
-Currently, Lance supports a growing list of expressions.
-
-* ``>``, ``>=``, ``<``, ``<=``, ``=``
-* ``AND``, ``OR``, ``NOT``
-* ``IS NULL``, ``IS NOT NULL``
-* ``IS TRUE``, ``IS NOT TRUE``, ``IS FALSE``, ``IS NOT FALSE``
-* ``IN``
-* ``LIKE``, ``NOT LIKE``
-* ``CAST``
-* ``regexp_match(column, pattern)``
-
-For example, the following filter string is acceptable:
-<!-- Setup Code
-```python 
-import lancedb
-import numpy as np
-uri = "data/sample-lancedb"
-db = lancedb.connect(uri)
-
-data = [{"vector": row, "item": f"item {i}"}
-     for i, row in enumerate(np.random.random((10_000, 2)).astype('int'))]
-
-tbl = db.create_table("my_vectors", data=data)
-```
--->
-<!-- Setup Code
-```javascript 
-const vectordb = require('vectordb')
-const db = await vectordb.connect('data/sample-lancedb')
-
-let data = []
-for (let i = 0; i < 10_000; i++) {
-     data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
-}
-const tbl = await db.createTable('my_vectors', data)
-```
--->
-=== "Python"
-
-    ```python
-    tbl.search([100, 102]) \
-       .where("""(
-        (label IN [10, 20])
-        AND
-        (note.email IS NOT NULL)
-        ) OR NOT note.created
-        """)
-
-    ```
-=== "Javascript"
-
-    ```javascript
-    tbl.search([100, 102])
-       .where(`(
-            (label IN [10, 20])
-            AND
-            (note.email IS NOT NULL)
-        ) OR NOT note.created
-       `)
-    ```
-
-
-If your column name contains special characters or is a [SQL Keyword](https://docs.rs/sqlparser/latest/sqlparser/keywords/index.html),
-you can use backtick (`` ` ``) to escape it. For nested fields, each segment of the
-path must be wrapped in backticks.
-
-=== "SQL"
-    ```sql
-    `CUBE` = 10 AND `column name with space` IS NOT NULL
-      AND `nested with space`.`inner with space` < 2
-    ```
-
-!!! warning
-    Field names containing periods (``.``) are not supported.
-
-Literals for dates, timestamps, and decimals can be written by writing the string
-value after the type name. For example
-
-=== "SQL"
-    ```sql
-    date_col = date '2021-01-01'
-    and timestamp_col = timestamp '2021-01-01 00:00:00'
-    and decimal_col = decimal(8,3) '1.000'
-    ```
-
-For timestamp columns, the precision can be specified as a number in the type
-parameter. Microsecond precision (6) is the default.
-
-| SQL              | Time unit    |
-|------------------|--------------|
-| ``timestamp(0)`` | Seconds      |
-| ``timestamp(3)`` | Milliseconds |
-| ``timestamp(6)`` | Microseconds |
-| ``timestamp(9)`` | Nanoseconds  |
-
-LanceDB internally stores data in [Apache Arrow](https://arrow.apache.org/) format.
-The mapping from SQL types to Arrow types is:
-
-| SQL type | Arrow type |
-|----------|------------|
-| ``boolean`` | ``Boolean`` |
-| ``tinyint`` / ``tinyint unsigned`` | ``Int8`` / ``UInt8`` |
-| ``smallint`` / ``smallint unsigned`` | ``Int16`` / ``UInt16`` |
-| ``int`` or ``integer`` / ``int unsigned`` or ``integer unsigned`` | ``Int32`` / ``UInt32`` |
-| ``bigint`` / ``bigint unsigned`` | ``Int64`` / ``UInt64`` |
-| ``float`` | ``Float32`` |
-| ``double`` | ``Float64`` |
-| ``decimal(precision, scale)`` | ``Decimal128`` |
-| ``date`` | ``Date32`` |
-| ``timestamp`` | ``Timestamp`` [^1] |
-| ``string`` | ``Utf8`` |
-| ``binary`` | ``Binary`` |
-
-[^1]: See precision mapping in previous table.
-

docs/src/styles/global.css

@@ -1,6 +0,0 @@
-:root {
-    --md-primary-fg-color:        #625eff;
-    --md-primary-fg-color--dark:  #4338ca;
-    --md-text-font: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
-    --md-code-font: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
-}

docs/test/md_testing.js

@@ -1,52 +0,0 @@
-const glob = require("glob");
-const fs = require("fs");
-const path = require("path");
-
-const excludedFiles = [
-  "../src/fts.md",
-  "../src/embedding.md",
-  "../src/examples/serverless_lancedb_with_s3_and_lambda.md",
-  "../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
-  "../src/examples/transformerjs_embedding_search_nodejs.md",
-  "../src/examples/youtube_transcript_bot_with_nodejs.md",
-];
-const nodePrefix = "javascript";
-const nodeFile = ".js";
-const nodeFolder = "node";
-const globString = "../src/**/*.md";
-const asyncPrefix = "(async () => {\n";
-const asyncSuffix = "})();";
-
-function* yieldLines(lines, prefix, suffix) {
-  let inCodeBlock = false;
-  for (const line of lines) {
-    if (line.trim().startsWith(prefix + nodePrefix)) {
-      inCodeBlock = true;
-    } else if (inCodeBlock && line.trim().startsWith(suffix)) {
-      inCodeBlock = false;
-      yield "\n";
-    } else if (inCodeBlock) {
-      yield line;
-    }
-  }
-}
-
-const files = glob.sync(globString, { recursive: true });
-
-for (const file of files.filter((file) => !excludedFiles.includes(file))) {
-  const lines = [];
-  const data = fs.readFileSync(file, "utf-8");
-  const fileLines = data.split("\n");
-
-  for (const line of yieldLines(fileLines, "```", "```")) {
-    lines.push(line);
-  }
-
-  if (lines.length > 0) {
-    const fileName = path.basename(file, ".md");
-    const outPath = path.join(nodeFolder, fileName, `${fileName}${nodeFile}`);
-    console.log(outPath)
-    fs.mkdirSync(path.dirname(outPath), { recursive: true });
-    fs.writeFileSync(outPath, asyncPrefix + "\n" + lines.join("\n") + asyncSuffix);
-  }
-}

docs/test/md_testing.py

@@ -1,41 +0,0 @@
-import glob
-from typing import Iterator
-from pathlib import Path
-
-excluded_files = [
-    "../src/fts.md",
-    "../src/embedding.md",
-    "../src/examples/serverless_lancedb_with_s3_and_lambda.md",
-    "../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
-    "../src/examples/youtube_transcript_bot_with_nodejs.md"
-]
-
-python_prefix = "py"
-python_file = ".py"
-python_folder = "python"
-glob_string = "../src/**/*.md"
-
-def yield_lines(lines: Iterator[str], prefix: str, suffix: str):
-    in_code_block = False
-    # Python code has strict indentation
-    strip_length = 0
-    for line in lines:
-        if line.strip().startswith(prefix + python_prefix):
-            in_code_block = True
-            strip_length = len(line) - len(line.lstrip())
-        elif in_code_block and line.strip().startswith(suffix):
-            in_code_block = False
-            yield "\n"
-        elif in_code_block:
-            yield line[strip_length:]
-
-for file in filter(lambda file: file not in excluded_files, glob.glob(glob_string, recursive=True)):
-    with open(file, "r") as f:
-        lines = list(yield_lines(iter(f), "```", "```"))
-
-    if len(lines) > 0:
-        out_path = Path(python_folder) / Path(file).name.strip(".md") / (Path(file).name.strip(".md") + python_file)
-        print(out_path)
-        out_path.parent.mkdir(exist_ok=True, parents=True)
-        with open(out_path, "w") as out:
-            out.writelines(lines)

docs/test/package.json

@@ -1,13 +0,0 @@
-{
-  "name": "lancedb-docs-test",
-  "version": "1.0.0",
-  "description": "",
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "fs": "^0.0.1-security",
-    "glob": "^10.2.7",
-    "path": "^0.12.7",
-    "vectordb": "https://gitpkg.now.sh/lancedb/lancedb/node?main"
-  }
-}

docs/test/requirements.txt

@@ -1,5 +0,0 @@
-lancedb @ git+https://github.com/lancedb/lancedb.git#egg=subdir&subdirectory=python
-numpy
-pandas
-pylance
-duckdb

node/.eslintrc.js

@@ -12,6 +12,5 @@ module.exports = {
     sourceType: 'module'
   },
   rules: {
-    "@typescript-eslint/method-signature-style": "off",
   }
 }

node/.npmignore

@@ -1,4 +1,2 @@
 gen_test_data.py
 index.node
-dist/lancedb*.tgz
-vectordb*.tgz

node/CHANGELOG.md

@@ -1,64 +0,0 @@
-# Changelog
-
-All notable changes to this project will be documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-## [0.1.5] - 2023-06-00
-
-### Added
-
-- Support for macOS X86
-
-## [0.1.4] - 2023-06-03
-
-### Added
-
-- Select / Project query API
-
-### Changed
-
--  Deprecated created_index in favor of createIndex
-
-## [0.1.3] - 2023-06-01
-
-### Added
-
-- Support S3 and Google Cloud Storage
-- Embedding functions support
-- OpenAI embedding function
-
-## [0.1.2] - 2023-05-27
-
-### Added
-
-- Append records API
-- Extra query params to to nodejs client
-- Create_index API
- 
-### Fixed
-
-- bugfix: string columns should be converted to Utf8Array (#94)
-
-## [0.1.1] - 2023-05-16
-
-### Added
-
-- create_table API
-- limit parameter for queries
-- Typescript / JavaScript examples
-- Linux support
-
-## [0.1.0] - 2023-05-16
-
-### Added
-
-- Initial  JavaScript / Node.js library for LanceDB
-- Read-only api to query LanceDB datasets
-- Supports macOS arm only
-
-## [pre-0.1.0]
-
-- Various prototypes / test builds
-

node/README.md

@@ -9,8 +9,7 @@ npm install vectordb
 ```
 
 This will download the appropriate native library for your platform. We currently
-support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
-yet support Windows or musl-based Linux (such as Alpine Linux).
+support x86_64 Linux, Intel MacOS, and ARM (M1/M2) MacOS.
 
 ## Usage
 
@@ -18,11 +17,9 @@ yet support Windows or musl-based Linux (such as Alpine Linux).
 
 ```javascript
 const lancedb = require('vectordb');
-const db = await lancedb.connect('data/sample-lancedb');
-const table = await db.createTable("my_table",
-      [{ id: 1, vector: [0.1, 1.0], item: "foo", price: 10.0 },
-      { id: 2, vector: [3.9, 0.5], item: "bar", price: 20.0 }])
-const results = await table.search([0.1, 0.3]).limit(20).execute();
+const db = lancedb.connect('<PATH_TO_LANCEDB_DATASET>');
+const table = await db.openTable('my_table');
+const query = await table.search([0.1, 0.3]).setLimit(20).execute();
 console.log(results);
 ```
 
@@ -30,42 +27,33 @@ The [examples](./examples) folder contains complete examples.
 
 ## Development
 
-To build everything fresh:
+Build and install the rust library with:
 
 ```bash
-npm install
-npm run tsc
 npm run build
+npm run pack-build
+npm install --no-save ./dist/vectordb-*.tgz
 ```
 
-Then you should be able to run the tests with:
+`npm run build` builds the Rust library, `npm run pack-build` packages the Rust
+binary into an npm module called `@vectordb/<platform>` (for example, 
+`@vectordb/darwin-arm64.node`), and then `npm run install ...` installs that
+module.
+
+The LanceDB javascript is built with npm:
+
+```bash
+npm run tsc
+```
+
+Run the tests with
 
 ```bash
 npm test
 ```
 
-### Rebuilding Rust library
-
-```bash
-npm run build
-```
-
-### Rebuilding Typescript
-
-```bash
-npm run tsc
-```
-
-### Fix lints
-
 To run the linter and have it automatically fix all errors
 
 ```bash
 npm run lint -- --fix
 ```
-
-To build documentation
-
-```bash
-npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
-```

node/examples/js-openai/index.js

@@ -1,41 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-'use strict'
-
-async function example () {
-  const lancedb = require('vectordb')
-  // You need to provide an OpenAI API key, here we read it from the OPENAI_API_KEY environment variable
-  const apiKey = process.env.OPENAI_API_KEY
-  // The embedding function will create embeddings for the 'text' column(text in this case)
-  const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
-
-  const db = await lancedb.connect('data/sample-lancedb')
-
-  const data = [
-    { id: 1, text: 'Black T-Shirt', price: 10 },
-    { id: 2, text: 'Leather Jacket', price: 50 }
-  ]
-
-  const table = await db.createTable('vectors', data, embedding)
-  console.log(await db.tableNames())
-
-  const results = await table
-    .search('keeps me warm')
-    .limit(1)
-    .execute()
-  console.log(results[0].text)
-}
-
-example().then(_ => { console.log('All done!') })

node/examples/js-openai/package.json

@@ -1,15 +0,0 @@
-{
-  "name": "vectordb-example-js-openai",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "author": "Lance Devs",
-  "license": "Apache-2.0",
-  "dependencies": {
-    "vectordb": "file:../..",
-    "openai": "^3.2.1"
-  }
-}

node/examples/js-transformers/index.js

@@ -1,66 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-'use strict'
-
-
-async function example() {
-
-    const lancedb = require('vectordb')
-
-    // Import transformers and the all-MiniLM-L6-v2 model (https://huggingface.co/Xenova/all-MiniLM-L6-v2)
-    const { pipeline } = await import('@xenova/transformers')
-    const pipe = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
-
-
-    // Create embedding function from pipeline which returns a list of vectors from batch
-    // sourceColumn is the name of the column in the data to be embedded
-    //
-    // Output of pipe is a Tensor { data: Float32Array(384) }, so filter for the vector
-    const embed_fun = {}
-    embed_fun.sourceColumn = 'text'
-    embed_fun.embed = async function (batch) {
-        let result = []
-        for (let text of batch) {
-            const res = await pipe(text, { pooling: 'mean', normalize: true })
-            result.push(Array.from(res['data']))
-        }
-        return (result)
-    }
-
-    // Link a folder and create a table with data
-    const db = await lancedb.connect('data/sample-lancedb')
-
-    const data = [
-        { id: 1, text: 'Cherry', type: 'fruit' },
-        { id: 2, text: 'Carrot', type: 'vegetable' },
-        { id: 3, text: 'Potato', type: 'vegetable' },
-        { id: 4, text: 'Apple', type: 'fruit' },
-        { id: 5, text: 'Banana', type: 'fruit' }
-    ]
-
-    const table = await db.createTable('food_table', data, "create", embed_fun)
-
-
-    // Query the table
-    const results = await table
-        .search("a sweet fruit to eat")
-        .metricType("cosine")
-        .limit(2)
-        .execute()
-    console.log(results.map(r => r.text))
-
-}
-
-example().then(_ => { console.log("Done!") })

node/examples/js-transformers/package.json

@@ -1,16 +0,0 @@
-{
-  "name": "vectordb-example-js-transformers",
-  "version": "1.0.0",
-  "description": "Example for using transformers.js with lancedb",
-  "main": "index.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "author": "Lance Devs",
-  "license": "Apache-2.0",
-  "dependencies": {
-    "@xenova/transformers": "^2.4.1",
-    "vectordb": "^0.1.12"
-  }
-
-}

node/examples/js-youtube-transcripts/index.js

@@ -1,122 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-'use strict'
-
-const lancedb = require('vectordb')
-const fs = require('fs/promises')
-const readline = require('readline/promises')
-const { stdin: input, stdout: output } = require('process')
-const { Configuration, OpenAIApi } = require('openai')
-
-// Download file from XYZ
-const INPUT_FILE_NAME = 'data/youtube-transcriptions_sample.jsonl';
-
-(async () => {
-  // You need to provide an OpenAI API key, here we read it from the OPENAI_API_KEY environment variable
-  const apiKey = process.env.OPENAI_API_KEY
-  // The embedding function will create embeddings for the 'context' column
-  const embedFunction = new lancedb.OpenAIEmbeddingFunction('context', apiKey)
-
-  // Connects to LanceDB
-  const db = await lancedb.connect('data/youtube-lancedb')
-
-  // Open the vectors table or create one if it does not exist
-  let tbl
-  if ((await db.tableNames()).includes('vectors')) {
-    tbl = await db.openTable('vectors', embedFunction)
-  } else {
-    tbl = await createEmbeddingsTable(db, embedFunction)
-  }
-
-  // Use OpenAI Completion API to generate and answer based on the context that LanceDB provides
-  const configuration = new Configuration({ apiKey })
-  const openai = new OpenAIApi(configuration)
-  const rl = readline.createInterface({ input, output })
-  try {
-    while (true) {
-      const query = await rl.question('Prompt: ')
-      const results = await tbl
-        .search(query)
-        .select(['title', 'text', 'context'])
-        .limit(3)
-        .execute()
-
-      // console.table(results)
-
-      const response = await openai.createCompletion({
-        model: 'text-davinci-003',
-        prompt: createPrompt(query, results),
-        max_tokens: 400,
-        temperature: 0,
-        top_p: 1,
-        frequency_penalty: 0,
-        presence_penalty: 0
-      })
-      console.log(response.data.choices[0].text)
-    }
-  } catch (err) {
-    console.log('Error: ', err)
-  } finally {
-    rl.close()
-  }
-  process.exit(1)
-})()
-
-async function createEmbeddingsTable (db, embedFunction) {
-  console.log(`Creating embeddings from ${INPUT_FILE_NAME}`)
-  // read the input file into a JSON array, skipping empty lines
-  const lines = (await fs.readFile(INPUT_FILE_NAME, 'utf-8'))
-    .toString()
-    .split('\n')
-    .filter(line => line.length > 0)
-    .map(line => JSON.parse(line))
-
-  const data = contextualize(lines, 20, 'video_id')
-  return await db.createTable('vectors', data, embedFunction)
-}
-
-// Each transcript has a small text column, we include previous transcripts in order to
-// have more context information when creating embeddings
-function contextualize (rows, contextSize, groupColumn) {
-  const grouped = []
-  rows.forEach(row => {
-    if (!grouped[row[groupColumn]]) {
-      grouped[row[groupColumn]] = []
-    }
-    grouped[row[groupColumn]].push(row)
-  })
-
-  const data = []
-  Object.keys(grouped).forEach(key => {
-    for (let i = 0; i < grouped[key].length; i++) {
-      const start = i - contextSize > 0 ? i - contextSize : 0
-      grouped[key][i].context = grouped[key].slice(start, i + 1).map(r => r.text).join(' ')
-    }
-    data.push(...grouped[key])
-  })
-  return data
-}
-
-// Creates a prompt by aggregating all relevant contexts
-function createPrompt (query, context) {
-  let prompt =
-      'Answer the question based on the context below.\n\n' +
-      'Context:\n'
-
-  // need to make sure our prompt is not larger than max size
-  prompt = prompt + context.map(c => c.context).join('\n\n---\n\n').substring(0, 3750)
-  prompt = prompt + `\n\nQuestion: ${query}\nAnswer:`
-  return prompt
-}

node/examples/js-youtube-transcripts/package.json

@@ -1,15 +0,0 @@
-{
-  "name": "vectordb-example-js-openai",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "author": "Lance Devs",
-  "license": "Apache-2.0",
-  "dependencies": {
-    "vectordb": "file:../..",
-    "openai": "^3.2.1"
-  }
-}

node/examples/js/package.json

@@ -9,6 +9,6 @@
   "author": "Lance Devs",
   "license": "Apache-2.0",
   "dependencies": {
-    "vectordb": "file:../.."
+    "vectordb": "^0.1.0"
   }
 }

node/examples/ts/package.json

@@ -17,6 +17,6 @@
     "typescript": "*"
   },
   "dependencies": {
-    "vectordb": "file:../.."
+    "vectordb": "^0.1.0"
   }
 }

node/gen_test_data.py

@@ -0,0 +1,8 @@
+import lancedb
+
+uri = "sample-lancedb"
+db = lancedb.connect(uri)
+table = db.create_table("my_table",
+                         data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                               {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+

node/native.js

@@ -17,20 +17,14 @@ const { currentTarget } = require('@neon-rs/load');
 let nativeLib;
 
 try {
-    nativeLib = require(`vectordb-${currentTarget()}`);
+    nativeLib = require(`@vectordb/${currentTarget()}`);
 } catch (e) {
-    try {
-        // Might be developing locally, so try that. But don't expose that error
-        // to the user.
-        nativeLib = require("./index.node");
-    } catch {
-        throw new Error(`vectordb: failed to load native library.
-  You may need to run \`npm install vectordb-${currentTarget()}\`.
-      
+    throw new Error(`vectordb: failed to load native library.
+  You may need to run \`npm install @vectordb/${currentTarget()}\`.
+
   If that does not work, please file a bug report at https://github.com/lancedb/lancedb/issues
-      
+
   Source error: ${e}`);
-    }   
 }
 
 // Dynamic require for runtime.

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.1.14",
+  "version": "0.1.2",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -8,11 +8,10 @@
     "tsc": "tsc -b",
     "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json",
     "build-release": "npm run build -- --release",
-    "test": "npm run tsc && mocha -recursive dist/test",
+    "cross-release": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cross build --message-format=json --release -p vectordb-node",
+    "test": "mocha -recursive dist/test",
     "lint": "eslint src --ext .js,.ts",
-    "clean": "rm -rf node_modules *.node dist/",
-    "pack-build": "neon pack-build",
-    "check-npm": "printenv && which node && which npm && npm --version"
+    "pack-build": "neon pack-build"
   },
   "repository": {
     "type": "git",
@@ -29,40 +28,31 @@
   "devDependencies": {
     "@neon-rs/cli": "^0.0.74",
     "@types/chai": "^4.3.4",
-    "@types/chai-as-promised": "^7.1.5",
     "@types/mocha": "^10.0.1",
     "@types/node": "^18.16.2",
-    "@types/sinon": "^10.0.15",
     "@types/temp": "^0.9.1",
     "@typescript-eslint/eslint-plugin": "^5.59.1",
     "cargo-cp-artifact": "^0.1",
     "chai": "^4.3.7",
-    "chai-as-promised": "^7.1.1",
     "eslint": "^8.39.0",
     "eslint-config-standard-with-typescript": "^34.0.1",
-    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-import": "^2.27.5",
     "eslint-plugin-n": "^15.7.0",
     "eslint-plugin-promise": "^6.1.1",
     "mocha": "^10.2.0",
-    "openai": "^3.2.1",
-    "sinon": "^15.1.0",
     "temp": "^0.9.4",
     "ts-node": "^10.9.1",
     "ts-node-dev": "^2.0.0",
-    "typedoc": "^0.24.7",
-    "typedoc-plugin-markdown": "^3.15.3",
     "typescript": "*"
   },
   "dependencies": {
     "@apache-arrow/ts": "^12.0.0",
     "@neon-rs/load": "^0.0.74",
-    "apache-arrow": "^12.0.0",
-    "axios": "^1.4.0"
+    "apache-arrow": "^12.0.0"
   },
   "os": [
     "darwin",
-    "linux",
-    "win32"
+    "linux"
   ],
   "cpu": [
     "x64",
@@ -70,18 +60,20 @@
   ],
   "neon": {
     "targets": {
-      "x86_64-apple-darwin": "vectordb-darwin-x64",
-      "aarch64-apple-darwin": "vectordb-darwin-arm64",
-      "x86_64-unknown-linux-gnu": "vectordb-linux-x64-gnu",
-      "aarch64-unknown-linux-gnu": "vectordb-linux-arm64-gnu",
-      "x86_64-pc-windows-msvc": "vectordb-win32-x64-msvc"
+      "x86_64-apple-darwin": "@vectordb/darwin-x64",
+      "aarch64-apple-darwin": "@vectordb/darwin-arm64",
+      "x86_64-unknown-linux-gnu": "@vectordb/linux-x64-gnu",
+      "x86_64-unknown-linux-musl": "@vectordb/linux-x64-musl",
+      "aarch64-unknown-linux-gnu": "@vectordb/linux-arm64-gnu",
+      "aarch64-unknown-linux-musl": "@vectordb/linux-arm64-musl"
     }
   },
   "optionalDependencies": {
-    "vectordb-darwin-arm64": "0.1.14",
-    "vectordb-darwin-x64": "0.1.14",
-    "vectordb-linux-arm64-gnu": "0.1.14",
-    "vectordb-linux-x64-gnu": "0.1.14",
-    "vectordb-win32-x64-msvc": "0.1.14"
+    "@vectordb/darwin-arm64": "0.1.2",
+    "@vectordb/darwin-x64": "0.1.2",
+    "@vectordb/linux-x64-gnu": "0.1.2",
+    "@vectordb/linux-x64-musl": "0.1.2",
+    "@vectordb/linux-arm64-gnu": "0.1.2",
+    "@vectordb/linux-arm64-musl": "0.1.2"
   }
 }

node/src/arrow.ts

@@ -15,16 +15,15 @@
 import {
   Field,
   Float32,
-  List, type ListBuilder,
+  List,
   makeBuilder,
   RecordBatchFileWriter,
-  Table, Utf8,
+  Table,
   type Vector,
   vectorFromArray
 } from 'apache-arrow'
-import { type EmbeddingFunction } from './index'
 
-export async function convertToTable<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Table> {
+export function convertToTable (data: Array<Record<string, unknown>>): Table {
   if (data.length === 0) {
     throw new Error('At least one record needs to be provided')
   }
@@ -34,7 +33,11 @@ export async function convertToTable<T> (data: Array<Record<string, unknown>>, e
 
   for (const columnsKey of columns) {
     if (columnsKey === 'vector') {
-      const listBuilder = newVectorListBuilder()
+      const children = new Field<Float32>('item', new Float32())
+      const list = new List(children)
+      const listBuilder = makeBuilder({
+        type: list
+      })
       const vectorSize = (data[0].vector as any[]).length
       for (const datum of data) {
         if ((datum[columnsKey] as any[]).length !== vectorSize) {
@@ -49,37 +52,15 @@ export async function convertToTable<T> (data: Array<Record<string, unknown>>, e
       for (const datum of data) {
         values.push(datum[columnsKey])
       }
-
-      if (columnsKey === embeddings?.sourceColumn) {
-        const vectors = await embeddings.embed(values as T[])
-        const listBuilder = newVectorListBuilder()
-        vectors.map(v => listBuilder.append(v))
-        records.vector = listBuilder.finish().toVector()
-      }
-
-      if (typeof values[0] === 'string') {
-        // `vectorFromArray` converts strings into dictionary vectors, forcing it back to a string column
-        records[columnsKey] = vectorFromArray(values, new Utf8())
-      } else {
-        records[columnsKey] = vectorFromArray(values)
-      }
+      records[columnsKey] = vectorFromArray(values)
     }
   }
 
   return new Table(records)
 }
 
-// Creates a new Arrow ListBuilder that stores a Vector column
-function newVectorListBuilder (): ListBuilder<Float32, any> {
-  const children = new Field<Float32>('item', new Float32())
-  const list = new List(children)
-  return makeBuilder({
-    type: list
-  })
-}
-
-export async function fromRecordsToBuffer<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Buffer> {
-  const table = await convertToTable(data, embeddings)
+export async function fromRecordsToBuffer (data: Array<Record<string, unknown>>): Promise<Buffer> {
+  const table = convertToTable(data)
   const writer = RecordBatchFileWriter.writeAll(table)
   return Buffer.from(await writer.toUint8Array())
 }

node/src/embedding/embedding_function.ts

@@ -1,28 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-/**
- * An embedding function that automatically creates vector representation for a given column.
- */
-export interface EmbeddingFunction<T> {
-  /**
-     * The name of the column that will be used as input for the Embedding Function.
-     */
-  sourceColumn: string
-
-  /**
-     * Creates a vector representation for the given values.
-     */
-  embed: (data: T[]) => Promise<number[][]>
-}

node/src/embedding/openai.ts

@@ -1,51 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { type EmbeddingFunction } from '../index'
-
-export class OpenAIEmbeddingFunction implements EmbeddingFunction<string> {
-  private readonly _openai: any
-  private readonly _modelName: string
-
-  constructor (sourceColumn: string, openAIKey: string, modelName: string = 'text-embedding-ada-002') {
-    let openai
-    try {
-      // eslint-disable-next-line @typescript-eslint/no-var-requires
-      openai = require('openai')
-    } catch {
-      throw new Error('please install openai using npm install openai')
-    }
-
-    this.sourceColumn = sourceColumn
-    const configuration = new openai.Configuration({
-      apiKey: openAIKey
-    })
-    this._openai = new openai.OpenAIApi(configuration)
-    this._modelName = modelName
-  }
-
-  async embed (data: string[]): Promise<number[][]> {
-    const response = await this._openai.createEmbedding({
-      model: this._modelName,
-      input: data
-    })
-    const embeddings: number[][] = []
-    for (let i = 0; i < response.data.data.length; i++) {
-      embeddings.push(response.data.data[i].embedding as number[])
-    }
-    return embeddings
-  }
-
-  sourceColumn: string
-}

node/src/index.ts

@@ -14,267 +14,58 @@
 
 import {
   RecordBatchFileWriter,
-  type Table as ArrowTable
+  type Table as ArrowTable,
+  tableFromIPC,
+  Vector
 } from 'apache-arrow'
 import { fromRecordsToBuffer } from './arrow'
-import type { EmbeddingFunction } from './embedding/embedding_function'
-import { RemoteConnection } from './remote'
-import { Query } from './query'
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete } = require('../native.js')
-
-export type { EmbeddingFunction }
-export { OpenAIEmbeddingFunction } from './embedding/openai'
-
-export interface AwsCredentials {
-  accessKeyId: string
-
-  secretKey: string
-
-  sessionToken?: string
-}
-
-export interface ConnectionOptions {
-  uri: string
-
-  awsCredentials?: AwsCredentials
-
-  // API key for the remote connections
-  apiKey?: string
-  // Region to connect
-  region?: string
-}
+const { databaseNew, databaseTableNames, databaseOpenTable, tableCreate, tableSearch, tableAdd } = require('../native.js')
 
 /**
  * Connect to a LanceDB instance at the given URI
  * @param uri The uri of the database.
  */
-export async function connect (uri: string): Promise<Connection>
-export async function connect (opts: Partial<ConnectionOptions>): Promise<Connection>
-export async function connect (arg: string | Partial<ConnectionOptions>): Promise<Connection> {
-  let opts: ConnectionOptions
-  if (typeof arg === 'string') {
-    opts = { uri: arg }
-  } else {
-    // opts = { uri: arg.uri, awsCredentials = arg.awsCredentials }
-    opts = Object.assign({
-      uri: '',
-      awsCredentials: undefined,
-      apiKey: undefined,
-      region: 'us-west-2'
-    }, arg)
-  }
-
-  if (opts.uri.startsWith('db://')) {
-    // Remote connection
-    return new RemoteConnection(opts)
-  }
-  const db = await databaseNew(opts.uri)
-  return new LocalConnection(db, opts)
-}
-
-/**
- * A LanceDB Connection that allows you to open tables and create new ones.
- *
- * Connection could be local against filesystem or remote against a server.
- */
-export interface Connection {
-  uri: string
-
-  tableNames(): Promise<string[]>
-
-  /**
-   * Open a table in the database.
-   *
-   * @param name The name of the table.
-   * @param embeddings An embedding function to use on this table
-   */
-  openTable<T>(name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
-
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param {string} name - The name of the table.
-   * @param data - Non-empty Array of Records to be inserted into the table
-   * @param {WriteMode} mode - The write mode to use when creating the table.
-   * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
-   */
-  createTable<T>(name: string, data: Array<Record<string, unknown>>, mode?: WriteMode, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
-
-  createTableArrow(name: string, table: ArrowTable): Promise<Table>
-
-  /**
-   * Drop an existing table.
-   * @param name The name of the table to drop.
-   */
-  dropTable(name: string): Promise<void>
-
-}
-
-/**
- * A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
- */
-export interface Table<T = number[]> {
-  name: string
-
-  /**
-   * Creates a search query to find the nearest neighbors of the given search term
-   * @param query The query search term
-   */
-  search: (query: T) => Query<T>
-
-  /**
-   * Insert records into this Table.
-   *
-   * @param data Records to be inserted into the Table
-   * @return The number of rows added to the table
-   */
-  add: (data: Array<Record<string, unknown>>) => Promise<number>
-
-  /**
-   * Insert records into this Table, replacing its contents.
-   *
-   * @param data Records to be inserted into the Table
-   * @return The number of rows added to the table
-   */
-  overwrite: (data: Array<Record<string, unknown>>) => Promise<number>
-
-  /**
-   * Create an ANN index on this Table vector index.
-   *
-   * @param indexParams The parameters of this Index, @see VectorIndexParams.
-   */
-  createIndex: (indexParams: VectorIndexParams) => Promise<any>
-
-  /**
-   * Returns the number of rows in this table.
-   */
-  countRows: () => Promise<number>
-
-  /**
-   * Delete rows from this table.
-   *
-   * This can be used to delete a single row, many rows, all rows, or
-   * sometimes no rows (if your predicate matches nothing).
-   *
-   * @param filter  A filter in the same format used by a sql WHERE clause. The
-   *                filter must not be empty.
-   *
-   * @examples
-   *
-   * ```ts
-   * const con = await lancedb.connect("./.lancedb")
-   * const data = [
-   *    {id: 1, vector: [1, 2]},
-   *    {id: 2, vector: [3, 4]},
-   *    {id: 3, vector: [5, 6]},
-   * ];
-   * const tbl = await con.createTable("my_table", data)
-   * await tbl.delete("id = 2")
-   * await tbl.countRows() // Returns 2
-   * ```
-   *
-   * If you have a list of values to delete, you can combine them into a
-   * stringified list and use the `IN` operator:
-   *
-   * ```ts
-   * const to_remove = [1, 5];
-   * await tbl.delete(`id IN (${to_remove.join(",")})`)
-   * await tbl.countRows() // Returns 1
-   * ```
-   */
-  delete: (filter: string) => Promise<void>
+export async function connect (uri: string): Promise<Connection> {
+  return new Connection(uri)
 }
 
 /**
  * A connection to a LanceDB database.
  */
-export class LocalConnection implements Connection {
-  private readonly _options: ConnectionOptions
+export class Connection {
+  private readonly _uri: string
   private readonly _db: any
 
-  constructor (db: any, options: ConnectionOptions) {
-    this._options = options
-    this._db = db
+  constructor (uri: string) {
+    this._uri = uri
+    this._db = databaseNew(uri)
   }
 
   get uri (): string {
-    return this._options.uri
+    return this._uri
   }
 
   /**
-   * Get the names of all tables in the database.
-   */
+     * Get the names of all tables in the database.
+     */
   async tableNames (): Promise<string[]> {
     return databaseTableNames.call(this._db)
   }
 
   /**
-   * Open a table in the database.
-   *
-   * @param name The name of the table.
-   */
-  async openTable (name: string): Promise<Table>
-
-  /**
-   * Open a table in the database.
-   *
-   * @param name The name of the table.
-   * @param embeddings An embedding function to use on this Table
-   */
-  async openTable<T> (name: string, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
-  async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
-  async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
+     * Open a table in the database.
+     * @param name The name of the table.
+     */
+  async openTable (name: string): Promise<Table> {
     const tbl = await databaseOpenTable.call(this._db, name)
-    if (embeddings !== undefined) {
-      return new LocalTable(tbl, name, this._options, embeddings)
-    } else {
-      return new LocalTable(tbl, name, this._options)
-    }
+    return new Table(tbl, name)
   }
 
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param name The name of the table.
-   * @param data Non-empty Array of Records to be inserted into the Table
-   * @param mode The write mode to use when creating the table.
-   */
-  async createTable (name: string, data: Array<Record<string, unknown>>, mode?: WriteMode): Promise<Table>
-  async createTable (name: string, data: Array<Record<string, unknown>>, mode: WriteMode): Promise<Table>
-
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param name The name of the table.
-   * @param data Non-empty Array of Records to be inserted into the Table
-   * @param mode The write mode to use when creating the table.
-   * @param embeddings An embedding function to use on this Table
-   */
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, mode: WriteMode, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, mode: WriteMode, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, mode: WriteMode, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
-    if (mode === undefined) {
-      mode = WriteMode.Create
-    }
-
-    const createArgs = [this._db, name, await fromRecordsToBuffer(data, embeddings), mode.toLowerCase()]
-    if (this._options.awsCredentials !== undefined) {
-      createArgs.push(this._options.awsCredentials.accessKeyId)
-      createArgs.push(this._options.awsCredentials.secretKey)
-      if (this._options.awsCredentials.sessionToken !== undefined) {
-        createArgs.push(this._options.awsCredentials.sessionToken)
-      }
-    }
-
-    const tbl = await tableCreate.call(...createArgs)
-
-    if (embeddings !== undefined) {
-      return new LocalTable(tbl, name, this._options, embeddings)
-    } else {
-      return new LocalTable(tbl, name, this._options)
-    }
+  async createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table> {
+    await tableCreate.call(this._db, name, await fromRecordsToBuffer(data))
+    return await this.openTable(name)
   }
 
   async createTableArrow (name: string, table: ArrowTable): Promise<Table> {
@@ -282,35 +73,18 @@ export class LocalConnection implements Connection {
     await tableCreate.call(this._db, name, Buffer.from(await writer.toUint8Array()))
     return await this.openTable(name)
   }
-
-  /**
-   * Drop an existing table.
-   * @param name The name of the table to drop.
-   */
-  async dropTable (name: string): Promise<void> {
-    await databaseDropTable.call(this._db, name)
-  }
 }
 
-export class LocalTable<T = number[]> implements Table<T> {
+/**
+ * A table in a LanceDB database.
+ */
+export class Table {
   private readonly _tbl: any
   private readonly _name: string
-  private readonly _embeddings?: EmbeddingFunction<T>
-  private readonly _options: ConnectionOptions
 
-  constructor (tbl: any, name: string, options: ConnectionOptions)
-  /**
-   * @param tbl
-   * @param name
-   * @param options
-   * @param embeddings An embedding function to use when interacting with this table
-   */
-  constructor (tbl: any, name: string, options: ConnectionOptions, embeddings: EmbeddingFunction<T>)
-  constructor (tbl: any, name: string, options: ConnectionOptions, embeddings?: EmbeddingFunction<T>) {
+  constructor (tbl: any, name: string) {
     this._tbl = tbl
     this._name = name
-    this._embeddings = embeddings
-    this._options = options
   }
 
   get name (): string {
@@ -318,160 +92,88 @@ export class LocalTable<T = number[]> implements Table<T> {
   }
 
   /**
-   * Creates a search query to find the nearest neighbors of the given search term
-   * @param query The query search term
-   */
-  search (query: T): Query<T> {
-    return new Query(query, this._tbl, this._embeddings)
+     * Create a search query to find the nearest neighbors of the given query vector.
+     * @param queryVector The query vector.
+     */
+  search (queryVector: number[]): Query {
+    return new Query(this._tbl, queryVector)
   }
 
   /**
-   * Insert records into this Table.
-   *
+   * Insert records into this Table
    * @param data Records to be inserted into the Table
+   *
+   * @param mode Append / Overwrite existing records. Default: Append
    * @return The number of rows added to the table
    */
   async add (data: Array<Record<string, unknown>>): Promise<number> {
-    const callArgs = [this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Append.toString()]
-    if (this._options.awsCredentials !== undefined) {
-      callArgs.push(this._options.awsCredentials.accessKeyId)
-      callArgs.push(this._options.awsCredentials.secretKey)
-      if (this._options.awsCredentials.sessionToken !== undefined) {
-        callArgs.push(this._options.awsCredentials.sessionToken)
-      }
-    }
-    return tableAdd.call(...callArgs)
+    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data), WriteMode.Append.toString())
   }
 
-  /**
-   * Insert records into this Table, replacing its contents.
-   *
-   * @param data Records to be inserted into the Table
-   * @return The number of rows added to the table
-   */
   async overwrite (data: Array<Record<string, unknown>>): Promise<number> {
-    const callArgs = [this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Overwrite.toString()]
-    if (this._options.awsCredentials !== undefined) {
-      callArgs.push(this._options.awsCredentials.accessKeyId)
-      callArgs.push(this._options.awsCredentials.secretKey)
-      if (this._options.awsCredentials.sessionToken !== undefined) {
-        callArgs.push(this._options.awsCredentials.sessionToken)
-      }
-    }
-    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Overwrite.toString())
-  }
-
-  /**
-   * Create an ANN index on this Table vector index.
-   *
-   * @param indexParams The parameters of this Index, @see VectorIndexParams.
-   */
-  async createIndex (indexParams: VectorIndexParams): Promise<any> {
-    return tableCreateVectorIndex.call(this._tbl, indexParams)
-  }
-
-  /**
-   * Returns the number of rows in this table.
-   */
-  async countRows (): Promise<number> {
-    return tableCountRows.call(this._tbl)
-  }
-
-  /**
-   * Delete rows from this table.
-   *
-   * @param filter A filter in the same format used by a sql WHERE clause.
-   */
-  async delete (filter: string): Promise<void> {
-    return tableDelete.call(this._tbl, filter)
+    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data), WriteMode.Overwrite.toString())
   }
 }
 
-/// Config to build IVF_PQ index.
-///
-export interface IvfPQIndexConfig {
-  /**
-   * The column to be indexed
-   */
-  column?: string
-
-  /**
-   * A unique name for the index
-   */
-  index_name?: string
-
-  /**
-   * Metric type, L2 or Cosine
-   */
-  metric_type?: MetricType
-
-  /**
-   * The number of partitions this index
-   */
-  num_partitions?: number
-
-  /**
-   * The max number of iterations for kmeans training.
-   */
-  max_iters?: number
-
-  /**
-   * Train as optimized product quantization.
-   */
-  use_opq?: boolean
-
-  /**
-   * Number of subvectors to build PQ code
-   */
-  num_sub_vectors?: number
-  /**
-   * The number of bits to present one PQ centroid.
-   */
-  num_bits?: number
-
-  /**
-   * Max number of iterations to train OPQ, if `use_opq` is true.
-   */
-  max_opq_iters?: number
-
-  /**
-   * Replace an existing index with the same name if it exists.
-   */
-  replace?: boolean
-
-  type: 'ivf_pq'
-}
-
-export type VectorIndexParams = IvfPQIndexConfig
-
 /**
- * Write mode for writing a table.
+ * A builder for nearest neighbor queries for LanceDB.
  */
+export class Query {
+  private readonly _tbl: any
+  private readonly _query_vector: number[]
+  private _limit: number
+  private readonly _refine_factor?: number
+  private readonly _nprobes: number
+  private readonly _columns?: string[]
+  private _filter?: string
+  private readonly _metric = 'L2'
+
+  constructor (tbl: any, queryVector: number[]) {
+    this._tbl = tbl
+    this._query_vector = queryVector
+    this._limit = 10
+    this._nprobes = 20
+    this._refine_factor = undefined
+    this._columns = undefined
+    this._filter = undefined
+  }
+
+  limit (value: number): Query {
+    this._limit = value
+    return this
+  }
+
+  filter (value: string): Query {
+    this._filter = value
+    return this
+  }
+
+  /**
+     * Execute the query and return the results as an Array of Objects
+     */
+  async execute<T = Record<string, unknown>> (): Promise<T[]> {
+    let buffer
+    if (this._filter != null) {
+      buffer = await tableSearch.call(this._tbl, this._query_vector, this._limit, this._filter)
+    } else {
+      buffer = await tableSearch.call(this._tbl, this._query_vector, this._limit)
+    }
+    const data = tableFromIPC(buffer)
+    return data.toArray().map((entry: Record<string, unknown>) => {
+      const newObject: Record<string, unknown> = {}
+      Object.keys(entry).forEach((key: string) => {
+        if (entry[key] instanceof Vector) {
+          newObject[key] = (entry[key] as Vector).toArray()
+        } else {
+          newObject[key] = entry[key]
+        }
+      })
+      return newObject as unknown as T
+    })
+  }
+}
+
 export enum WriteMode {
-  /** Create a new {@link Table}. */
-  Create = 'create',
-  /** Overwrite the existing {@link Table} if presented. */
   Overwrite = 'overwrite',
-  /** Append new data to the table. */
   Append = 'append'
 }
-
-/**
- * Distance metrics type.
- */
-export enum MetricType {
-  /**
-   * Euclidean distance
-   */
-  L2 = 'l2',
-
-  /**
-   * Cosine distance
-   */
-  Cosine = 'cosine',
-
-  /**
-   * Dot product
-   */
-  Dot = 'dot'
-}

node/src/query.ts

@@ -1,130 +0,0 @@
-// Copyright 2023 LanceDB Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { Vector, tableFromIPC } from 'apache-arrow'
-import { type EmbeddingFunction } from './embedding/embedding_function'
-import { type MetricType } from '.'
-
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const { tableSearch } = require('../native.js')
-
-/**
- * A builder for nearest neighbor queries for LanceDB.
- */
-export class Query<T = number[]> {
-  private readonly _query: T
-  private readonly _tbl?: any
-  private _queryVector?: number[]
-  private _limit: number
-  private _refineFactor?: number
-  private _nprobes: number
-  private _select?: string[]
-  private _filter?: string
-  private _metricType?: MetricType
-  protected readonly _embeddings?: EmbeddingFunction<T>
-
-  constructor (query: T, tbl?: any, embeddings?: EmbeddingFunction<T>) {
-    this._tbl = tbl
-    this._query = query
-    this._limit = 10
-    this._nprobes = 20
-    this._refineFactor = undefined
-    this._select = undefined
-    this._filter = undefined
-    this._metricType = undefined
-    this._embeddings = embeddings
-  }
-
-  /***
-     * Sets the number of results that will be returned
-     * @param value number of results
-     */
-  limit (value: number): Query<T> {
-    this._limit = value
-    return this
-  }
-
-  /**
-     * Refine the results by reading extra elements and re-ranking them in memory.
-     * @param value refine factor to use in this query.
-     */
-  refineFactor (value: number): Query<T> {
-    this._refineFactor = value
-    return this
-  }
-
-  /**
-     * The number of probes used. A higher number makes search more accurate but also slower.
-     * @param value The number of probes used.
-     */
-  nprobes (value: number): Query<T> {
-    this._nprobes = value
-    return this
-  }
-
-  /**
-     * A filter statement to be applied to this query.
-     * @param value A filter in the same format used by a sql WHERE clause.
-     */
-  filter (value: string): Query<T> {
-    this._filter = value
-    return this
-  }
-
-  where = this.filter
-
-  /** Return only the specified columns.
-     *
-     * @param value Only select the specified columns. If not specified, all columns will be returned.
-     */
-  select (value: string[]): Query<T> {
-    this._select = value
-    return this
-  }
-
-  /**
-     * The MetricType used for this Query.
-     * @param value The metric to the. @see MetricType for the different options
-     */
-  metricType (value: MetricType): Query<T> {
-    this._metricType = value
-    return this
-  }
-
-  /**
-     * Execute the query and return the results as an Array of Objects
-     */
-  async execute<T = Record<string, unknown>> (): Promise<T[]> {
-    if (this._embeddings !== undefined) {
-      this._queryVector = (await this._embeddings.embed([this._query]))[0]
-    } else {
-      this._queryVector = this._query as number[]
-    }
-
-    const buffer = await tableSearch.call(this._tbl, this)
-    const data = tableFromIPC(buffer)
-
-    return data.toArray().map((entry: Record<string, unknown>) => {
-      const newObject: Record<string, unknown> = {}
-      Object.keys(entry).forEach((key: string) => {
-        if (entry[key] instanceof Vector) {
-          newObject[key] = (entry[key] as Vector).toArray()
-        } else {
-          newObject[key] = entry[key]
-        }
-      })
-      return newObject as unknown as T
-    })
-  }
-}

node/src/remote/client.ts

@@ -1,100 +0,0 @@
-// Copyright 2023 LanceDB Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import axios, { type AxiosResponse } from 'axios'
-
-import { tableFromIPC, type Table as ArrowTable } from 'apache-arrow'
-
-export class HttpLancedbClient {
-  private readonly _url: string
-
-  public constructor (url: string, private readonly _apiKey: string) {
-    this._url = url
-  }
-
-  get uri (): string {
-    return this._url
-  }
-
-  public async search (
-    tableName: string,
-    vector: number[],
-    k: number,
-    nprobes: number,
-    refineFactor?: number,
-    columns?: string[],
-    filter?: string
-  ): Promise<ArrowTable<any>> {
-    const response = await axios.post(
-              `${this._url}/v1/table/${tableName}`,
-              {
-                vector,
-                k,
-                nprobes,
-                refineFactor,
-                columns,
-                filter
-              },
-              {
-                headers: {
-                  'Content-Type': 'application/json',
-                  'x-api-key': this._apiKey
-                },
-                responseType: 'arraybuffer',
-                timeout: 10000
-              }
-    ).catch((err) => {
-      console.error('error: ', err)
-      return err.response
-    })
-    if (response.status !== 200) {
-      const errorData = new TextDecoder().decode(response.data)
-      throw new Error(
-        `Server Error, status: ${response.status as number}, ` +
-        `message: ${response.statusText as string}: ${errorData}`
-      )
-    }
-
-    const table = tableFromIPC(response.data)
-    return table
-  }
-
-  /**
-   * Sent GET request.
-   */
-  public async get (path: string, params?: Record<string, string | number>): Promise<AxiosResponse> {
-    const response = await axios.get(
-      `${this._url}${path}`,
-      {
-        headers: {
-          'Content-Type': 'application/json',
-          'x-api-key': this._apiKey
-        },
-        params,
-        timeout: 10000
-      }
-    ).catch((err) => {
-      console.error('error: ', err)
-      return err.response
-    })
-    if (response.status !== 200) {
-      const errorData = new TextDecoder().decode(response.data)
-      throw new Error(
-        `Server Error, status: ${response.status as number}, ` +
-        `message: ${response.statusText as string}: ${errorData}`
-      )
-    }
-    return response
-  }
-}

node/src/remote/index.ts

@@ -1,163 +0,0 @@
-// Copyright 2023 LanceDB Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import {
-  type EmbeddingFunction, type Table, type VectorIndexParams, type Connection,
-  type ConnectionOptions
-} from '../index'
-import { Query } from '../query'
-
-import { type Table as ArrowTable, Vector } from 'apache-arrow'
-import { HttpLancedbClient } from './client'
-
-/**
- * Remote connection.
- */
-export class RemoteConnection implements Connection {
-  private readonly _client: HttpLancedbClient
-  private readonly _dbName: string
-
-  constructor (opts: ConnectionOptions) {
-    if (!opts.uri.startsWith('db://')) {
-      throw new Error(`Invalid remote DB URI: ${opts.uri}`)
-    }
-    if (opts.apiKey === undefined || opts.region === undefined) {
-      throw new Error('API key and region are not supported for remote connections')
-    }
-
-    this._dbName = opts.uri.slice('db://'.length)
-    const server = `https://${this._dbName}.${opts.region}.api.lancedb.com`
-    this._client = new HttpLancedbClient(server, opts.apiKey)
-  }
-
-  get uri (): string {
-    // add the lancedb+ prefix back
-    return 'db://' + this._client.uri
-  }
-
-  async tableNames (): Promise<string[]> {
-    const response = await this._client.get('/v1/table/')
-    return response.data.tables
-  }
-
-  async openTable (name: string): Promise<Table>
-  async openTable<T> (name: string, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
-  async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
-    if (embeddings !== undefined) {
-      return new RemoteTable(this._client, name, embeddings)
-    } else {
-      return new RemoteTable(this._client, name)
-    }
-  }
-
-  async createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table>
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
-    throw new Error('Not implemented')
-  }
-
-  async createTableArrow (name: string, table: ArrowTable): Promise<Table> {
-    throw new Error('Not implemented')
-  }
-
-  async dropTable (name: string): Promise<void> {
-    throw new Error('Not implemented')
-  }
-}
-
-export class RemoteQuery<T = number[]> extends Query<T> {
-  constructor (query: T, private readonly _client: HttpLancedbClient,
-    private readonly _name: string, embeddings?: EmbeddingFunction<T>) {
-    super(query, undefined, embeddings)
-  }
-
-  // TODO: refactor this to a base class + queryImpl pattern
-  async execute<T = Record<string, unknown>>(): Promise<T[]> {
-    const embeddings = this._embeddings
-    const query = (this as any)._query
-    let queryVector: number[]
-
-    if (embeddings !== undefined) {
-      queryVector = (await embeddings.embed([query]))[0]
-    } else {
-      queryVector = query as number[]
-    }
-
-    const data = await this._client.search(
-      this._name,
-      queryVector,
-      (this as any)._limit,
-      (this as any)._nprobes,
-      (this as any)._refineFactor,
-      (this as any)._select,
-      (this as any)._filter
-    )
-
-    return data.toArray().map((entry: Record<string, unknown>) => {
-      const newObject: Record<string, unknown> = {}
-      Object.keys(entry).forEach((key: string) => {
-        if (entry[key] instanceof Vector) {
-          newObject[key] = (entry[key] as Vector).toArray()
-        } else {
-          newObject[key] = entry[key]
-        }
-      })
-      return newObject as unknown as T
-    })
-  }
-}
-
-// we are using extend until we have next next version release
-// Table and Connection has both been refactored to interfaces
-export class RemoteTable<T = number[]> implements Table<T> {
-  private readonly _client: HttpLancedbClient
-  private readonly _embeddings?: EmbeddingFunction<T>
-  private readonly _name: string
-
-  constructor (client: HttpLancedbClient, name: string)
-  constructor (client: HttpLancedbClient, name: string, embeddings: EmbeddingFunction<T>)
-  constructor (client: HttpLancedbClient, name: string, embeddings?: EmbeddingFunction<T>) {
-    this._client = client
-    this._name = name
-    this._embeddings = embeddings
-  }
-
-  get name (): string {
-    return this._name
-  }
-
-  search (query: T): Query<T> {
-    return new RemoteQuery(query, this._client, this._name)//, this._embeddings_new)
-  }
-
-  async add (data: Array<Record<string, unknown>>): Promise<number> {
-    throw new Error('Not implemented')
-  }
-
-  async overwrite (data: Array<Record<string, unknown>>): Promise<number> {
-    throw new Error('Not implemented')
-  }
-
-  async createIndex (indexParams: VectorIndexParams): Promise<any> {
-    throw new Error('Not implemented')
-  }
-
-  async countRows (): Promise<number> {
-    throw new Error('Not implemented')
-  }
-
-  async delete (filter: string): Promise<void> {
-    throw new Error('Not implemented')
-  }
-}

node/src/test/embedding/openai.ts

@@ -1,50 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { describe } from 'mocha'
-import { assert } from 'chai'
-
-import { OpenAIEmbeddingFunction } from '../../embedding/openai'
-
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const { OpenAIApi } = require('openai')
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const { stub } = require('sinon')
-
-describe('OpenAPIEmbeddings', function () {
-  const stubValue = {
-    data: {
-      data: [
-        {
-          embedding: Array(1536).fill(1.0)
-        },
-        {
-          embedding: Array(1536).fill(2.0)
-        }
-      ]
-    }
-  }
-
-  describe('#embed', function () {
-    it('should create vector embeddings', async function () {
-      const openAIStub = stub(OpenAIApi.prototype, 'createEmbedding').returns(stubValue)
-      const f = new OpenAIEmbeddingFunction('text', 'sk-key')
-      const vectors = await f.embed(['abc', 'def'])
-      assert.isTrue(openAIStub.calledOnce)
-      assert.equal(vectors.length, 2)
-      assert.deepEqual(vectors[0], stubValue.data.data[0].embedding)
-      assert.deepEqual(vectors[1], stubValue.data.data[1].embedding)
-    })
-  })
-})

node/src/test/io.ts

@@ -1,74 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// IO tests
-
-import { describe } from 'mocha'
-import { assert } from 'chai'
-
-import * as lancedb from '../index'
-import { type ConnectionOptions } from '../index'
-
-describe('LanceDB S3 client', function () {
-  if (process.env.TEST_S3_BASE_URL != null) {
-    const baseUri = process.env.TEST_S3_BASE_URL
-    it('should have a valid url', async function () {
-      const opts = { uri: `${baseUri}/valid_url` }
-      const table = await createTestDB(opts, 2, 20)
-      const con = await lancedb.connect(opts)
-      assert.equal(con.uri, opts.uri)
-
-      const results = await table.search([0.1, 0.3]).limit(5).execute()
-      assert.equal(results.length, 5)
-    }).timeout(10_000)
-  } else {
-    describe.skip('Skip S3 test', function () {})
-  }
-
-  if (process.env.TEST_S3_BASE_URL != null && process.env.TEST_AWS_ACCESS_KEY_ID != null && process.env.TEST_AWS_SECRET_ACCESS_KEY != null) {
-    const baseUri = process.env.TEST_S3_BASE_URL
-    it('use custom credentials', async function () {
-      const opts: ConnectionOptions = {
-        uri: `${baseUri}/custom_credentials`,
-        awsCredentials: {
-          accessKeyId: process.env.TEST_AWS_ACCESS_KEY_ID as string,
-          secretKey: process.env.TEST_AWS_SECRET_ACCESS_KEY as string
-        }
-      }
-      const table = await createTestDB(opts, 2, 20)
-      const con = await lancedb.connect(opts)
-      assert.equal(con.uri, opts.uri)
-
-      const results = await table.search([0.1, 0.3]).limit(5).execute()
-      assert.equal(results.length, 5)
-    }).timeout(10_000)
-  } else {
-    describe.skip('Skip S3 test', function () {})
-  }
-})
-
-async function createTestDB (opts: ConnectionOptions, numDimensions: number = 2, numRows: number = 2): Promise<lancedb.Table> {
-  const con = await lancedb.connect(opts)
-
-  const data = []
-  for (let i = 0; i < numRows; i++) {
-    const vector = []
-    for (let j = 0; j < numDimensions; j++) {
-      vector.push(i + (j * 0.1))
-    }
-    data.push({ id: i + 1, name: `name_${i}`, price: i + 10, is_active: (i % 2 === 0), vector })
-  }
-
-  return await con.createTable('vectors', data)
-}

node/src/test/test.ts

@@ -1,4 +1,4 @@
-// Copyright 2023 LanceDB Developers.
+// Copyright 2023 Lance Developers.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,17 +13,10 @@
 // limitations under the License.
 
 import { describe } from 'mocha'
+import { assert } from 'chai'
 import { track } from 'temp'
-import * as chai from 'chai'
-import * as chaiAsPromised from 'chai-as-promised'
 
 import * as lancedb from '../index'
-import { type AwsCredentials, type EmbeddingFunction, MetricType, WriteMode } from '../index'
-import { Query } from '../query'
-
-const expect = chai.expect
-const assert = chai.assert
-chai.use(chaiAsPromised)
 
 describe('LanceDB client', function () {
   describe('when creating a connection to lancedb', function () {
@@ -33,22 +26,6 @@ describe('LanceDB client', function () {
       assert.equal(con.uri, uri)
     })
 
-    it('should accept an options object', async function () {
-      const uri = await createTestDB()
-      const con = await lancedb.connect({ uri })
-      assert.equal(con.uri, uri)
-    })
-
-    it('should accept custom aws credentials', async function () {
-      const uri = await createTestDB()
-      const awsCredentials: AwsCredentials = {
-        accessKeyId: '',
-        secretKey: ''
-      }
-      const con = await lancedb.connect({ uri, awsCredentials })
-      assert.equal(con.uri, uri)
-    })
-
     it('should return the existing table names', async function () {
       const uri = await createTestDB()
       const con = await lancedb.connect(uri)
@@ -86,36 +63,13 @@ describe('LanceDB client', function () {
       assert.equal(results[0].id, 1)
     })
 
-    it('uses a filter / where clause', async function () {
-      // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
-      const assertResults = (results: Array<Record<string, unknown>>) => {
-        assert.equal(results.length, 1)
-        assert.equal(results[0].id, 2)
-      }
-
+    it('uses a filter', async function () {
       const uri = await createTestDB()
       const con = await lancedb.connect(uri)
       const table = await con.openTable('vectors')
-      let results = await table.search([0.1, 0.1]).filter('id == 2').execute()
-      assertResults(results)
-      results = await table.search([0.1, 0.1]).where('id == 2').execute()
-      assertResults(results)
-    })
-
-    it('select only a subset of columns', async function () {
-      const uri = await createTestDB()
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-      const results = await table.search([0.1, 0.1]).select(['is_active']).execute()
-      assert.equal(results.length, 2)
-      // vector and score are always returned
-      assert.isDefined(results[0].vector)
-      assert.isDefined(results[0].score)
-      assert.isDefined(results[0].is_active)
-
-      assert.isUndefined(results[0].id)
-      assert.isUndefined(results[0].name)
-      assert.isUndefined(results[0].price)
+      const results = await table.search([0.1, 0.3]).filter('id == 2').execute()
+      assert.equal(results.length, 1)
+      assert.equal(results[0].id, 2)
     })
   })
 
@@ -132,10 +86,12 @@ describe('LanceDB client', function () {
       const tableName = `vectors_${Math.floor(Math.random() * 100)}`
       const table = await con.createTable(tableName, data)
       assert.equal(table.name, tableName)
-      assert.equal(await table.countRows(), 2)
+
+      const results = await table.search([0.1, 0.3]).execute()
+      assert.equal(results.length, 2)
     })
 
-    it('use overwrite flag to overwrite existing table', async function () {
+    it('appends records to an existing table ', async function () {
       const dir = await track().mkdir('lancejs')
       const con = await lancedb.connect(dir)
 
@@ -144,40 +100,17 @@ describe('LanceDB client', function () {
         { id: 2, vector: [1.1, 1.2], price: 50 }
       ]
 
-      const tableName = 'overwrite'
-      await con.createTable(tableName, data, WriteMode.Create)
-
-      const newData = [
-        { id: 1, vector: [0.1, 0.2], price: 10 },
-        { id: 2, vector: [1.1, 1.2], price: 50 },
-        { id: 3, vector: [1.1, 1.2], price: 50 }
-      ]
-
-      await expect(con.createTable(tableName, newData)).to.be.rejectedWith(Error, 'already exists')
-
-      const table = await con.createTable(tableName, newData, WriteMode.Overwrite)
-      assert.equal(table.name, tableName)
-      assert.equal(await table.countRows(), 3)
-    })
-
-    it('appends records to an existing table ', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-
-      const data = [
-        { id: 1, vector: [0.1, 0.2], price: 10, name: 'a' },
-        { id: 2, vector: [1.1, 1.2], price: 50, name: 'b' }
-      ]
-
       const table = await con.createTable('vectors', data)
-      assert.equal(await table.countRows(), 2)
+      const results = await table.search([0.1, 0.3]).execute()
+      assert.equal(results.length, 2)
 
       const dataAdd = [
-        { id: 3, vector: [2.1, 2.2], price: 10, name: 'c' },
-        { id: 4, vector: [3.1, 3.2], price: 50, name: 'd' }
+        { id: 3, vector: [2.1, 2.2], price: 10 },
+        { id: 4, vector: [3.1, 3.2], price: 50 }
       ]
       await table.add(dataAdd)
-      assert.equal(await table.countRows(), 4)
+      const resultsAdd = await table.search([0.1, 0.3]).execute()
+      assert.equal(resultsAdd.length, 4)
     })
 
     it('overwrite all records in a table', async function () {
@@ -185,136 +118,29 @@ describe('LanceDB client', function () {
       const con = await lancedb.connect(uri)
 
       const table = await con.openTable('vectors')
-      assert.equal(await table.countRows(), 2)
+      const results = await table.search([0.1, 0.3]).execute()
+      assert.equal(results.length, 2)
 
       const dataOver = [
         { vector: [2.1, 2.2], price: 10, name: 'foo' },
         { vector: [3.1, 3.2], price: 50, name: 'bar' }
       ]
       await table.overwrite(dataOver)
-      assert.equal(await table.countRows(), 2)
-    })
-
-    it('can delete records from a table', async function () {
-      const uri = await createTestDB()
-      const con = await lancedb.connect(uri)
-
-      const table = await con.openTable('vectors')
-      assert.equal(await table.countRows(), 2)
-
-      await table.delete('price = 10')
-      assert.equal(await table.countRows(), 1)
-    })
-  })
-
-  describe('when creating a vector index', function () {
-    it('overwrite all records in a table', async function () {
-      const uri = await createTestDB(32, 300)
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
-    }).timeout(10_000) // Timeout is high partially because GH macos runner is pretty slow
-
-    it('replace an existing index', async function () {
-      const uri = await createTestDB(16, 300)
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-
-      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
-
-      // Replace should fail if the index already exists
-      await expect(table.createIndex({
-        type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2, replace: false
-      })
-      ).to.be.rejectedWith('LanceError(Index)')
-
-      // Default replace = true
-      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
-    }).timeout(50_000)
-  })
-
-  describe('when using a custom embedding function', function () {
-    class TextEmbedding implements EmbeddingFunction<string> {
-      sourceColumn: string
-
-      constructor (targetColumn: string) {
-        this.sourceColumn = targetColumn
-      }
-
-      _embedding_map = new Map<string, number[]>([
-        ['foo', [2.1, 2.2]],
-        ['bar', [3.1, 3.2]]
-      ])
-
-      async embed (data: string[]): Promise<number[][]> {
-        return data.map(datum => this._embedding_map.get(datum) ?? [0.0, 0.0])
-      }
-    }
-
-    it('should encode the original data into embeddings', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-      const embeddings = new TextEmbedding('name')
-
-      const data = [
-        { price: 10, name: 'foo' },
-        { price: 50, name: 'bar' }
-      ]
-      const table = await con.createTable('vectors', data, WriteMode.Create, embeddings)
-      const results = await table.search('foo').execute()
-      assert.equal(results.length, 2)
+      const resultsAdd = await table.search([0.1, 0.3]).execute()
+      assert.equal(resultsAdd.length, 2)
     })
   })
 })
 
-describe('Query object', function () {
-  it('sets custom parameters', async function () {
-    const query = new Query([0.1, 0.3])
-      .limit(1)
-      .metricType(MetricType.Cosine)
-      .refineFactor(100)
-      .select(['a', 'b'])
-      .nprobes(20) as Record<string, any>
-    assert.equal(query._limit, 1)
-    assert.equal(query._metricType, MetricType.Cosine)
-    assert.equal(query._refineFactor, 100)
-    assert.equal(query._nprobes, 20)
-    assert.deepEqual(query._select, ['a', 'b'])
-  })
-})
-
-async function createTestDB (numDimensions: number = 2, numRows: number = 2): Promise<string> {
+async function createTestDB (): Promise<string> {
   const dir = await track().mkdir('lancejs')
   const con = await lancedb.connect(dir)
 
-  const data = []
-  for (let i = 0; i < numRows; i++) {
-    const vector = []
-    for (let j = 0; j < numDimensions; j++) {
-      vector.push(i + (j * 0.1))
-    }
-    data.push({ id: i + 1, name: `name_${i}`, price: i + 10, is_active: (i % 2 === 0), vector })
-  }
+  const data = [
+    { id: 1, vector: [0.1, 0.2], name: 'foo', price: 10, is_active: true },
+    { id: 2, vector: [1.1, 1.2], name: 'bar', price: 50, is_active: false }
+  ]
 
   await con.createTable('vectors', data)
   return dir
 }
-
-describe('Drop table', function () {
-  it('drop a table', async function () {
-    const dir = await track().mkdir('lancejs')
-    const con = await lancedb.connect(dir)
-
-    const data = [
-      { price: 10, name: 'foo', vector: [1, 2, 3] },
-      { price: 50, name: 'bar', vector: [4, 5, 6] }
-    ]
-    await con.createTable('t1', data)
-    await con.createTable('t2', data)
-
-    assert.deepEqual(await con.tableNames(), ['t1', 't2'])
-
-    await con.dropTable('t1')
-    assert.deepEqual(await con.tableNames(), ['t2'])
-  })
-})