add dbg prints

.bumpversion.cfg

@@ -1,12 +0,0 @@
-[bumpversion]
-current_version = 0.1.13
-commit = True
-message = Bump version: {current_version} → {new_version}
-tag = True
-tag_name = v{new_version}
-
-[bumpversion:file:node/package.json]
-
-[bumpversion:file:rust/ffi/node/Cargo.toml]
-
-[bumpversion:file:rust/vectordb/Cargo.toml]

.github/workflows/cargo-publish.yml

@@ -1,29 +0,0 @@
-name: Cargo Publish
-
-on:
-  release:
-    types: [ published ]
-
-env:
-  # This env var is used by Swatinem/rust-cache@v2 for the cache
-  # key, so we set it to make sure it is always consistent.
-  CARGO_TERM_COLOR: always
-
-jobs:
-  build:
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    steps:
-      - uses: actions/checkout@v3
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Publish the package
-        run: |
-          cargo publish -p vectordb --all-features --token ${{ secrets.CARGO_REGISTRY_TOKEN }}

.github/workflows/docs.yml

@@ -39,28 +39,6 @@ jobs:
         run: |
           python -m pip install -e .
           python -m pip install -r ../docs/requirements.txt
-      - name: Set up node
-        uses: actions/setup-node@v3
-        with:
-          node-version: ${{ matrix.node-version }}
-          cache: 'npm'
-          cache-dependency-path: node/package-lock.json
-      - uses: Swatinem/rust-cache@v2
-      - name: Install node dependencies
-        working-directory: node
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Build node
-        working-directory: node
-        run: |
-          npm ci
-          npm run build
-          npm run tsc
-      - name: Create markdown files
-        working-directory: node
-        run: |
-          npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
       - name: Build docs
         run: |
           PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
@@ -72,4 +50,4 @@ jobs:
           path: "docs/site"
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v1
+        uses: actions/deploy-pages@v1

.github/workflows/docs_test.yml

@@ -1,93 +0,0 @@
-name: Documentation Code Testing
-
-on:
-  push:
-    branches:
-      - main
-    paths:
-      - docs/**
-      - .github/workflows/docs_test.yml
-  pull_request:
-    paths:
-      - docs/**
-      - .github/workflows/docs_test.yml
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-env:
-  # Disable full debug symbol generation to speed up CI build and keep memory down
-  # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1"
-  RUST_BACKTRACE: "1"
-
-jobs:
-  test-python:
-    name: Test doc python code
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        python-minor-version: [ "11" ]
-        os: ["ubuntu-22.04"]
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.${{ matrix.python-minor-version }}
-        cache: "pip"
-        cache-dependency-path: "docs/test/requirements.txt"
-    - name: Build Python
-      working-directory: docs/test
-      run:
-        python -m pip install -r requirements.txt
-    - name: Create test files
-      run: |
-        cd docs/test
-        python md_testing.py
-    - name: Test
-      run: |
-        cd docs/test/python
-        for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
-  test-node:
-    name: Test doc nodejs code
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        node-version: [ "18" ]
-        os: ["ubuntu-22.04"]
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Set up Node
-      uses: actions/setup-node@v3
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: Install dependecies needed for ubuntu
-      if: ${{ matrix.os == 'ubuntu-22.04' }}
-      run: |
-        sudo apt install -y protobuf-compiler libssl-dev
-    - name: Install node dependencies
-      run: |
-        cd docs/test
-        npm install 
-    - name: Rust cache
-      uses: swatinem/rust-cache@v2
-    - name: Install LanceDB
-      run: |
-        cd docs/test/node_modules/vectordb 
-        npm ci
-        npm run build-release
-        npm run tsc
-    - name: Create test files
-      run: |
-        cd docs/test
-        node md_testing.js
-    - name: Test
-      run: |
-        cd docs/test/node
-        for d in *; do cd "$d"; echo "$d".js; node "$d".js; cd ..; done

.github/workflows/make-release-commit.yml

@@ -1,55 +0,0 @@
-name: Create release commit
-
-on:
-  workflow_dispatch:
-    inputs:
-      dry_run:
-        description: 'Dry run (create the local commit/tags but do not push it)'
-        required: true
-        default: "false"
-        type: choice
-        options:
-          - "true"
-          - "false"
-      part:
-        description: 'What kind of release is this?'
-        required: true
-        default: 'patch'
-        type: choice
-        options:
-          - patch
-          - minor
-          - major
-
-jobs:
-  bump-version:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out main
-      uses: actions/checkout@v3
-      with:
-        ref: main
-        persist-credentials: false
-        fetch-depth: 0
-        lfs: true
-    - name: Set git configs for bumpversion
-      shell: bash
-      run: |
-        git config user.name 'Lance Release'
-        git config user.email 'lance-dev@lancedb.com'
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-    - name: Bump version, create tag and commit
-      run: |
-        pip install bump2version
-        bumpversion --verbose ${{ inputs.part }}
-    - name: Push new version and tag
-      if: ${{ inputs.dry_run }} == "false"
-      uses: ad-m/github-push-action@master
-      with:
-        github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-        branch: main
-        tags: true
-

.github/workflows/node.yml

@@ -71,8 +71,6 @@ jobs:
         npm run build
         npm run pack-build
         npm install --no-save ./dist/vectordb-*.tgz
-        # Remove index.node to test with dependency installed
-        rm index.node
     - name: Test
       run: npm run test
   macos:
@@ -102,8 +100,6 @@ jobs:
         npm run build
         npm run pack-build
         npm install --no-save ./dist/vectordb-*.tgz
-        # Remove index.node to test with dependency installed
-        rm index.node
     - name: Test
       run: |
         npm run test

.github/workflows/npm-publish.yml

@@ -1,171 +0,0 @@
-name: NPM Publish
-
-on:
-  release:
-    types: [ published ]
-
-jobs:
-  node:
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    defaults:
-      run:
-        shell: bash
-        working-directory: node
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 20
-          cache: 'npm'
-          cache-dependency-path: node/package-lock.json
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Build
-        run: |
-          npm ci
-          npm run tsc
-          npm pack
-      - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: node-package
-          path: |
-            node/vectordb-*.tgz
-
-  node-macos:
-    runs-on: macos-12
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64-apple-darwin, aarch64-apple-darwin]
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-    - name: Install system dependencies
-      run: brew install protobuf
-    - name: Install npm dependencies
-      run: |
-        cd node
-        npm ci
-    - name: Install rustup target
-      if: ${{ matrix.target == 'aarch64-apple-darwin' }}
-      run: rustup target add aarch64-apple-darwin
-    - name: Build MacOS native node modules
-      run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
-    - name: Upload Darwin Artifacts
-      uses: actions/upload-artifact@v3
-      with:
-        name: darwin-native
-        path: |
-          node/dist/vectordb-darwin*.tgz
-
-  node-linux:
-    name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }})
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        libc:
-          - gnu
-          # TODO: re-enable musl once we have refactored to pre-built containers
-          # Right now we have to build node from source which is too expensive.
-          # - musl
-        arch:
-          - x86_64
-          # Building on aarch64 is too slow for now
-          # - aarch64
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-    - name: Change owner to root (for npm)
-      # The docker container is run as root, so we need the files to be owned by root
-      # Otherwise npm is a nightmare: https://github.com/npm/cli/issues/3773
-      run: sudo chown -R root:root .
-    - name: Set up QEMU
-      if: ${{ matrix.arch == 'aarch64' }}
-      uses: docker/setup-qemu-action@v2
-      with:
-        platforms: arm64
-    - name: Build Linux GNU native node modules
-      if: ${{ matrix.libc == 'gnu' }}
-      run: |
-        docker run \
-          -v $(pwd):/io -w /io \
-          rust:1.70-bookworm \
-          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu
-    - name: Build musl Linux native node modules
-      if: ${{ matrix.libc == 'musl' }}
-      run: |
-        docker run --platform linux/arm64/v8 \
-          -v $(pwd):/io -w /io \
-          quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
-          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
-    - name: Upload Linux Artifacts
-      uses: actions/upload-artifact@v3
-      with:
-        name: linux-native
-        path: |
-          node/dist/vectordb-linux*.tgz
-
-  node-windows:
-    runs-on: windows-2022
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64-pc-windows-msvc]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Install Protoc v21.12
-        working-directory: C:\
-        run: |
-          New-Item -Path 'C:\protoc' -ItemType Directory
-          Set-Location C:\protoc
-          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-          7z x protoc.zip
-          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-        shell: powershell
-      - name: Install npm dependencies
-        run: |
-          cd node
-          npm ci
-      - name: Build Windows native node modules
-        run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }}
-      - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: windows-native
-          path: |
-            node/dist/vectordb-win32*.tgz
-
-  release:
-    needs: [node, node-macos, node-linux]
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    steps:
-      - uses: actions/download-artifact@v3
-      - name: Display structure of downloaded files
-        run: ls -R
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 20
-      - name: Publish to NPM
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
-        run: |
-          mv */*.tgz .
-          for filename in *.tgz; do
-            npm publish $filename
-          done

.github/workflows/pypi-publish.yml

@@ -1,31 +0,0 @@
-name: PyPI Publish
-
-on:
-  release:
-    types: [ published ]
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the python-make-release action
-    if: startsWith(github.ref, 'refs/tags/python-v')
-    defaults:
-      run:
-        shell: bash
-        working-directory: python
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.8"
-      - name: Build distribution
-        run: |
-          ls -la
-          pip install wheel setuptools --upgrade
-          python setup.py sdist bdist_wheel
-      - name: Publish
-        uses: pypa/gh-action-pypi-publish@v1.8.5
-        with:
-          password: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
-          packages-dir: python/dist

.github/workflows/python-make-release-commit.yml

@@ -1,56 +0,0 @@
-name: Python - Create release commit
-
-on:
-  workflow_dispatch:
-    inputs:
-      dry_run:
-        description: 'Dry run (create the local commit/tags but do not push it)'
-        required: true
-        default: "false"
-        type: choice
-        options:
-          - "true"
-          - "false"
-      part:
-        description: 'What kind of release is this?'
-        required: true
-        default: 'patch'
-        type: choice
-        options:
-          - patch
-          - minor
-          - major
-
-jobs:
-  bump-version:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out main
-      uses: actions/checkout@v3
-      with:
-        ref: main
-        persist-credentials: false
-        fetch-depth: 0
-        lfs: true
-    - name: Set git configs for bumpversion
-      shell: bash
-      run: |
-        git config user.name 'Lance Release'
-        git config user.email 'lance-dev@lancedb.com'
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-    - name: Bump version, create tag and commit
-      working-directory: python
-      run: |
-        pip install bump2version
-        bumpversion --verbose ${{ inputs.part }}
-    - name: Push new version and tag
-      if: ${{ inputs.dry_run }} == "false"
-      uses: ad-m/github-push-action@master
-      with:
-        github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-        branch: main
-        tags: true
-

.github/workflows/python.yml

@@ -30,17 +30,10 @@ jobs:
         python-version: 3.${{ matrix.python-minor-version }}
     - name: Install lancedb
       run: |
-        pip install -e .
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock black isort
-    - name: Black
-      run: black --check --diff --no-color --quiet .
-    - name: isort
-      run: isort --check --diff --quiet .
+        pip install -e ".[fts]"
+        pip install pytest
     - name: Run tests
       run: pytest -x -v --durations=30 tests
-    - name: doctest
-      run: pytest --doctest-modules lancedb
   mac:
     timeout-minutes: 30
     runs-on: "macos-12"
@@ -59,10 +52,7 @@ jobs:
         python-version: "3.11"
     - name: Install lancedb
       run: |
-        pip install -e .
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock black
-    - name: Black
-      run: black --check --diff --no-color --quiet .
+        pip install -e ".[fts]"
+        pip install pytest
     - name: Run tests
-      run: pytest -x -v --durations=30 tests
+      run: pytest -x -v --durations=30 tests

.github/workflows/release.yml

@@ -0,0 +1,194 @@
+name: Prepare Release
+
+# Based on https://github.com/dherman/neon-prebuild-example/blob/eaa4d33d682e5eb7abbc3da7aed153a1b1acb1b3/.github/workflows/publish.yml
+
+on:
+  push:
+    tags:
+      - v*
+
+jobs:
+  draft-release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          prerelease: true # hardcoded on for now
+          generate_release_notes: true
+  
+  rust:
+    runs-on: ubuntu-latest
+    needs: draft-release
+    defaults:
+      run:
+        shell: bash
+        working-directory: rust/vectordb
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Package Rust
+        run: cargo package --all-features
+      - uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          files: target/package/vectordb-*.crate
+          fail_on_unmatched_files: true
+    
+  python:
+    runs-on: ubuntu-latest
+    needs: draft-release
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Build wheel
+        run: |
+          pip install wheel
+          python setup.py sdist bdist_wheel
+      - uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          files: |
+            python/dist/lancedb-*.tar.gz 
+            python/dist/lancedb-*.whl
+          fail_on_unmatched_files: true
+
+  node:
+    runs-on: ubuntu-latest
+    needs: draft-release
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          cache: 'npm'
+          cache-dependency-path: node/package-lock.json
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Build
+        run: |
+          npm ci
+          npm run tsc
+          npm pack
+      - uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          files: node/vectordb-*.tgz
+          fail_on_unmatched_files: true
+
+  node-macos:
+    runs-on: macos-12
+    needs: draft-release
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64-apple-darwin, aarch64-apple-darwin]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Install system dependencies
+      run: brew install protobuf
+    - name: Install npm dependencies
+      run: |
+        cd node
+        npm ci
+    - name: Build MacOS native node modules
+      run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
+    - uses: softprops/action-gh-release@v1
+      with:
+        draft: true
+        files: node/dist/vectordb-darwin*.tgz
+        fail_on_unmatched_files: true
+
+  node-linux:
+    name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }})
+    runs-on: ubuntu-latest
+    needs: draft-release
+    strategy:
+      fail-fast: false
+      matrix:
+        libc:
+          - gnu
+          # TODO: re-enable musl once we have refactored to pre-built containers
+          # Right now we have to build node from source which is too expensive.
+          # - musl
+        arch:
+          - x86_64
+          - aarch64
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Set up QEMU
+      if: ${{ matrix.arch == 'aarch64' }}
+      uses: docker/setup-qemu-action@v2  
+      with:
+        platforms: arm64
+    - name: Build Linux GNU native node modules
+      if: ${{ matrix.libc == 'gnu' }}
+      run: |
+        docker run \
+          -v $(pwd):/io -w /io \
+          quay.io/pypa/manylinux2014_${{ matrix.arch }} \
+          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu
+    - name: Build musl Linux native node modules
+      if: ${{ matrix.libc == 'musl' }}
+      run: |
+        docker run --platform linux/arm64/v8 \
+          -v $(pwd):/io -w /io \
+          quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
+          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
+    - uses: softprops/action-gh-release@v1
+      with:
+        draft: true
+        files: node/dist/vectordb-linux*.tgz
+        fail_on_unmatched_files: true
+
+  release:
+    needs: [python, node, node-macos, node-linux, rust]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v3
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: |
+          python -m twine upload --non-interactive \
+            --skip-existing \
+            --repository testpypi python/dist/*
+      - name: Publish to NPM
+        run: |
+          for filename in node/dist/*.tgz; do
+            npm publish --dry-run $filename
+          done
+      - name: Publish to crates.io
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+        run: |
+          cargo publish --dry-run --no-verify rust/target/vectordb-*.crate
+      # - uses: softprops/action-gh-release@v1
+      #   with:
+      #     draft: false

.github/workflows/rust.yml

@@ -1,89 +0,0 @@
-name: Rust
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths:
-      - Cargo.toml
-      - rust/**
-      - .github/workflows/rust.yml
-
-env:
-  # This env var is used by Swatinem/rust-cache@v2 for the cache
-  # key, so we set it to make sure it is always consistent.
-  CARGO_TERM_COLOR: always
-  # Disable full debug symbol generation to speed up CI build and keep memory down
-  # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1"
-  RUST_BACKTRACE: "1"
-
-jobs:
-  linux:
-    timeout-minutes: 30
-    runs-on: ubuntu-22.04
-    defaults:
-      run:
-        shell: bash
-        working-directory: rust
-    steps:
-    - uses: actions/checkout@v3
-      with:
-          fetch-depth: 0
-          lfs: true
-    - uses: Swatinem/rust-cache@v2
-      with:
-          workspaces: rust
-    - name: Install dependencies
-      run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-    - name: Build
-      run: cargo build --all-features
-    - name: Run tests
-      run: cargo test --all-features
-  macos:
-    runs-on: macos-12
-    timeout-minutes: 30
-    defaults:
-      run:
-        shell: bash
-        working-directory: rust
-    steps:
-      - uses: actions/checkout@v3
-        with:
-            fetch-depth: 0
-            lfs: true
-      - name: CPU features
-        run: sysctl -a | grep cpu
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
-      - name: Install dependencies
-        run: brew install protobuf
-      - name: Build
-        run: cargo build --all-features
-      - name: Run tests
-        run: cargo test --all-features
-  windows:
-    runs-on: windows-2022
-    steps:
-      - uses: actions/checkout@v3
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
-      - name: Install Protoc v21.12
-        working-directory: C:\
-        run: |
-          New-Item -Path 'C:\protoc' -ItemType Directory
-          Set-Location C:\protoc
-          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-          7z x protoc.zip
-          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-        shell: powershell
-      - name: Run tests
-        run: |
-          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
-          cargo build
-          cargo test

.gitignore

@@ -3,7 +3,6 @@
 *.egg-info
 **/__pycache__
 .DS_Store
-venv
 
 .vscode
 
@@ -18,7 +17,7 @@ site
 python/build
 python/dist
 
-**/.ipynb_checkpoints
+notebooks/.ipynb_checkpoints
 
 **/.hypothesis
 
@@ -33,4 +32,3 @@ node/examples/**/dist
 ## Rust
 target
 
-Cargo.lock

.pre-commit-config.yaml

@@ -8,14 +8,4 @@ repos:
 -   repo: https://github.com/psf/black
     rev: 22.12.0
     hooks:
-    -   id: black
--   repo: https://github.com/astral-sh/ruff-pre-commit
-    # Ruff version.
-    rev: v0.0.277
-    hooks:
-    - id: ruff
--   repo: https://github.com/pycqa/isort
-    rev: 5.12.0
-    hooks:
-    - id: isort
-      name: isort (python)
+    -   id: black

Cargo.toml

@@ -4,13 +4,3 @@ members = [
     "rust/ffi/node"
 ]
 resolver = "2"
-
-[workspace.dependencies]
-lance = "=0.5.8"
-arrow-array = "42.0"
-arrow-data = "42.0"
-arrow-schema = "42.0"
-arrow-ipc = "42.0"
-half = { "version" = "2.2.1", default-features = false }
-object_store = "0.6.1"
-

Cross.toml

@@ -0,0 +1,39 @@
+# These make sure our builds are compatible with old glibc versions.
+[target.x86_64-unknown-linux-gnu]
+pre-build = [
+    # Install newer gfortran
+    "yum install -y openssl-devel unzip gcc-gfortran",
+    "scl enable devtoolset-11 bash",
+    # protobuf is too old, so we directly download binaries
+    "PB_REL=https://github.com/protocolbuffers/protobuf/releases",
+    "PB_VERSION=23.1",
+    "curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip",
+    "unzip protoc-$PB_VERSION-linux-x86_64.zip -d /usr/local",
+]
+image = "ghcr.io/cross-rs/x86_64-unknown-linux-gnu:main-centos"
+
+[target.aarch64-unknown-linux-gnu]
+pre-build = [
+    "yum install -y openssl-devel unzip",
+    # protobuf is too old, so we directly download binaries
+    "PB_REL=https://github.com/protocolbuffers/protobuf/releases",
+    "PB_VERSION=23.1",
+    "curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip",
+    "unzip protoc-$PB_VERSION-linux-x86_64.zip -d /usr/local",
+]
+# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.aarch64-unknown-linux-gnu.centos
+image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main-centos"
+
+[target.x86_64-unknown-linux-musl]
+# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.x86_64-unknown-linux-musl
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH",
+]
+
+[target.aarch64-unknown-linux-musl]
+# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.aarch64-unknown-linux-musl
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH",
+]

README.md

@@ -10,10 +10,6 @@
 <a href="https://discord.gg/zMM32dvNtd">Discord</a> •
 <a href="https://twitter.com/lancedb">Twitter</a>
 
-</p>
-
-<img max-width="750px" alt="LanceDB Multimodal Search" src="https://github.com/lancedb/lancedb/assets/917119/09c5afc5-7816-4687-bae4-f2ca194426ec">
-
 </p>
 </div>
 
@@ -27,15 +23,13 @@ The key features of LanceDB include:
 
 * Store, query and filter vectors, metadata and multi-modal data (text, images, videos, point clouds, and more).
 
-* Support for vector similarity search, full-text search and SQL.
-
 * Native Python and Javascript/Typescript support.
 
 * Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure.
 
 * Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lanecdb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
 
-LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
+LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/eto-ai/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
 
 ## Quick Start
 
@@ -65,7 +59,7 @@ pip install lancedb
 ```python
 import lancedb
 
-uri = "data/sample-lancedb"
+uri = "/tmp/lancedb"
 db = lancedb.connect(uri)
 table = db.create_table("my_table",
                          data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
@@ -75,4 +69,4 @@ result = table.search([100, 100]).limit(2).to_df()
 
 ## Blogs, Tutorials & Videos
 * 📈 <a href="https://blog.eto.ai/benchmarking-random-access-in-lance-ed690757a826">2000x better performance with Lance over Parquet</a>
-* 🤖 <a href="https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb">Build a question and answer bot with LanceDB</a>
+* 🤖 <a href="https://github.com/lancedb/lancedb/blob/main/notebooks/youtube_transcript_search.ipynb">Build a question and answer bot with LanceDB</a>

ci/build_linux_artifacts.sh

@@ -17,10 +17,22 @@ setup_dependencies() {
         # musllinux
         apk add openssl-dev
     else
-        # rust / debian
-        apt update
-        apt install -y libssl-dev protobuf-compiler
+        # manylinux2014
+        yum install -y openssl-devel unzip   
     fi
+
+    if [[ $1 == x86_64* ]]; then
+        ARCH=x86_64
+    else
+        # gnu target
+        ARCH=aarch_64
+    fi
+    
+    # Install new enough protobuf (yum-provided is old)
+    PB_REL=https://github.com/protocolbuffers/protobuf/releases
+    PB_VERSION=23.1
+    curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
+    unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local
 }
 
 install_node() {
@@ -32,30 +44,40 @@ install_node() {
         # This node version is 15, we need 16 or higher:
         # apk add nodejs-current npm 
         # So instead we install from source (nvm doesn't provide binaries for musl):
-        nvm install -s --no-progress 17
+        nvm install -s 17
     else
-        nvm install --no-progress 17 # latest that supports glibc 2.17
+        nvm install 17 # latest that supports glibc 2.17
     fi
+
+    printenv
+    echo "Node version:"
+    npm --version
+    which npm
+    which node
+}
+
+install_rust() {
+    echo "Installing rust..."
+    curl https://sh.rustup.rs -sSf | bash -s -- -y
+
+    printenv
+
+    export PATH="$PATH:/root/.cargo/bin"
+
+    printenv
 }
 
 build_node_binary() {
     echo "Building node library for $1..."
     pushd node
-
-    npm ci
     
     if [[ $1 == *musl ]]; then
         # This is needed for cargo to allow build cdylibs with musl
         export RUSTFLAGS="-C target-feature=-crt-static"
     fi
-
-    # Cargo can run out of memory while pulling dependencies, especially when running
-    # in QEMU. This is a workaround for that.
-    export CARGO_NET_GIT_FETCH_WITH_CLI=true
-
     # We don't pass in target, since the native target here already matches
-    # We need to pass OPENSSL_LIB_DIR and OPENSSL_INCLUDE_DIR for static build to work https://github.com/sfackler/rust-openssl/issues/877
-    OPENSSL_STATIC=1 OPENSSL_LIB_DIR=/usr/lib/x86_64-linux-gnu OPENSSL_INCLUDE_DIR=/usr/include/openssl/ npm run build-release
+    # and openblas-src doesn't do well with cross-compilation.
+    npm run build-release
     npm run pack-build
 
     popd
@@ -69,4 +91,5 @@ TARGET=${1:-x86_64-unknown-linux-gnu}
 
 setup_dependencies $TARGET
 install_node $TARGET
+install_rust
 build_node_binary $TARGET

ci/build_macos_artifacts.sh

@@ -1,21 +1,16 @@
 # Builds the macOS artifacts (node binaries).
-# Usage: ./ci/build_macos_artifacts.sh [target]
+# Usage: ./build_macos_artifacts.sh [target]
 # Targets supported: x86_64-apple-darwin aarch64-apple-darwin
 
-prebuild_rust() {
-    # Building here for the sake of easier debugging.
-    pushd rust/ffi/node
-    echo "Building rust library for $1"
-    export RUST_BACKTRACE=1
-    cargo build --release --target $1
-    popd
-}
-
 build_node_binaries() {
     pushd node
-    echo "Building node library for $1"
-    npm run build-release -- --target $1
-    npm run pack-build -- --target $1
+    
+    for target in $1
+    do
+        echo "Building node library for $target"
+        npm run build-release -- --target $target
+        npm run pack-build -- --target $target
+    done
     popd
 }
 
@@ -24,10 +19,4 @@ if [ -n "$1" ]; then
 else
     targets="x86_64-apple-darwin aarch64-apple-darwin"
 fi
-
-echo "Building artifacts for targets: $targets"
-for target in $targets
-    do
-    prebuild_rust $target
-    build_node_binaries $target
-done
+build_node_binaries $targets

ci/build_windows_artifacts.ps1

@@ -1,41 +0,0 @@
-# Builds the Windows artifacts (node binaries).
-# Usage:  .\ci\build_windows_artifacts.ps1 [target]
-# Targets supported:
-# - x86_64-pc-windows-msvc
-# - i686-pc-windows-msvc
-
-function Prebuild-Rust {
-    param (
-        [string]$target
-    )
-
-    # Building here for the sake of easier debugging.
-    Push-Location -Path "rust/ffi/node"
-    Write-Host "Building rust library for $target"
-    $env:RUST_BACKTRACE=1
-    cargo build --release --target $target
-    Pop-Location
-}
-
-function Build-NodeBinaries {
-    param (
-        [string]$target
-    )
-
-    Push-Location -Path "node"
-    Write-Host "Building node library for $target"
-    npm run build-release -- --target $target
-    npm run pack-build -- --target $target
-    Pop-Location
-}
-
-$targets = $args[0]
-if (-not $targets) {
-    $targets = "x86_64-pc-windows-msvc"
-}
-
-Write-Host "Building artifacts for targets: $targets"
-foreach ($target in $targets) {
-    Prebuild-Rust $target
-    Build-NodeBinaries $target
-}

ci/release_process.md

@@ -0,0 +1,121 @@
+
+How to release the node module
+
+### 1. Bump the versions
+
+<!-- TODO: we also need to bump the optional dependencies for node! -->
+
+```shell
+pushd rust/vectordb
+cargo bump minor
+popd
+
+pushd rust/ffi/node
+cargo bump minor
+popd
+
+pushd python
+cargo bump minor
+popd
+
+pushd node
+npm version minor
+popd
+
+git add -u
+git commit -m "Bump versions"
+git push
+```
+
+### 2. Push a new tag
+
+```shell
+git tag vX.X.X
+git push --tag vX.X.X
+```
+
+When the tag is pushed, GitHub actions will start building the libraries and
+will upload them to a draft release. Wait for those jobs to complete.
+
+### 3. Publish the release
+
+Once the jobs are complete, you can edit the 
+
+2. Push a tag, such as vX.X.X. Once the tag is pushrf, GitHub actions will start
+   building the native libraries and uploading them to a draft release. Wait for
+   those jobs to complete.
+3. If the libraries are successful, edit the changelog and then publish the
+   release. Once you publish, a new action will start and upload all the 
+   release artifacts to npm.
+
+## Manual process
+
+You can build the artifacts locally on a MacOS machine.
+
+### Build the MacOS release libraries
+
+One-time setup:
+
+```shell
+rustup target add x86_64-apple-darwin aarch64-apple-darwin
+```
+
+To build:
+
+```shell
+bash ci/build_macos_artifacts.sh
+```
+
+### Build the Linux release libraries
+
+To build a Linux library, we need to use docker with a different build script:
+
+```shell
+ARCH=aarch64
+docker run \
+    -v $(pwd):/io -w /io \
+    quay.io/pypa/manylinux2014_$ARCH \
+    bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-gnu
+```
+
+You can change `ARCH` to `x86_64`.
+
+Similar script for musl binaries:
+
+```shell
+ARCH=aarch64
+docker run \
+    -v $(pwd):/io -w /io \
+    quay.io/pypa/musllinux_1_1_$ARCH \
+    bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-musl
+```
+
+<!--
+
+For debugging, use these snippets:
+
+```shell
+ARCH=aarch64
+docker run -it \
+    -v $(pwd):/io -w /io \
+    quay.io/pypa/manylinux2014_$ARCH \
+    bash
+```
+
+```shell
+ARCH=aarch64
+docker run -it \
+    -v $(pwd):/io -w /io \
+    quay.io/pypa/musllinux_1_1_$ARCH \
+    bash
+```
+
+Note: musllinux_1_1 is Alpine Linux 3.12
+-->
+
+```
+docker run \
+    -v $(pwd):/io -w /io \
+    quay.io/pypa/musllinux_1_1_aarch64 \
+    bash alpine_repro.sh
+```

docs/mkdocs.yml

@@ -1,84 +1,33 @@
-site_name: LanceDB Docs
-repo_url: https://github.com/lancedb/lancedb
-repo_name: lancedb/lancedb
+site_name: LanceDB Documentation
 docs_dir: src
 
 theme:
   name: "material"
-  logo: assets/logo.png
-  favicon: assets/logo.png
   features:
     - content.code.copy
-    - content.tabs.link
-  icon:
-    repo: fontawesome/brands/github
-  custom_dir: overrides
 
 plugins:
 - search
-- autorefs
 - mkdocstrings:
     handlers:
       python:
         paths: [../python]
-        selection:
-          docstring_style: numpy
-        rendering:
-          heading_level: 4
-          show_source: false
-          show_symbol_type_in_heading: true
-          show_signature_annotations: true
-          show_root_heading: true
-          members_order: source
-        import:
-          # for cross references
-          - https://arrow.apache.org/docs/objects.inv
-          - https://pandas.pydata.org/docs/objects.inv
 - mkdocs-jupyter
 
+nav:
+- Home: index.md
+- Basics: basic.md
+- Embeddings: embedding.md
+- Indexing: ann_indexes.md
+- Full-text search: fts.md
+- Integrations: integrations.md
+- Python API: python.md
+
 markdown_extensions:
-- admonition
-- footnotes
-- pymdownx.superfences
-- pymdownx.details
 - pymdownx.highlight:
     anchor_linenums: true
     line_spans: __span
     pygments_lang_class: true
 - pymdownx.inlinehilite
 - pymdownx.snippets
-- pymdownx.superfences
-- pymdownx.tabbed:
-    alternate_style: true
-- md_in_html
-
-nav:
-- Home: index.md
-- Basics: basic.md
-- Embeddings: embedding.md
-- Python full-text search: fts.md
-- Python integrations:
-  - Pandas and PyArrow: python/arrow.md
-  - DuckDB: python/duckdb.md
-  - LangChain 🦜️🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
-  - LlamaIndex 🦙: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
-  - Pydantic: python/pydantic.md
-- Python examples:
-  - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
-  - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
-  - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
-  - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
-  - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
-- Javascript examples:
-  - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
-  - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
-- References:
-  - Vector Search: search.md
-  - SQL filters: sql.md
-  - Indexing: ann_indexes.md
-- API references:
-  - Python API: python/python.md
-  - Javascript API: javascript/modules.md
-
-extra_css:
-  - styles/global.css
+- pymdownx.superfences

docs/overrides/partials/header.html

@@ -1,176 +0,0 @@
-<!--
-  Copyright (c) 2016-2023 Martin Donath <martin.donath@squidfunk.com>
-
-  Permission is hereby granted, free of charge, to any person obtaining a copy
-  of this software and associated documentation files (the "Software"), to
-  deal in the Software without restriction, including without limitation the
-  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-  sell copies of the Software, and to permit persons to whom the Software is
-  furnished to do so, subject to the following conditions:
-
-  The above copyright notice and this permission notice shall be included in
-  all copies or substantial portions of the Software.
-
-  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-  FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
-  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-  IN THE SOFTWARE.
--->
-
-{% set class = "md-header" %}
-{% if "navigation.tabs.sticky" in features %}
-  {% set class = class ~ " md-header--shadow md-header--lifted" %}
-{% elif "navigation.tabs" not in features %}
-  {% set class = class ~ " md-header--shadow" %}
-{% endif %}
-
-<!-- Header -->
-<header class="{{ class }}" data-md-component="header">
-  <nav
-    class="md-header__inner md-grid"
-    aria-label="{{ lang.t('header') }}"
-  >
-
-    <!-- Link to home -->
-    <a
-      href="{{ config.extra.homepage | d(nav.homepage.url, true) | url }}"
-      title="{{ config.site_name | e }}"
-      class="md-header__button md-logo"
-      aria-label="{{ config.site_name }}"
-      data-md-component="logo"
-    >
-      {% include "partials/logo.html" %}
-    </a>
-
-    <!-- Button to open drawer -->
-    <label class="md-header__button md-icon" for="__drawer">
-      {% include ".icons/material/menu" ~ ".svg" %}
-    </label>
-
-    <!-- Header title -->
-    <div class="md-header__title" style="width: auto !important;" data-md-component="header-title">
-      <div class="md-header__ellipsis">
-        <div class="md-header__topic">
-          <span class="md-ellipsis">
-            {{ config.site_name }}
-          </span>
-        </div>
-        <div class="md-header__topic" data-md-component="header-topic">
-          <span class="md-ellipsis">
-            {% if page.meta and page.meta.title %}
-              {{ page.meta.title }}
-            {% else %}
-              {{ page.title }}
-            {% endif %}
-          </span>
-        </div>
-      </div>
-    </div>
-
-    <!-- Color palette -->
-    {% if config.theme.palette %}
-      {% if not config.theme.palette is mapping %}
-        <form class="md-header__option" data-md-component="palette">
-          {% for option in config.theme.palette %}
-            {% set scheme  = option.scheme  | d("default", true) %}
-            {% set primary = option.primary | d("indigo", true) %}
-            {% set accent  = option.accent  | d("indigo", true) %}
-            <input
-              class="md-option"
-              data-md-color-media="{{ option.media }}"
-              data-md-color-scheme="{{ scheme | replace(' ', '-') }}"
-              data-md-color-primary="{{ primary | replace(' ', '-') }}"
-              data-md-color-accent="{{ accent | replace(' ', '-') }}"
-              {% if option.toggle %}
-                aria-label="{{ option.toggle.name }}"
-              {% else  %}
-                aria-hidden="true"
-              {% endif %}
-              type="radio"
-              name="__palette"
-              id="__palette_{{ loop.index }}"
-            />
-            {% if option.toggle %}
-              <label
-                class="md-header__button md-icon"
-                title="{{ option.toggle.name }}"
-                for="__palette_{{ loop.index0 or loop.length }}"
-                hidden
-              >
-                {% include ".icons/" ~ option.toggle.icon ~ ".svg" %}
-              </label>
-            {% endif %}
-          {% endfor %}
-        </form>
-      {% endif %}
-    {% endif %}
-
-    <!-- Site language selector -->
-    {% if config.extra.alternate %}
-      <div class="md-header__option">
-        <div class="md-select">
-          {% set icon = config.theme.icon.alternate or "material/translate" %}
-          <button
-            class="md-header__button md-icon"
-            aria-label="{{ lang.t('select.language') }}"
-          >
-            {% include ".icons/" ~ icon ~ ".svg" %}
-          </button>
-          <div class="md-select__inner">
-            <ul class="md-select__list">
-              {% for alt in config.extra.alternate %}
-                <li class="md-select__item">
-                  <a
-                    href="{{ alt.link | url }}"
-                    hreflang="{{ alt.lang }}"
-                    class="md-select__link"
-                  >
-                    {{ alt.name }}
-                  </a>
-                </li>
-              {% endfor %}
-            </ul>
-          </div>
-        </div>
-      </div>
-    {% endif %}
-
-    <!-- Button to open search modal -->
-    {% if "material/search" in config.plugins %}
-      <label class="md-header__button md-icon" for="__search">
-        {% include ".icons/material/magnify.svg" %}
-      </label>
-
-      <!-- Search interface -->
-      {% include "partials/search.html" %}
-    {% endif %}
-
-    <div style="margin-left: 10px; margin-right: 5px;">
-        <a href="https://discord.com/invite/zMM32dvNtd" target="_blank" rel="noopener noreferrer">
-            <svg fill="#FFFFFF" xmlns="http://www.w3.org/2000/svg"  viewBox="0 0 50 50" width="25px" height="25px"><path d="M 41.625 10.769531 C 37.644531 7.566406 31.347656 7.023438 31.078125 7.003906 C 30.660156 6.96875 30.261719 7.203125 30.089844 7.589844 C 30.074219 7.613281 29.9375 7.929688 29.785156 8.421875 C 32.417969 8.867188 35.652344 9.761719 38.578125 11.578125 C 39.046875 11.867188 39.191406 12.484375 38.902344 12.953125 C 38.710938 13.261719 38.386719 13.429688 38.050781 13.429688 C 37.871094 13.429688 37.6875 13.378906 37.523438 13.277344 C 32.492188 10.15625 26.210938 10 25 10 C 23.789063 10 17.503906 10.15625 12.476563 13.277344 C 12.007813 13.570313 11.390625 13.425781 11.101563 12.957031 C 10.808594 12.484375 10.953125 11.871094 11.421875 11.578125 C 14.347656 9.765625 17.582031 8.867188 20.214844 8.425781 C 20.0625 7.929688 19.925781 7.617188 19.914063 7.589844 C 19.738281 7.203125 19.34375 6.960938 18.921875 7.003906 C 18.652344 7.023438 12.355469 7.566406 8.320313 10.8125 C 6.214844 12.761719 2 24.152344 2 34 C 2 34.175781 2.046875 34.34375 2.132813 34.496094 C 5.039063 39.605469 12.972656 40.941406 14.78125 41 C 14.789063 41 14.800781 41 14.8125 41 C 15.132813 41 15.433594 40.847656 15.621094 40.589844 L 17.449219 38.074219 C 12.515625 36.800781 9.996094 34.636719 9.851563 34.507813 C 9.4375 34.144531 9.398438 33.511719 9.765625 33.097656 C 10.128906 32.683594 10.761719 32.644531 11.175781 33.007813 C 11.234375 33.0625 15.875 37 25 37 C 34.140625 37 38.78125 33.046875 38.828125 33.007813 C 39.242188 32.648438 39.871094 32.683594 40.238281 33.101563 C 40.601563 33.515625 40.5625 34.144531 40.148438 34.507813 C 40.003906 34.636719 37.484375 36.800781 32.550781 38.074219 L 34.378906 40.589844 C 34.566406 40.847656 34.867188 41 35.1875 41 C 35.199219 41 35.210938 41 35.21875 41 C 37.027344 40.941406 44.960938 39.605469 47.867188 34.496094 C 47.953125 34.34375 48 34.175781 48 34 C 48 24.152344 43.785156 12.761719 41.625 10.769531 Z M 18.5 30 C 16.566406 30 15 28.210938 15 26 C 15 23.789063 16.566406 22 18.5 22 C 20.433594 22 22 23.789063 22 26 C 22 28.210938 20.433594 30 18.5 30 Z M 31.5 30 C 29.566406 30 28 28.210938 28 26 C 28 23.789063 29.566406 22 31.5 22 C 33.433594 22 35 23.789063 35 26 C 35 28.210938 33.433594 30 31.5 30 Z"/></svg>
-        </a>
-    </div>
-    <div style="margin-left: 5px; margin-right: 5px;">
-        <a href="https://twitter.com/lancedb" target="_blank" rel="noopener noreferrer">
-            <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0,0,256,256" width="25px" height="25px" fill-rule="nonzero"><g fill-opacity="0" fill="#ffffff" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><path d="M0,256v-256h256v256z" id="bgRectangle"></path></g><g fill="#ffffff" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(4,4)"><path d="M57,17.114c-1.32,1.973 -2.991,3.707 -4.916,5.097c0.018,0.423 0.028,0.847 0.028,1.274c0,13.013 -9.902,28.018 -28.016,28.018c-5.562,0 -12.81,-1.948 -15.095,-4.423c0.772,0.092 1.556,0.138 2.35,0.138c4.615,0 8.861,-1.575 12.23,-4.216c-4.309,-0.079 -7.946,-2.928 -9.199,-6.84c1.96,0.308 4.447,-0.17 4.447,-0.17c0,0 -7.7,-1.322 -7.899,-9.779c2.226,1.291 4.46,1.231 4.46,1.231c0,0 -4.441,-2.734 -4.379,-8.195c0.037,-3.221 1.331,-4.953 1.331,-4.953c8.414,10.361 20.298,10.29 20.298,10.29c0,0 -0.255,-1.471 -0.255,-2.243c0,-5.437 4.408,-9.847 9.847,-9.847c2.832,0 5.391,1.196 7.187,3.111c2.245,-0.443 4.353,-1.263 6.255,-2.391c-0.859,3.44 -4.329,5.448 -4.329,5.448c0,0 2.969,-0.329 5.655,-1.55z"></path></g></g></svg>
-        </a>
-    </div>
-
-    <!-- Repository information -->
-    {% if config.repo_url %}
-      <div class="md-header__source" style="margin-left: -5px !important;">
-        {% include "partials/source.html" %}
-      </div>
-    {% endif %}
-  </nav>
-
-  <!-- Navigation tabs (sticky) -->
-  {% if "navigation.tabs.sticky" in features %}
-    {% if "navigation.tabs" in features %}
-      {% include "partials/tabs.html" %}
-    {% endif %}
-  {% endif %}
-</header>

docs/src/ann_indexes.md

@@ -1,7 +1,7 @@
 # ANN (Approximate Nearest Neighbor) Indexes
 
 You can create an index over your vector data to make search faster.
-Vector indexes are faster but less accurate than exhaustive search (KNN or Flat Search).
+Vector indexes are faster but less accurate than exhaustive search.
 LanceDB provides many parameters to fine-tune the index's size, the speed of queries, and the accuracy of results.
 
 Currently, LanceDB does *not* automatically create the ANN index.
@@ -10,64 +10,36 @@ If you can live with <100ms latency, skipping index creation is a simpler workfl
 
 In the future we will look to automatically create and configure the ANN index.
 
-## Types of Index
+## Creating an ANN Index
 
-Lance can support multiple index types, the most widely used one is `IVF_PQ`.
+Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
 
-* `IVF_PQ`: use **Inverted File Index (IVF)** to first divide the dataset into `N` partitions,
-   and then use **Product Quantization** to compress vectors in each partition.
-* `DISKANN` (**Experimental**): organize the vector as a on-disk graph, where the vertices approximately
-   represent the nearest neighbors of each vector.
+```python
+import lancedb
+import numpy as np
+uri = "~/.lancedb"
+db = lancedb.connect(uri)
 
-## Creating an IVF_PQ Index
+# Create 10,000 sample vectors
+data = [{"vector": row, "item": f"item {i}"}
+   for i, row in enumerate(np.random.random((10_000, 768)).astype('float32'))]
 
-Lance supports `IVF_PQ` index type by default.
+# Add the vectors to a table
+tbl = db.create_table("my_vectors", data=data)
 
-=== "Python"
-     Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
+# Create and train the index - you need to have enough data in the table for an effective training step
+tbl.create_index(num_partitions=256, num_sub_vectors=96)
+```
 
-     ```python
-     import lancedb
-     import numpy as np
-     uri = "data/sample-lancedb"
-     db = lancedb.connect(uri)
-
-     # Create 10,000 sample vectors
-     data = [{"vector": row, "item": f"item {i}"}
-        for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]
-
-     # Add the vectors to a table
-     tbl = db.create_table("my_vectors", data=data)
-
-     # Create and train the index - you need to have enough data in the table for an effective training step
-     tbl.create_index(num_partitions=256, num_sub_vectors=96)
-     ```
-
-=== "Javascript"
-     ```javascript
-     const vectordb = require('vectordb')
-     const db = await vectordb.connect('data/sample-lancedb')
-
-     let data = []
-     for (let i = 0; i < 10_000; i++) {
-         data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
-     }
-     const table = await db.createTable('my_vectors', data)
-     await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 256, num_sub_vectors: 96 })
-     ```
-
-- **metric** (default: "L2"): The distance metric to use. By default it uses euclidean distance "`L2`".
-We also support "cosine" and "dot" distance as well.
-- **num_partitions** (default: 256): The number of partitions of the index.
-- **num_sub_vectors** (default: 96): The number of sub-vectors (M) that will be created during Product Quantization (PQ).
-For D dimensional vector, it will be divided into `M` of `D/M` sub-vectors, each of which is presented by
-a single PQ code.
-
-<figure markdown>
-  ![IVF PQ](./assets/ivf_pq.png)
-  <figcaption>IVF_PQ index with <code>num_partitions=2, num_sub_vectors=4</code></figcaption>
-</figure>
+Since `create_index` has a training step, it can take a few minutes to finish for large tables. You can control the index
+creation by providing the following parameters:
 
+- **metric** (default: "L2"): The distance metric to use. By default we use euclidean distance. We also support cosine distance.
+- **num_partitions** (default: 256): The number of partitions of the index. The number of partitions should be configured so each partition has 3-5K vectors. For example, a table
+with ~1M vectors should use 256 partitions. You can specify arbitrary number of partitions but powers of 2 is most conventional.
+A higher number leads to faster queries, but it makes index generation slower.
+- **num_sub_vectors** (default: 96): The number of subvectors (M) that will be created during Product Quantization (PQ). A larger number makes
+search more accurate, but also makes the index larger and slower to build.
 
 ## Querying an ANN Index
 
@@ -81,33 +53,22 @@ There are a couple of parameters that can be used to fine-tune the search:
   e.g., for 1M vectors divided up into 256 partitions, nprobes should be set to ~20-40.<br/>
   Note: nprobes is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
 - **refine_factor** (default: None): Refine the results by reading extra elements and re-ranking them in memory.<br/>
-  A higher number makes search more accurate but also slower. If you find the recall is less than ideal, try refine_factor=10 to start.<br/>
+  A higher number makes search more accurate but also slower. If you find the recall is less than idea, try refine_factor=10 to start.<br/>
   e.g., for 1M vectors divided into 256 partitions, if you're looking for top 20, then refine_factor=200 reranks the whole partition.<br/>
   Note: refine_factor is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
 
-=== "Python"
-     ```python
-     tbl.search(np.random.random((1536))) \
-         .limit(2) \
-         .nprobes(20) \
-         .refine_factor(10) \
-         .to_df()
-     ```
-     ```
-                                              vector       item       score
-     0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
-     1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
-     ```
 
-=== "Javascript"
-     ```javascript
-     const results_1 = await table
-         .search(Array(1536).fill(1.2))
-         .limit(2)
-         .nprobes(20)
-         .refineFactor(10)
-         .execute()
-     ```
+```python
+tbl.search(np.random.random((768))) \
+    .limit(2) \
+    .nprobes(20) \
+    .refine_factor(10) \
+    .to_df()
+
+                                              vector       item       score
+0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
+1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
+```
 
 The search will return the data requested in addition to the score of each item.
 
@@ -117,66 +78,18 @@ The search will return the data requested in addition to the score of each item.
 
 You can further filter the elements returned by a search using a where clause.
 
-=== "Python"
-     ```python
-     tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_df()
-     ```
-
-=== "Javascript"
-     ```javascript
-     const results_2 = await table
-         .search(Array(1536).fill(1.2))
-         .where("id != '1141'")
-         .execute()
-     ```
+```python
+tbl.search(np.random.random((768))).where("item != 'item 1141'").to_df()
+```
 
 ### Projections (select clause)
 
 You can select the columns returned by the query using a select clause.
 
-=== "Python"
-     ```python
-     tbl.search(np.random.random((1536))).select(["vector"]).to_df()
-     ```
-     ```
-     vector      score
-     0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
-     1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
-     ...
-     ```
-
-=== "Javascript"
-     ```javascript
-     const results_3 = await table
-         .search(Array(1536).fill(1.2))
-         .select(["id"])
-         .execute()
-     ```
-
-## FAQ
-
-### When is it necessary to create an ANN vector index.
-
-`LanceDB` has manually tuned SIMD code for computing vector distances.
-In our benchmarks, computing 100K pairs of 1K dimension vectors only take less than 20ms.
-For small dataset (<100K rows) or the applications which can accept 100ms latency, vector indices are usually not necessary.
-
-For large-scale or higher dimension vectors, it is beneficial to create vector index.
-
-### How big is my index, and how many memory will it take.
-
-In LanceDB, all vector indices are disk-based, meaning that when responding to a vector query, only the relevant pages from the index file are loaded from disk and cached in memory. Additionally, each sub-vector is usually encoded into 1 byte PQ code.
-
-For example, with a 1024-dimension dataset, if we choose `num_sub_vectors=64`, each sub-vector has `1024 / 64 = 16` float32 numbers.
-Product quantization can lead to approximately `16 * sizeof(float32) / 1 = 64` times of space reduction.
-
-### How to choose `num_partitions` and `num_sub_vectors` for `IVF_PQ` index.
-
-`num_partitions` is used to decide how many partitions the first level `IVF` index uses.
-Higher number of partitions could lead to more efficient I/O during queries and better accuracy, but it takes much more time to train.
-On `SIFT-1M` dataset, our benchmark shows that keeping each partition 1K-4K rows lead to a good latency / recall.
-
-`num_sub_vectors` decides how many Product Quantization code to generate on each vector. Because
-Product Quantization is a lossy compression of the original vector, the more `num_sub_vectors` usually results to
-less space distortion, and thus yield better accuracy. However, similarly, more `num_sub_vectors` causes heavier I/O and
-more PQ computation, thus, higher latency. `dimension / num_sub_vectors` should be aligned with 8 for better SIMD efficiency.
+```python
+tbl.search(np.random.random((768))).select(["vector"]).to_df()
+                                              vector      score
+0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
+1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
+...
+```

docs/src/basic.md

@@ -1,142 +1,74 @@
 # Basic LanceDB Functionality
 
-We'll cover the basics of using LanceDB on your local machine in this section.
-
-??? info "LanceDB runs embedded on your backend application, so there is no need to run a separate server."
-
-      <img src="../assets/lancedb_embedded_explanation.png" width="650px" />
-
-## Installation
-
-=== "Python"
-      ```shell
-      pip install lancedb
-      ```
-
-=== "Javascript"
-      ```shell
-      npm install vectordb
-      ```
-
 ## How to connect to a database
 
-=== "Python"
-      ```python
-      import lancedb
-      uri = "data/sample-lancedb"
-      db = lancedb.connect(uri)
-      ```
+In local mode, LanceDB stores data in a directory on your local machine. To connect to a local database, you can use the following code:
+```python
+import lancedb
+uri = "~/.lancedb"
+db = lancedb.connect(uri)
+```
 
-      LanceDB will create the directory if it doesn't exist (including parent directories).
+LanceDB will create the directory if it doesn't exist (including parent directories).
 
-      If you need a reminder of the uri, use the `db.uri` property.
-
-=== "Javascript"
-      ```javascript
-      const lancedb = require("vectordb");
-
-      const uri = "data/sample-lancedb";
-      const db = await lancedb.connect(uri);
-      ```
-      
-      LanceDB will create the directory if it doesn't exist (including parent directories).
-
-      If you need a reminder of the uri, you can call `db.uri()`.
+If you need a reminder of the uri, use the `db.uri` property.
 
 ## How to create a table
 
-=== "Python"
-      ```python
-      tbl = db.create_table("my_table",
-                        data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                              {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      ```
+To create a table, you can use the following code:
+```python
+tbl = db.create_table("my_table",
+                      data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                            {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+```
 
-      If the table already exists, LanceDB will raise an error by default.
-      If you want to overwrite the table, you can pass in `mode="overwrite"`
-      to the `create_table` method.
+Under the hood, LanceDB is converting the input data into an Apache Arrow table
+and persisting it to disk in [Lance format](github.com/eto-ai/lance).
 
-      You can also pass in a pandas DataFrame directly:
-      ```python
-      import pandas as pd
-      df = pd.DataFrame([{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      tbl = db.create_table("table_from_df", data=df)
-      ```
+If the table already exists, LanceDB will raise an error by default.
+If you want to overwrite the table, you can pass in `mode="overwrite"`
+to the `create_table` method.
 
-=== "Javascript"
-      ```javascript
-      const tb = await db.createTable("my_table",
-                        data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                              {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      ```
-      
-!!! warning
-
-      If the table already exists, LanceDB will raise an error by default.
-      If you want to overwrite the table, you can pass in `mode="overwrite"`
-      to the `createTable` function.
-
-??? info "Under the hood, LanceDB is converting the input data into an Apache Arrow table and persisting it to disk in [Lance format](https://www.github.com/lancedb/lance)."
+You can also pass in a pandas DataFrame directly:
+```python
+import pandas as pd
+df = pd.DataFrame([{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                   {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+tbl = db.create_table("table_from_df", data=df)
+```
 
 ## How to open an existing table
 
 Once created, you can open a table using the following code:
+```python
+tbl = db.open_table("my_table")
+```
 
-=== "Python"
-      ```python
-      tbl = db.open_table("my_table")
-      ```
+If you forget the name of your table, you can always get a listing of all table names:
 
-      If you forget the name of your table, you can always get a listing of all table names:
-
-      ```python
-      print(db.table_names())
-      ```
-
-=== "Javascript"
-      ```javascript
-      const tbl = await db.openTable("my_table");
-      ```
-
-      If you forget the name of your table, you can always get a listing of all table names:
-
-      ```javascript
-      console.log(await db.tableNames());
-      ```
+```python
+db.table_names()
+```
 
 ## How to add data to a table
 
 After a table has been created, you can always add more data to it using
 
-=== "Python"
-      ```python
-      df = pd.DataFrame([{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
-                        {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}])
-      tbl.add(df)
-      ```
-
-=== "Javascript"
-      ```javascript
-      await tbl.add([{vector: [1.3, 1.4], item: "fizz", price: 100.0},
-              {vector: [9.5, 56.2], item: "buzz", price: 200.0}])
-      ```
+```python
+df = pd.DataFrame([{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
+                   {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}])
+tbl.add(df)
+```
 
 ## How to search for (approximate) nearest neighbors
 
 Once you've embedded the query, you can find its nearest neighbors using the following code:
 
-=== "Python"
-      ```python
-      tbl.search([100, 100]).limit(2).to_df()
-      ```
+```python
+tbl.search([100, 100]).limit(2).to_df()
+```
 
-      This returns a pandas DataFrame with the results.
-
-=== "Javascript"
-      ```javascript
-      const query = await tbl.search([100, 100]).limit(2).execute();
-      ```
+This returns a pandas DataFrame with the results.
 
 ## What's next
 

docs/src/embedding.md

@@ -25,88 +25,55 @@ def embed_func(batch):
     return [model.encode(sentence) for sentence in batch]
 ```
 
-Please note that currently HuggingFace is only supported in the Python SDK.
-
 ### OpenAI example
 
 You can also use an external API like OpenAI to generate embeddings
 
-=== "Python"
-      ```python
-        import openai
-        import os
+```python
+import openai
+import os
 
-        # Configuring the environment variable OPENAI_API_KEY
-        if "OPENAI_API_KEY" not in os.environ:
-        # OR set the key here as a variable
-        openai.api_key = "sk-..."
+# Configuring the environment variable OPENAI_API_KEY
+if "OPENAI_API_KEY" not in os.environ:
+    # OR set the key here as a variable
+    openai.api_key = "sk-..."
 
-        # verify that the API key is working
-        assert len(openai.Model.list()["data"]) > 0
+# verify that the API key is working
+assert len(openai.Model.list()["data"]) > 0
 
-        def embed_func(c):
-            rs = openai.Embedding.create(input=c, engine="text-embedding-ada-002")
-            return [record["embedding"] for record in rs["data"]]
-      ```
-
-=== "Javascript"
-      ```javascript
-        const lancedb = require("vectordb");
-
-        // You need to provide an OpenAI API key
-        const apiKey = "sk-..."
-        // The embedding function will create embeddings for the 'text' column
-        const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
-      ```
+def embed_func(c):
+    rs = openai.Embedding.create(input=c, engine="text-embedding-ada-002")
+    return [record["embedding"] for record in rs["data"]]
+```
 
 ## Applying an embedding function
 
-=== "Python"
-    Using an embedding function, you can apply it to raw data
-    to generate embeddings for each row.
+Using an embedding function, you can apply it to raw data
+to generate embeddings for each row.
 
-    Say if you have a pandas DataFrame with a `text` column that you want to be embedded,
-    you can use the [with_embeddings](https://lancedb.github.io/lancedb/python/#lancedb.embeddings.with_embeddings)
-    function to generate embeddings and add create a combined pyarrow table:
+Say if you have a pandas DataFrame with a `text` column that you want to be embedded,
+you can use the [with_embeddings](https://lancedb.github.io/lancedb/python/#lancedb.embeddings.with_embeddings)
+function to generate embeddings and add create a combined pyarrow table:
 
+```python
+import pandas as pd
+from lancedb.embeddings import with_embeddings
 
-     ```python
-     import pandas as pd
-     from lancedb.embeddings import with_embeddings
+df = pd.DataFrame([{"text": "pepperoni"},
+                   {"text": "pineapple"}])
+data = with_embeddings(embed_func, df)
 
-     df = pd.DataFrame([{"text": "pepperoni"},
-                        {"text": "pineapple"}])
-     data = with_embeddings(embed_func, df)
+# The output is used to create / append to a table
+# db.create_table("my_table", data=data)
+```
 
-     # The output is used to create / append to a table
-     # db.create_table("my_table", data=data)
-     ```
+If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
 
-     If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
-
-     By default, LanceDB calls the function with batches of 1000 rows. This can be configured
-     using the `batch_size` parameter to `with_embeddings`.
-
-     LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
-     API call is reliable.
-
-=== "Javascript"
-     Using an embedding function, you can apply it to raw data
-     to generate embeddings for each row.
-
-     You can just pass the embedding function created previously and LanceDB will automatically generate
-     embededings for your data.
-
-      ```javascript
-      const db = await lancedb.connect("data/sample-lancedb");
-      const data = [
-        { text: 'pepperoni'  },
-        { text: 'pineapple' }
-      ]
-
-      const table = await db.createTable('vectors', data, embedding)
-      ```
+By default, LanceDB calls the function with batches of 1000 rows. This can be configured
+using the `batch_size` parameter to `with_embeddings`.
 
+LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
+API call is reliable.
 
 ## Searching with an embedding function
 
@@ -114,25 +81,13 @@ At inference time, you also need the same embedding function to embed your query
 It's important that you use the same model / function otherwise the embedding vectors don't
 belong in the same latent space and your results will be nonsensical.
 
-=== "Python"
-     ```python
-     query = "What's the best pizza topping?"
-     query_vector = embed_func([query])[0]
-     tbl.search(query_vector).limit(10).to_df()
-     ```
-
-     The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
-
-=== "Javascript"
-     ```javascript
-      const results = await table
-        .search("What's the best pizza topping?")
-        .limit(10)
-        .execute()
-     ```
-
-     The above snippet returns an array of records with the 10 closest vectors to the query.
+```python
+query = "What's the best pizza topping?"
+query_vector = embed_func([query])[0]
+tbl.search(query_vector).limit(10).to_df()
+```
 
+The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
 
 ## Roadmap
 

docs/src/examples/langchain.md

@@ -4,4 +4,4 @@
 
 <img id="splash" width="400" alt="langchain" src="https://user-images.githubusercontent.com/917119/236580868-61a246a9-e587-4c2b-8ae5-6fe5f7b7e81e.png">
 
-This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/code_qa_bot.ipynb)
+This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/notebooks/code_qa_bot.ipynb)

docs/src/examples/modal_langchain.py

@@ -1,117 +0,0 @@
-import pickle
-import re
-import sys
-import zipfile
-from pathlib import Path
-
-import requests
-from langchain.chains import RetrievalQA
-from langchain.document_loaders import UnstructuredHTMLLoader
-from langchain.embeddings import OpenAIEmbeddings
-from langchain.llms import OpenAI
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.vectorstores import LanceDB
-from modal import Image, Secret, Stub, web_endpoint
-
-import lancedb
-
-lancedb_image = Image.debian_slim().pip_install(
-    "lancedb", "langchain", "openai", "pandas", "tiktoken", "unstructured", "tabulate"
-)
-
-stub = Stub(
-    name="example-langchain-lancedb",
-    image=lancedb_image,
-    secrets=[Secret.from_name("my-openai-secret")],
-)
-
-docsearch = None
-docs_path = Path("docs.pkl")
-db_path = Path("lancedb")
-
-
-def get_document_title(document):
-    m = str(document.metadata["source"])
-    title = re.findall("pandas.documentation(.*).html", m)
-    if title[0] is not None:
-        return title[0]
-    return ""
-
-
-def download_docs():
-    pandas_docs = requests.get(
-        "https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip"
-    )
-    with open(Path("pandas.documentation.zip"), "wb") as f:
-        f.write(pandas_docs.content)
-
-    file = zipfile.ZipFile(Path("pandas.documentation.zip"))
-    file.extractall(path=Path("pandas_docs"))
-
-
-def store_docs():
-    docs = []
-
-    if not docs_path.exists():
-        for p in Path("pandas_docs/pandas.documentation").rglob("*.html"):
-            if p.is_dir():
-                continue
-            loader = UnstructuredHTMLLoader(p)
-            raw_document = loader.load()
-
-            m = {}
-            m["title"] = get_document_title(raw_document[0])
-            m["version"] = "2.0rc0"
-            raw_document[0].metadata = raw_document[0].metadata | m
-            raw_document[0].metadata["source"] = str(raw_document[0].metadata["source"])
-            docs = docs + raw_document
-
-        with docs_path.open("wb") as fh:
-            pickle.dump(docs, fh)
-    else:
-        with docs_path.open("rb") as fh:
-            docs = pickle.load(fh)
-
-    return docs
-
-
-def qanda_langchain(query):
-    download_docs()
-    docs = store_docs()
-
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200,)
-    documents = text_splitter.split_documents(docs)
-    embeddings = OpenAIEmbeddings()
-
-    db = lancedb.connect(db_path)
-    table = db.create_table(
-        "pandas_docs",
-        data=[
-            {
-                "vector": embeddings.embed_query("Hello World"),
-                "text": "Hello World",
-                "id": "1",
-            }
-        ],
-        mode="overwrite",
-    )
-    docsearch = LanceDB.from_documents(documents, embeddings, connection=table)
-    qa = RetrievalQA.from_chain_type(
-        llm=OpenAI(), chain_type="stuff", retriever=docsearch.as_retriever()
-    )
-    return qa.run(query)
-
-
-@stub.function()
-@web_endpoint(method="GET")
-def web(query: str):
-    answer = qanda_langchain(query)
-    return {
-        "answer": answer,
-    }
-
-
-@stub.function()
-def cli(query: str):
-    answer = qanda_langchain(query)
-    print(answer)

docs/src/examples/multimodal_search.md

@@ -1,7 +0,0 @@
-# Image multimodal search
-
-## Search through an image dataset using natural language, full text and SQL
-
-<img id="splash" width="400" alt="multimodal search" src="https://github.com/lancedb/lancedb/assets/917119/993a7c9f-be01-449d-942e-1ce1d4ed63af">
-
-This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/multimodal_search.ipynb)

docs/src/examples/nodejs.md

@@ -0,0 +1,99 @@
+# YouTube transcript QA bot with NodeJS
+
+## use LanceDB's Javascript API and OpenAI to build a QA bot for YouTube transcripts
+
+<img id="splash" width="400" alt="nodejs" src="https://github.com/lancedb/lancedb/assets/917119/3a140e75-bf8e-438a-a1e4-af14a72bcf98">
+
+This Q&A bot will allow you to search through youtube transcripts using natural language! We'll introduce how you can use LanceDB's Javascript API to store and manage your data easily.
+
+For this example we're using a HuggingFace dataset that contains YouTube transcriptions: `jamescalam/youtube-transcriptions`, to make it easier, we've converted it to a LanceDB `db` already, which you can download and put in a working directory:
+
+```wget -c https://eto-public.s3.us-west-2.amazonaws.com/lancedb_demo.tar.gz -O - | tar -xz -C .```
+
+Now, we'll create a simple app that can:
+1. Take a text based query and search for contexts in our corpus, using embeddings generated from the OpenAI Embedding API.
+2. Create a prompt with the contexts, and call the OpenAI Completion API to answer the text based query.
+
+Dependencies and setup of OpenAI API:
+
+```javascript
+const lancedb = require("vectordb");
+const { Configuration, OpenAIApi } = require("openai");
+
+const configuration = new Configuration({
+    apiKey: process.env.OPENAI_API_KEY,
+    });
+const openai = new OpenAIApi(configuration);
+```
+
+First, let's set our question and the context amount. The context amount will be used to query similar documents in our corpus.
+
+```javascript
+const QUESTION = "who was the 12th person on the moon and when did they land?";
+const CONTEXT_AMOUNT = 3;
+```
+
+Now, let's generate an embedding from this question:
+
+```javascript
+const embeddingResponse = await openai.createEmbedding({
+    model: "text-embedding-ada-002",
+    input: QUESTION,
+});
+
+const embedding = embeddingResponse.data["data"][0]["embedding"];
+```
+
+Once we have the embedding, we can connect to LanceDB (using the database we downloaded earlier), and search through the chatbot table.
+We'll extract 3 similar documents found.
+
+```javascript
+const db = await lancedb.connect('./lancedb');
+const tbl = await db.openTable('chatbot');
+const query = tbl.search(embedding);
+query.limit = CONTEXT_AMOUNT;
+const context = await query.execute();
+```
+
+Let's combine the context together so we can pass it into our prompt:
+
+```javascript
+for (let i = 1; i < context.length; i++) {
+    context[0]["text"] += " " + context[i]["text"];
+}
+```
+
+Lastly, let's construct the prompt. You could play around with this to create more accurate/better prompts to yield results.
+
+```javascript
+const prompt = "Answer the question based on the context below.\n\n" +
+    "Context:\n" +
+    `${context[0]["text"]}\n` +
+    `\n\nQuestion: ${QUESTION}\nAnswer:`;
+```
+
+We pass the prompt, along with the context, to the completion API.
+
+```javascript
+const completion = await openai.createCompletion({
+    model: "text-davinci-003",
+    prompt,
+    temperature: 0,
+    max_tokens: 400,
+    top_p: 1,
+    frequency_penalty: 0,
+    presence_penalty: 0,
+});
+```
+
+And that's it!
+
+```javascript
+console.log(completion.data.choices[0].text);
+```
+
+The response is (which is non deterministic):
+
+```
+The 12th person on the moon was Harrison Schmitt and he landed on December 11, 1972.
+```

docs/src/examples/serverless_qa_bot_with_modal_and_langchain.md

@@ -1,166 +0,0 @@
-# Serverless QA Bot with Modal and LangChain
-
-## use LanceDB's LangChain integration with Modal to run a serverless app
-
-<img id="splash" width="400" alt="modal" src="https://github.com/lancedb/lancedb/assets/917119/7d80a40f-60d7-48a6-972f-dab05000eccf">
-
-We're going to build a QA bot for your documentation using LanceDB's LangChain integration and use Modal for deployment.
-
-Modal is an end-to-end compute platform for model inference, batch jobs, task queues, web apps and more. It's a great way to deploy your LanceDB models and apps.
-
-To get started, ensure that you have created an account and logged into [Modal](https://modal.com/). To follow along, the full source code is available on Github [here](https://github.com/lancedb/lancedb/blob/main/docs/src/examples/modal_langchain.py).
-
-### Setting up Modal
-
-We'll start by specifying our dependencies and creating a new Modal `Stub`:
-
-```python
-lancedb_image = Image.debian_slim().pip_install(
-    "lancedb",
-    "langchain",
-    "openai",
-    "pandas",
-    "tiktoken",
-    "unstructured",
-    "tabulate"
-)
-
-stub = Stub(
-    name="example-langchain-lancedb",
-    image=lancedb_image,
-    secrets=[Secret.from_name("my-openai-secret")],
-)
-```
-
-We're using Modal's Secrets injection to secure our OpenAI key. To set your own, you can access the Modal UI and enter your key.
-
-### Setting up caches for LanceDB and LangChain
-
-Next, we can setup some globals to cache our LanceDB database, as well as our LangChain docsource:
-
-```python
-docsearch = None
-docs_path = Path("docs.pkl")
-db_path = Path("lancedb")
-```
-
-### Downloading our dataset
-
-We're going use a pregenerated dataset, which stores HTML files of the Pandas 2.0 documentation. 
-You could switch this out for your own dataset.
-
-```python
-def download_docs():
-    pandas_docs = requests.get("https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip")
-    with open(Path("pandas.documentation.zip"), "wb") as f:
-        f.write(pandas_docs.content)
-
-    file = zipfile.ZipFile(Path("pandas.documentation.zip"))
-    file.extractall(path=Path("pandas_docs"))
-```
-
-### Pre-processing the dataset and generating metadata
-
-Once we've downloaded it, we want to parse and pre-process them using LangChain, and then vectorize them and store it in LanceDB.
-Let's first create a function that uses LangChains `UnstructuredHTMLLoader` to parse them.
-We can then add our own metadata to it and store it alongside the data, we'll later be able to use this for filtering metadata.
-
-```python
-def store_docs():
-    docs = []
-
-    if not docs_path.exists():
-        for p in Path("pandas_docs/pandas.documentation").rglob("*.html"):
-            if p.is_dir():
-                continue
-            loader = UnstructuredHTMLLoader(p)
-            raw_document = loader.load()
-
-            m = {}
-            m["title"] = get_document_title(raw_document[0])
-            m["version"] = "2.0rc0"
-            raw_document[0].metadata = raw_document[0].metadata | m
-            raw_document[0].metadata["source"] = str(raw_document[0].metadata["source"])
-            docs = docs + raw_document
-
-        with docs_path.open("wb") as fh:
-            pickle.dump(docs, fh)
-    else:
-        with docs_path.open("rb") as fh:
-            docs = pickle.load(fh)
-
-    return docs
-```
-
-### Simple LangChain chain for a QA bot
-
-Now we can create a simple LangChain chain for our QA bot. We'll use the `RecursiveCharacterTextSplitter` to split our documents into chunks, and then use the `OpenAIEmbeddings` to vectorize them.
-
-Lastly, we'll create a LanceDB table and store the vectorized documents in it, then create a `RetrievalQA` model from the chain and return it.
-
-```python
-def qanda_langchain(query):
-    download_docs()
-    docs = store_docs()
-
-    text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=1000,
-        chunk_overlap=200,
-    )
-    documents = text_splitter.split_documents(docs)
-    embeddings = OpenAIEmbeddings()
-
-    db = lancedb.connect(db_path) 
-    table = db.create_table("pandas_docs", data=[
-        {"vector": embeddings.embed_query("Hello World"), "text": "Hello World", "id": "1"}
-    ], mode="overwrite")
-    docsearch = LanceDB.from_documents(documents, embeddings, connection=table)
-    qa = RetrievalQA.from_chain_type(llm=OpenAI(), chain_type="stuff", retriever=docsearch.as_retriever())
-    return qa.run(query)
-```
-
-### Creating our Modal entry points
-
-Now we can create our Modal entry points for our CLI and web endpoint:
-
-```python
-@stub.function()
-@web_endpoint(method="GET")
-def web(query: str):
-    answer = qanda_langchain(query)
-    return {
-        "answer": answer,
-    }
-    
-@stub.function()
-def cli(query: str):
-    answer = qanda_langchain(query)
-    print(answer)
-```
-
-# Testing it out!
-
-Testing the CLI:
-
-```bash
-modal run modal_langchain.py --query "What are the major differences in pandas 2.0?"
-```
-
-Testing the web endpoint:
-
-```bash
-modal serve modal_langchain.py
-```
-
-In the CLI, Modal will provide you a web endpoint. Copy this endpoint URI for the next step.
-Once this is served, then we can hit it with `curl`. 
-
-Note, the first time this runs, it will take a few minutes to download the dataset and vectorize it.
-An actual production example would pre-cache/load the dataset and vectorized documents prior
-
-```bash
-curl --get --data-urlencode "query=What are the major differences in pandas 2.0?" https://your-modal-endpoint-app.modal.run
-
-{"answer":" The major differences in pandas 2.0 include the ability to use any numpy numeric dtype in a Index, installing optional dependencies with pip extras, and enhancements, bug fixes, and performance improvements."}
-```
-

docs/src/examples/transformerjs_embedding_search_nodejs.md

@@ -1,121 +0,0 @@
-# Vector embedding search using TransformersJS
-
-## Embed and query data from LacneDB using TransformersJS
-
-<img id="splash" width="400" alt="transformersjs" src="https://github.com/lancedb/lancedb/assets/43097991/88a31e30-3d6f-4eef-9216-4b7c688f1b4f">
-
-This example shows how to use the [transformers.js](https://github.com/xenova/transformers.js) library to perform vector embedding search using LanceDB's Javascript API.
-
-
-### Setting up
-First, install the dependencies:
-```bash
-npm install vectordb
-npm i @xenova/transformers
-```
-
-We will also be using the [all-MiniLM-L6-v2](https://huggingface.co/Xenova/all-MiniLM-L6-v2) model to make it compatible with Transformers.js
-
-Within our `index.js` file we will import the necessary libraries and define our model and database:
-
-```javascript
-const lancedb = require('vectordb')
-const { pipeline } = await import('@xenova/transformers')
-const pipe = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
-```
-
-### Creating the embedding function
-
-Next, we will create a function that will take in a string and return the vector embedding of that string. We will use the `pipe` function we defined earlier to get the vector embedding of the string.
-
-```javascript
-// Define the function. `sourceColumn` is required for LanceDB to know
-// which column to use as input.
-const embed_fun = {}
-embed_fun.sourceColumn = 'text'
-embed_fun.embed = async function (batch) {
-    let result = []
-    // Given a batch of strings, we will use the `pipe` function to get
-    // the vector embedding of each string.
-    for (let text of batch) {
-        // 'mean' pooling and normalizing allows the embeddings to share the
-        // same length.
-        const res = await pipe(text, { pooling: 'mean', normalize: true })
-        result.push(Array.from(res['data']))
-    }
-    return (result)
-}
-```
-
-### Creating the database
-
-Now, we will create the LanceDB database and add the embedding function we defined earlier.
-
-```javascript
-// Link a folder and create a table with data
-const db = await lancedb.connect('data/sample-lancedb')
-
-// You can also import any other data, but make sure that you have a column
-// for the embedding function to use.
-const data = [
-    { id: 1, text: 'Cherry', type: 'fruit' },
-    { id: 2, text: 'Carrot', type: 'vegetable' },
-    { id: 3, text: 'Potato', type: 'vegetable' },
-    { id: 4, text: 'Apple', type: 'fruit' },
-    { id: 5, text: 'Banana', type: 'fruit' }
-]
-
-// Create the table with the embedding function
-const table = await db.createTable('food_table', data, "create", embed_fun)
-```
-
-### Performing the search
-
-Now, we can perform the search using the `search` function. LanceDB automatically uses the embedding function we defined earlier to get the vector embedding of the query string.
-
-```javascript
-// Query the table
-const results = await table
-    .search("a sweet fruit to eat")
-    .metricType("cosine")
-    .limit(2)
-    .execute()
-console.log(results.map(r => r.text))
-```
-```bash
-[ 'Banana', 'Cherry' ]
-```
-
-Output of `results`:
-```bash
-[
-  {
-    vector: Float32Array(384) [
-      -0.057455405592918396,
-      0.03617725893855095,
-      -0.0367760956287384,
-      ... 381 more items
-    ],
-    id: 5,
-    text: 'Banana',
-    type: 'fruit',
-    score: 0.4919965863227844
-  },
-  {
-    vector: Float32Array(384) [
-      0.0009714411571621895,
-      0.008223623037338257,
-      0.009571489877998829,
-      ... 381 more items
-    ],
-    id: 1,
-    text: 'Cherry',
-    type: 'fruit',
-    score: 0.5540297031402588
-  }
-]
-```
-
-### Wrapping it up
-
-In this example, we showed how to use the `transformers.js` library to perform vector embedding search using LanceDB's Javascript API. You can find the full code for this example on [Github](https://github.com/lancedb/lancedb/blob/main/node/examples/js-transformers/index.js)!

docs/src/examples/youtube_transcript_bot_with_nodejs.md

@@ -1,139 +0,0 @@
-# YouTube transcript QA bot with NodeJS
-
-## use LanceDB's Javascript API and OpenAI to build a QA bot for YouTube transcripts
-
-<img id="splash" width="400" alt="nodejs" src="https://github.com/lancedb/lancedb/assets/917119/3a140e75-bf8e-438a-a1e4-af14a72bcf98">
-
-This Q&A bot will allow you to search through youtube transcripts using natural language! We'll introduce how to use LanceDB's Javascript API to store and manage your data easily.
-
-```bash
-npm install vectordb
-```
-
-## Download the data
-
-For this example, we're using a sample of a HuggingFace dataset that contains YouTube transcriptions: `jamescalam/youtube-transcriptions`. Download and extract this file under the `data` folder:
-
-```bash
-wget -c https://eto-public.s3.us-west-2.amazonaws.com/datasets/youtube_transcript/youtube-transcriptions_sample.jsonl
-```
-
-## Prepare Context
-
-Each item in the dataset contains just a short chunk of text. We'll need to merge a bunch of these chunks together on a rolling basis. For this demo, we'll look back 20 records to create a more complete context for each sentence.
-
-First, we need to read and parse the input file.
-
-```javascript
-const lines = (await fs.readFile(INPUT_FILE_NAME, 'utf-8'))
-  .toString()
-  .split('\n')
-  .filter(line => line.length > 0)
-  .map(line => JSON.parse(line))
-
-const data = contextualize(lines, 20, 'video_id')
-```
-
-The contextualize function groups the transcripts by video_id and then creates the expanded context for each item.
-
-```javascript
-function contextualize (rows, contextSize, groupColumn) {
-  const grouped = []
-  rows.forEach(row => {
-    if (!grouped[row[groupColumn]]) {
-      grouped[row[groupColumn]] = []
-    }
-    grouped[row[groupColumn]].push(row)
-  })
-
-  const data = []
-  Object.keys(grouped).forEach(key => {
-    for (let i = 0; i < grouped[key].length; i++) {
-      const start = i - contextSize > 0 ? i - contextSize : 0
-      grouped[key][i].context = grouped[key].slice(start, i + 1).map(r => r.text).join(' ')
-    }
-    data.push(...grouped[key])
-  })
-  return data
-}
-```
-
-## Create the LanceDB Table
-
-To load our data into LanceDB, we need to create embedding (vectors) for each item. For this example, we will use the OpenAI embedding functions, which have a native integration with LanceDB.
-
-```javascript
-// You need to provide an OpenAI API key, here we read it from the OPENAI_API_KEY environment variable
-const apiKey = process.env.OPENAI_API_KEY
-// The embedding function will create embeddings for the 'context' column
-const embedFunction = new lancedb.OpenAIEmbeddingFunction('context', apiKey)
-// Connects to LanceDB
-const db = await lancedb.connect('data/youtube-lancedb')
-const tbl = await db.createTable('vectors', data, embedFunction)
-```
-
-## Create and answer the prompt
-
-We will accept questions in natural language and use our corpus stored in LanceDB to answer them. First, we need to set up the OpenAI client:
-
-```javascript
-const configuration = new Configuration({ apiKey })
-const openai = new OpenAIApi(configuration)
-```
-
-Then we can prompt questions and use LanceDB to retrieve the three most relevant transcripts for this prompt.
-
-```javascript
-const query = await rl.question('Prompt: ')
-const results = await tbl
-  .search(query)
-  .select(['title', 'text', 'context'])
-  .limit(3)
-  .execute()
-```
-
-The query and the transcripts' context are appended together in a single prompt:
-
-```javascript
-function createPrompt (query, context) {
-    let prompt =
-        'Answer the question based on the context below.\n\n' +
-        'Context:\n'
-
-    // need to make sure our prompt is not larger than max size
-    prompt = prompt + context.map(c => c.context).join('\n\n---\n\n').substring(0, 3750)
-    prompt = prompt + `\n\nQuestion: ${query}\nAnswer:`
-    return prompt
-}
-```
-
-We can now use the OpenAI Completion API to process our custom prompt and give us an answer.
-
-```javascript
-const response = await openai.createCompletion({
-  model: 'text-davinci-003',
-  prompt: createPrompt(query, results),
-  max_tokens: 400,
-  temperature: 0,
-  top_p: 1,
-  frequency_penalty: 0,
-  presence_penalty: 0
-})
-console.log(response.data.choices[0].text)
-```
-
-## Let's put it all together now
-
-Now we can provide queries and have them answered based on your local LanceDB data.
-
-```bash
-Prompt: who was the 12th person on the moon and when did they land?
- The 12th person on the moon was Harrison Schmitt and he landed on December 11, 1972.
-Prompt: Which training method should I use for sentence transformers when I only have pairs of related sentences?
- NLI with multiple negative ranking loss.
-```
-
-## That's a wrap
-
-In this example, you learned how to use LanceDB to store and query embedding representations of your local data. The complete example code is on [GitHub](https://github.com/lancedb/lancedb/tree/main/node/examples), and you can also download the LanceDB dataset using [this link](https://eto-public.s3.us-west-2.amazonaws.com/datasets/youtube_transcript/youtube-lancedb.zip).
-

docs/src/examples/youtube_transcript_search.md

@@ -4,4 +4,4 @@
 
 <img id="splash" width="400" alt="youtube transcript search" src="https://user-images.githubusercontent.com/917119/236965568-def7394d-171c-45f2-939d-8edfeaadd88c.png">
 
-This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb)
+This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/notebooks/youtube_transcript_search.ipynb)

docs/src/fts.md

@@ -6,10 +6,9 @@ to make this available for JS as well.
 
 ## Installation
 
-To use full text search, you must install optional dependency tantivy-py:
+To use full text search, you must install the fts optional dependencies:
 
-# tantivy 0.19.2
-pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+`pip install lancedb[fts]`
 
 
 ## Quickstart
@@ -18,20 +17,6 @@ Assume:
 1. `table` is a LanceDB Table
 2. `text` is the name of the Table column that we want to index
 
-For example,
-
-```python
-import lancedb
-
-uri = "data/sample-lancedb"
-db = lancedb.connect(uri)
-
-table = db.create_table("my_table",
-            data=[{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
-                  {"vector": [5.9, 26.5], "text": "There are several kittens playing"}])
-
-```
-
 To create the index:
 
 ```python

docs/src/index.md

@@ -1,6 +1,6 @@
 # Welcome to LanceDB's Documentation
 
-LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrevial, filtering and management of embeddings.
+LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrivial, filtering and management of embeddings.
 
 The key features of LanceDB include:
 
@@ -8,65 +8,43 @@ The key features of LanceDB include:
 
 * Store, query and filter vectors, metadata and multi-modal data (text, images, videos, point clouds, and more).
 
-* Support for vector similarity search, full-text search and SQL.
-
-* Native Python and Javascript/Typescript support.
+* Native Python and Javascript/Typescript support (coming soon).
 
 * Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure.
 
-* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
+* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lanecdb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
 
-LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
+LanceDB's core is written in Rust 🦀 and is built using Lance, an open-source columnar format designed for performant ML workloads.
 
-## Quick Start
 
-=== "Python"
-      ```shell
-      pip install lancedb
-      ```
+## Installation
 
-      ```python
-      import lancedb
+```shell
+pip install lancedb
+```
 
-      uri = "data/sample-lancedb"
-      db = lancedb.connect(uri)
-      table = db.create_table("my_table",
-                              data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                                    {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      result = table.search([100, 100]).limit(2).to_df()
-      ```
+## Quickstart
 
-=== "Javascript"
-      ```shell
-      npm install vectordb
-      ```
+```python
+import lancedb
 
-      ```javascript
-      const lancedb = require("vectordb");
+db = lancedb.connect(".")
+table = db.create_table("my_table",
+                         data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                               {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+result = table.search([100, 100]).limit(2).to_df()
+```
 
-      const uri = "data/sample-lancedb";
-      const db = await lancedb.connect(uri);
-      const table = await db.createTable("my_table",
-            [{ id: 1, vector: [3.1, 4.1], item: "foo", price: 10.0 },
-            { id: 2, vector: [5.9, 26.5], item: "bar", price: 20.0 }])
-      const results = await table.search([100, 100]).limit(2).execute();
-      ```
+## Complete Demos
 
-## Complete Demos (Python)
-- [YouTube Transcript Search](notebooks/youtube_transcript_search.ipynb)
-- [Documentation QA Bot using LangChain](notebooks/code_qa_bot.ipynb)
-- [Multimodal search using CLIP](notebooks/multimodal_search.ipynb)
-- [Serverless QA Bot with S3 and Lambda](examples/serverless_lancedb_with_s3_and_lambda.md)
-- [Serverless QA Bot with Modal](examples/serverless_qa_bot_with_modal_and_langchain.md)
+We will be adding completed demo apps built using LanceDB.
+- [YouTube Transcript Search](../notebooks/youtube_transcript_search.ipynb)
 
-## Complete Demos (JavaScript)
-- [YouTube Transcript Search](examples/youtube_transcript_bot_with_nodejs.md)
 
 ## Documentation Quick Links
 * [`Basic Operations`](basic.md) - basic functionality of LanceDB.
 * [`Embedding Functions`](embedding.md) - functions for working with embeddings.
 * [`Indexing`](ann_indexes.md) - create vector indexes to speed up queries.
 * [`Full text search`](fts.md) - [EXPERIMENTAL] full-text search API
-* [`Ecosystem Integrations`](python/integration.md) - integrating LanceDB with python data tooling ecosystem.
-* [`Python API Reference`](python/python.md) - detailed documentation for the LanceDB Python SDK.
-* [`Node API Reference`](javascript/modules.md) - detailed documentation for the LanceDB Python SDK.
+* [`Ecosystem Integrations`](integrations.md) - integrating LanceDB with python data tooling ecosystem.
+* [`API Reference`](python.md) - detailed documentation for the LanceDB Python SDK.

docs/src/integrations.md

@@ -0,0 +1,111 @@
+# Integrations
+
+Built on top of Apache Arrow, `LanceDB` is easy to integrate with the Python ecosystem, including Pandas, PyArrow and DuckDB.
+
+## Pandas and PyArrow
+
+First, we need to connect to a `LanceDB` database.
+
+``` py
+
+import lancedb
+
+db = lancedb.connect("/tmp/lancedb")
+```
+
+And write a `Pandas DataFrame` to LanceDB directly.
+
+```py
+import pandas as pd
+
+data = pd.DataFrame({
+    "vector": [[3.1, 4.1], [5.9, 26.5]],
+    "item": ["foo", "bar"],
+    "price": [10.0, 20.0]
+})
+table = db.create_table("pd_table", data=data)
+
+# Optionally, create a IVF_PQ index
+table.create_index(num_partitions=256, num_sub_vectors=96)
+```
+
+You will find detailed instructions of creating dataset and index in [Basic Operations](basic.md) and [Indexing](indexing.md)
+sections.
+
+
+We can now perform similarity searches via `LanceDB`.
+
+```py
+# Open the table previously created.
+table = db.open_table("pd_table")
+
+query_vector = [100, 100]
+# Pandas DataFrame
+df = table.search(query_vector).limit(1).to_df()
+print(df)
+```
+
+```
+    vector     item  price        score
+0  [5.9, 26.5]  bar   20.0  14257.05957
+```
+
+If you have a simple filter, it's faster to provide a where clause to `LanceDB`'s search query.
+If you have more complex criteria, you can always apply the filter to the resulting pandas `DataFrame` from the search query.
+
+```python
+
+# Apply the filter via LanceDB
+results = table.search([100, 100]).where("price < 15").to_df()
+assert len(results) == 1
+assert results["item"].iloc[0] == "foo"
+
+# Apply the filter via Pandas
+df = results = table.search([100, 100]).to_df()
+results = df[df.price < 15]
+assert len(results) == 1
+assert results["item"].iloc[0] == "foo"
+```
+
+## DuckDB
+
+`LanceDB` works with `DuckDB` via [PyArrow integration](https://duckdb.org/docs/guides/python/sql_on_arrow).
+
+Let us start with installing `duckdb` and `lancedb`.
+
+```shell
+pip install duckdb lancedb
+```
+
+We will re-use the dataset created previously
+
+```python
+import lancedb
+
+db = lancedb.connect("/tmp/lancedb")
+table = db.open_table("pd_table")
+arrow_table = table.to_arrow()
+```
+
+`DuckDB` can directly query the `arrow_table`:
+
+```python
+In [15]: duckdb.query("SELECT * FROM t")
+Out[15]:
+┌─────────────┬─────────┬────────┐
+│   vector    │  item   │ price  │
+│   float[]   │ varchar │ double │
+├─────────────┼─────────┼────────┤
+│ [3.1, 4.1]  │ foo     │   10.0 │
+│ [5.9, 26.5] │ bar     │   20.0 │
+└─────────────┴─────────┴────────┘
+
+In [16]: duckdb.query("SELECT mean(price) FROM t")
+Out[16]:
+┌─────────────┐
+│ mean(price) │
+│   double    │
+├─────────────┤
+│        15.0 │
+└─────────────┘
+```

docs/src/javascript/.nojekyll

@@ -1 +0,0 @@
-TypeDoc added this file to prevent GitHub Pages from using Jekyll. You can turn off this behavior by setting the `githubPages` option to false.

docs/src/javascript/README.md

@@ -1,47 +0,0 @@
-vectordb / [Exports](modules.md)
-
-# LanceDB
-
-A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb).
-
-## Installation
-
-```bash
-npm install vectordb
-```
-
-## Usage
-
-### Basic Example
-
-```javascript
-const lancedb = require('vectordb');
-const db = await lancedb.connect('data/sample-lancedb');
-const table = await db.createTable("my_table",
-      [{ id: 1, vector: [0.1, 1.0], item: "foo", price: 10.0 },
-      { id: 2, vector: [3.9, 0.5], item: "bar", price: 20.0 }])
-const results = await table.search([0.1, 0.3]).limit(20).execute();
-console.log(results);
-```
-
-The [examples](./examples) folder contains complete examples.
-
-## Development
-
-Run the tests with
-
-```bash
-npm test
-```
-
-To run the linter and have it automatically fix all errors
-
-```bash
-npm run lint -- --fix
-```
-
-To build documentation
-
-```bash
-npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
-```

docs/src/javascript/classes/LocalConnection.md

@@ -1,294 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / LocalConnection
-
-# Class: LocalConnection
-
-A connection to a LanceDB database.
-
-## Implements
-
-- [`Connection`](../interfaces/Connection.md)
-
-## Table of contents
-
-### Constructors
-
-- [constructor](LocalConnection.md#constructor)
-
-### Properties
-
-- [\_db](LocalConnection.md#_db)
-- [\_uri](LocalConnection.md#_uri)
-
-### Accessors
-
-- [uri](LocalConnection.md#uri)
-
-### Methods
-
-- [createTable](LocalConnection.md#createtable)
-- [createTableArrow](LocalConnection.md#createtablearrow)
-- [dropTable](LocalConnection.md#droptable)
-- [openTable](LocalConnection.md#opentable)
-- [tableNames](LocalConnection.md#tablenames)
-
-## Constructors
-
-### constructor
-
-• **new LocalConnection**(`db`, `uri`)
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `db` | `any` |
-| `uri` | `string` |
-
-#### Defined in
-
-[index.ts:132](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L132)
-
-## Properties
-
-### \_db
-
-• `Private` `Readonly` **\_db**: `any`
-
-#### Defined in
-
-[index.ts:130](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L130)
-
-___
-
-### \_uri
-
-• `Private` `Readonly` **\_uri**: `string`
-
-#### Defined in
-
-[index.ts:129](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L129)
-
-## Accessors
-
-### uri
-
-• `get` **uri**(): `string`
-
-#### Returns
-
-`string`
-
-#### Implementation of
-
-[Connection](../interfaces/Connection.md).[uri](../interfaces/Connection.md#uri)
-
-#### Defined in
-
-[index.ts:137](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L137)
-
-## Methods
-
-### createTable
-
-▸ **createTable**(`name`, `data`, `mode?`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-Creates a new Table and initialize it with new data.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
-| `mode?` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-#### Implementation of
-
-[Connection](../interfaces/Connection.md).[createTable](../interfaces/Connection.md#createtable)
-
-#### Defined in
-
-[index.ts:177](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L177)
-
-▸ **createTable**(`name`, `data`, `mode`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` |
-| `data` | `Record`<`string`, `unknown`\>[] |
-| `mode` | [`WriteMode`](../enums/WriteMode.md) |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-#### Implementation of
-
-Connection.createTable
-
-#### Defined in
-
-[index.ts:178](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L178)
-
-▸ **createTable**<`T`\>(`name`, `data`, `mode`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-Creates a new Table and initialize it with new data.
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
-| `mode` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-#### Implementation of
-
-Connection.createTable
-
-#### Defined in
-
-[index.ts:188](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L188)
-
-___
-
-### createTableArrow
-
-▸ **createTableArrow**(`name`, `table`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` |
-| `table` | `Table`<`any`\> |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-#### Implementation of
-
-[Connection](../interfaces/Connection.md).[createTableArrow](../interfaces/Connection.md#createtablearrow)
-
-#### Defined in
-
-[index.ts:201](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L201)
-
-___
-
-### dropTable
-
-▸ **dropTable**(`name`): `Promise`<`void`\>
-
-Drop an existing table.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table to drop. |
-
-#### Returns
-
-`Promise`<`void`\>
-
-#### Implementation of
-
-[Connection](../interfaces/Connection.md).[dropTable](../interfaces/Connection.md#droptable)
-
-#### Defined in
-
-[index.ts:211](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L211)
-
-___
-
-### openTable
-
-▸ **openTable**(`name`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-Open a table in the database.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-#### Implementation of
-
-[Connection](../interfaces/Connection.md).[openTable](../interfaces/Connection.md#opentable)
-
-#### Defined in
-
-[index.ts:153](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L153)
-
-▸ **openTable**<`T`\>(`name`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-Open a table in the database.
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-#### Implementation of
-
-Connection.openTable
-
-#### Defined in
-
-[index.ts:160](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L160)
-
-___
-
-### tableNames
-
-▸ **tableNames**(): `Promise`<`string`[]\>
-
-Get the names of all tables in the database.
-
-#### Returns
-
-`Promise`<`string`[]\>
-
-#### Implementation of
-
-[Connection](../interfaces/Connection.md).[tableNames](../interfaces/Connection.md#tablenames)
-
-#### Defined in
-
-[index.ts:144](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L144)

docs/src/javascript/classes/LocalTable.md

@@ -1,289 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / LocalTable
-
-# Class: LocalTable<T\>
-
-A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
-
-## Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-## Implements
-
-- [`Table`](../interfaces/Table.md)<`T`\>
-
-## Table of contents
-
-### Constructors
-
-- [constructor](LocalTable.md#constructor)
-
-### Properties
-
-- [\_embeddings](LocalTable.md#_embeddings)
-- [\_name](LocalTable.md#_name)
-- [\_tbl](LocalTable.md#_tbl)
-
-### Accessors
-
-- [name](LocalTable.md#name)
-
-### Methods
-
-- [add](LocalTable.md#add)
-- [countRows](LocalTable.md#countrows)
-- [createIndex](LocalTable.md#createindex)
-- [delete](LocalTable.md#delete)
-- [overwrite](LocalTable.md#overwrite)
-- [search](LocalTable.md#search)
-
-## Constructors
-
-### constructor
-
-• **new LocalTable**<`T`\>(`tbl`, `name`)
-
-#### Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `tbl` | `any` |
-| `name` | `string` |
-
-#### Defined in
-
-[index.ts:221](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L221)
-
-• **new LocalTable**<`T`\>(`tbl`, `name`, `embeddings`)
-
-#### Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `tbl` | `any` |  |
-| `name` | `string` |  |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use when interacting with this table |
-
-#### Defined in
-
-[index.ts:227](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L227)
-
-## Properties
-
-### \_embeddings
-
-• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
-
-#### Defined in
-
-[index.ts:219](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L219)
-
-___
-
-### \_name
-
-• `Private` `Readonly` **\_name**: `string`
-
-#### Defined in
-
-[index.ts:218](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L218)
-
-___
-
-### \_tbl
-
-• `Private` `Readonly` **\_tbl**: `any`
-
-#### Defined in
-
-[index.ts:217](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L217)
-
-## Accessors
-
-### name
-
-• `get` **name**(): `string`
-
-#### Returns
-
-`string`
-
-#### Implementation of
-
-[Table](../interfaces/Table.md).[name](../interfaces/Table.md#name)
-
-#### Defined in
-
-[index.ts:234](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L234)
-
-## Methods
-
-### add
-
-▸ **add**(`data`): `Promise`<`number`\>
-
-Insert records into this Table.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
-
-#### Returns
-
-`Promise`<`number`\>
-
-The number of rows added to the table
-
-#### Implementation of
-
-[Table](../interfaces/Table.md).[add](../interfaces/Table.md#add)
-
-#### Defined in
-
-[index.ts:252](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L252)
-
-___
-
-### countRows
-
-▸ **countRows**(): `Promise`<`number`\>
-
-Returns the number of rows in this table.
-
-#### Returns
-
-`Promise`<`number`\>
-
-#### Implementation of
-
-[Table](../interfaces/Table.md).[countRows](../interfaces/Table.md#countrows)
-
-#### Defined in
-
-[index.ts:278](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L278)
-
-___
-
-### createIndex
-
-▸ **createIndex**(`indexParams`): `Promise`<`any`\>
-
-Create an ANN index on this Table vector index.
-
-**`See`**
-
-VectorIndexParams.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `indexParams` | `IvfPQIndexConfig` | The parameters of this Index, |
-
-#### Returns
-
-`Promise`<`any`\>
-
-#### Implementation of
-
-[Table](../interfaces/Table.md).[createIndex](../interfaces/Table.md#createindex)
-
-#### Defined in
-
-[index.ts:271](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L271)
-
-___
-
-### delete
-
-▸ **delete**(`filter`): `Promise`<`void`\>
-
-Delete rows from this table.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `filter` | `string` | A filter in the same format used by a sql WHERE clause. |
-
-#### Returns
-
-`Promise`<`void`\>
-
-#### Implementation of
-
-[Table](../interfaces/Table.md).[delete](../interfaces/Table.md#delete)
-
-#### Defined in
-
-[index.ts:287](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L287)
-
-___
-
-### overwrite
-
-▸ **overwrite**(`data`): `Promise`<`number`\>
-
-Insert records into this Table, replacing its contents.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
-
-#### Returns
-
-`Promise`<`number`\>
-
-The number of rows added to the table
-
-#### Implementation of
-
-[Table](../interfaces/Table.md).[overwrite](../interfaces/Table.md#overwrite)
-
-#### Defined in
-
-[index.ts:262](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L262)
-
-___
-
-### search
-
-▸ **search**(`query`): [`Query`](Query.md)<`T`\>
-
-Creates a search query to find the nearest neighbors of the given search term
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `query` | `T` | The query search term |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Implementation of
-
-[Table](../interfaces/Table.md).[search](../interfaces/Table.md#search)
-
-#### Defined in
-
-[index.ts:242](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L242)

docs/src/javascript/classes/OpenAIEmbeddingFunction.md

@@ -1,105 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / OpenAIEmbeddingFunction
-
-# Class: OpenAIEmbeddingFunction
-
-An embedding function that automatically creates vector representation for a given column.
-
-## Implements
-
-- [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`string`\>
-
-## Table of contents
-
-### Constructors
-
-- [constructor](OpenAIEmbeddingFunction.md#constructor)
-
-### Properties
-
-- [\_modelName](OpenAIEmbeddingFunction.md#_modelname)
-- [\_openai](OpenAIEmbeddingFunction.md#_openai)
-- [sourceColumn](OpenAIEmbeddingFunction.md#sourcecolumn)
-
-### Methods
-
-- [embed](OpenAIEmbeddingFunction.md#embed)
-
-## Constructors
-
-### constructor
-
-• **new OpenAIEmbeddingFunction**(`sourceColumn`, `openAIKey`, `modelName?`)
-
-#### Parameters
-
-| Name | Type | Default value |
-| :------ | :------ | :------ |
-| `sourceColumn` | `string` | `undefined` |
-| `openAIKey` | `string` | `undefined` |
-| `modelName` | `string` | `'text-embedding-ada-002'` |
-
-#### Defined in
-
-[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L21)
-
-## Properties
-
-### \_modelName
-
-• `Private` `Readonly` **\_modelName**: `string`
-
-#### Defined in
-
-[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L19)
-
-___
-
-### \_openai
-
-• `Private` `Readonly` **\_openai**: `any`
-
-#### Defined in
-
-[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L18)
-
-___
-
-### sourceColumn
-
-• **sourceColumn**: `string`
-
-The name of the column that will be used as input for the Embedding Function.
-
-#### Implementation of
-
-[EmbeddingFunction](../interfaces/EmbeddingFunction.md).[sourceColumn](../interfaces/EmbeddingFunction.md#sourcecolumn)
-
-#### Defined in
-
-[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L50)
-
-## Methods
-
-### embed
-
-▸ **embed**(`data`): `Promise`<`number`[][]\>
-
-Creates a vector representation for the given values.
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `data` | `string`[] |
-
-#### Returns
-
-`Promise`<`number`[][]\>
-
-#### Implementation of
-
-[EmbeddingFunction](../interfaces/EmbeddingFunction.md).[embed](../interfaces/EmbeddingFunction.md#embed)
-
-#### Defined in
-
-[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L38)

docs/src/javascript/classes/Query.md

@@ -1,349 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / Query
-
-# Class: Query<T\>
-
-A builder for nearest neighbor queries for LanceDB.
-
-## Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-## Table of contents
-
-### Constructors
-
-- [constructor](Query.md#constructor)
-
-### Properties
-
-- [\_embeddings](Query.md#_embeddings)
-- [\_filter](Query.md#_filter)
-- [\_limit](Query.md#_limit)
-- [\_metricType](Query.md#_metrictype)
-- [\_nprobes](Query.md#_nprobes)
-- [\_query](Query.md#_query)
-- [\_queryVector](Query.md#_queryvector)
-- [\_refineFactor](Query.md#_refinefactor)
-- [\_select](Query.md#_select)
-- [\_tbl](Query.md#_tbl)
-- [where](Query.md#where)
-
-### Methods
-
-- [execute](Query.md#execute)
-- [filter](Query.md#filter)
-- [limit](Query.md#limit)
-- [metricType](Query.md#metrictype)
-- [nprobes](Query.md#nprobes)
-- [refineFactor](Query.md#refinefactor)
-- [select](Query.md#select)
-
-## Constructors
-
-### constructor
-
-• **new Query**<`T`\>(`tbl`, `query`, `embeddings?`)
-
-#### Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `tbl` | `any` |
-| `query` | `T` |
-| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
-
-#### Defined in
-
-[index.ts:362](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L362)
-
-## Properties
-
-### \_embeddings
-
-• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
-
-#### Defined in
-
-[index.ts:360](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L360)
-
-___
-
-### \_filter
-
-• `Private` `Optional` **\_filter**: `string`
-
-#### Defined in
-
-[index.ts:358](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L358)
-
-___
-
-### \_limit
-
-• `Private` **\_limit**: `number`
-
-#### Defined in
-
-[index.ts:354](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L354)
-
-___
-
-### \_metricType
-
-• `Private` `Optional` **\_metricType**: [`MetricType`](../enums/MetricType.md)
-
-#### Defined in
-
-[index.ts:359](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L359)
-
-___
-
-### \_nprobes
-
-• `Private` **\_nprobes**: `number`
-
-#### Defined in
-
-[index.ts:356](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L356)
-
-___
-
-### \_query
-
-• `Private` `Readonly` **\_query**: `T`
-
-#### Defined in
-
-[index.ts:352](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L352)
-
-___
-
-### \_queryVector
-
-• `Private` `Optional` **\_queryVector**: `number`[]
-
-#### Defined in
-
-[index.ts:353](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L353)
-
-___
-
-### \_refineFactor
-
-• `Private` `Optional` **\_refineFactor**: `number`
-
-#### Defined in
-
-[index.ts:355](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L355)
-
-___
-
-### \_select
-
-• `Private` `Optional` **\_select**: `string`[]
-
-#### Defined in
-
-[index.ts:357](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L357)
-
-___
-
-### \_tbl
-
-• `Private` `Readonly` **\_tbl**: `any`
-
-#### Defined in
-
-[index.ts:351](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L351)
-
-___
-
-### where
-
-• **where**: (`value`: `string`) => [`Query`](Query.md)<`T`\>
-
-#### Type declaration
-
-▸ (`value`): [`Query`](Query.md)<`T`\>
-
-A filter statement to be applied to this query.
-
-##### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | `string` | A filter in the same format used by a sql WHERE clause. |
-
-##### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:410](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L410)
-
-## Methods
-
-### execute
-
-▸ **execute**<`T`\>(): `Promise`<`T`[]\>
-
-Execute the query and return the results as an Array of Objects
-
-#### Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `Record`<`string`, `unknown`\> |
-
-#### Returns
-
-`Promise`<`T`[]\>
-
-#### Defined in
-
-[index.ts:433](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L433)
-
-___
-
-### filter
-
-▸ **filter**(`value`): [`Query`](Query.md)<`T`\>
-
-A filter statement to be applied to this query.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | `string` | A filter in the same format used by a sql WHERE clause. |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:405](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L405)
-
-___
-
-### limit
-
-▸ **limit**(`value`): [`Query`](Query.md)<`T`\>
-
-Sets the number of results that will be returned
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | `number` | number of results |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:378](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L378)
-
-___
-
-### metricType
-
-▸ **metricType**(`value`): [`Query`](Query.md)<`T`\>
-
-The MetricType used for this Query.
-
-**`See`**
-
-MetricType for the different options
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | [`MetricType`](../enums/MetricType.md) | The metric to the. |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:425](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L425)
-
-___
-
-### nprobes
-
-▸ **nprobes**(`value`): [`Query`](Query.md)<`T`\>
-
-The number of probes used. A higher number makes search more accurate but also slower.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | `number` | The number of probes used. |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:396](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L396)
-
-___
-
-### refineFactor
-
-▸ **refineFactor**(`value`): [`Query`](Query.md)<`T`\>
-
-Refine the results by reading extra elements and re-ranking them in memory.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | `number` | refine factor to use in this query. |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:387](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L387)
-
-___
-
-### select
-
-▸ **select**(`value`): [`Query`](Query.md)<`T`\>
-
-Return only the specified columns.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | `string`[] | Only select the specified columns. If not specified, all columns will be returned. |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:416](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L416)

docs/src/javascript/enums/MetricType.md

@@ -1,49 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / MetricType
-
-# Enumeration: MetricType
-
-Distance metrics type.
-
-## Table of contents
-
-### Enumeration Members
-
-- [Cosine](MetricType.md#cosine)
-- [Dot](MetricType.md#dot)
-- [L2](MetricType.md#l2)
-
-## Enumeration Members
-
-### Cosine
-
-• **Cosine** = ``"cosine"``
-
-Cosine distance
-
-#### Defined in
-
-[index.ts:481](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L481)
-
-___
-
-### Dot
-
-• **Dot** = ``"dot"``
-
-Dot product
-
-#### Defined in
-
-[index.ts:486](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L486)
-
-___
-
-### L2
-
-• **L2** = ``"l2"``
-
-Euclidean distance
-
-#### Defined in
-
-[index.ts:476](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L476)

docs/src/javascript/enums/WriteMode.md

@@ -1,49 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / WriteMode
-
-# Enumeration: WriteMode
-
-Write mode for writing a table.
-
-## Table of contents
-
-### Enumeration Members
-
-- [Append](WriteMode.md#append)
-- [Create](WriteMode.md#create)
-- [Overwrite](WriteMode.md#overwrite)
-
-## Enumeration Members
-
-### Append
-
-• **Append** = ``"append"``
-
-Append new data to the table.
-
-#### Defined in
-
-[index.ts:466](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L466)
-
-___
-
-### Create
-
-• **Create** = ``"create"``
-
-Create a new [Table](../interfaces/Table.md).
-
-#### Defined in
-
-[index.ts:462](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L462)
-
-___
-
-### Overwrite
-
-• **Overwrite** = ``"overwrite"``
-
-Overwrite the existing [Table](../interfaces/Table.md) if presented.
-
-#### Defined in
-
-[index.ts:464](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L464)

docs/src/javascript/interfaces/Connection.md

@@ -1,152 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / Connection
-
-# Interface: Connection
-
-A LanceDB Connection that allows you to open tables and create new ones.
-
-Connection could be local against filesystem or remote against a server.
-
-## Implemented by
-
-- [`LocalConnection`](../classes/LocalConnection.md)
-
-## Table of contents
-
-### Properties
-
-- [uri](Connection.md#uri)
-
-### Methods
-
-- [createTable](Connection.md#createtable)
-- [createTableArrow](Connection.md#createtablearrow)
-- [dropTable](Connection.md#droptable)
-- [openTable](Connection.md#opentable)
-- [tableNames](Connection.md#tablenames)
-
-## Properties
-
-### uri
-
-• **uri**: `string`
-
-#### Defined in
-
-[index.ts:45](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L45)
-
-## Methods
-
-### createTable
-
-▸ **createTable**<`T`\>(`name`, `data`, `mode?`, `embeddings?`): `Promise`<[`Table`](Table.md)<`T`\>\>
-
-Creates a new Table and initialize it with new data.
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
-| `mode?` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
-| `embeddings?` | [`EmbeddingFunction`](EmbeddingFunction.md)<`T`\> | An embedding function to use on this table |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`T`\>\>
-
-#### Defined in
-
-[index.ts:65](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L65)
-
-___
-
-### createTableArrow
-
-▸ **createTableArrow**(`name`, `table`): `Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` |
-| `table` | `Table`<`any`\> |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-#### Defined in
-
-[index.ts:67](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L67)
-
-___
-
-### dropTable
-
-▸ **dropTable**(`name`): `Promise`<`void`\>
-
-Drop an existing table.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table to drop. |
-
-#### Returns
-
-`Promise`<`void`\>
-
-#### Defined in
-
-[index.ts:73](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L73)
-
-___
-
-### openTable
-
-▸ **openTable**<`T`\>(`name`, `embeddings?`): `Promise`<[`Table`](Table.md)<`T`\>\>
-
-Open a table in the database.
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `embeddings?` | [`EmbeddingFunction`](EmbeddingFunction.md)<`T`\> | An embedding function to use on this table |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`T`\>\>
-
-#### Defined in
-
-[index.ts:55](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L55)
-
-___
-
-### tableNames
-
-▸ **tableNames**(): `Promise`<`string`[]\>
-
-#### Returns
-
-`Promise`<`string`[]\>
-
-#### Defined in
-
-[index.ts:47](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L47)

docs/src/javascript/interfaces/EmbeddingFunction.md

@@ -1,60 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / EmbeddingFunction
-
-# Interface: EmbeddingFunction<T\>
-
-An embedding function that automatically creates vector representation for a given column.
-
-## Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-## Implemented by
-
-- [`OpenAIEmbeddingFunction`](../classes/OpenAIEmbeddingFunction.md)
-
-## Table of contents
-
-### Properties
-
-- [embed](EmbeddingFunction.md#embed)
-- [sourceColumn](EmbeddingFunction.md#sourcecolumn)
-
-## Properties
-
-### embed
-
-• **embed**: (`data`: `T`[]) => `Promise`<`number`[][]\>
-
-#### Type declaration
-
-▸ (`data`): `Promise`<`number`[][]\>
-
-Creates a vector representation for the given values.
-
-##### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `data` | `T`[] |
-
-##### Returns
-
-`Promise`<`number`[][]\>
-
-#### Defined in
-
-[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/embedding_function.ts#L27)
-
-___
-
-### sourceColumn
-
-• **sourceColumn**: `string`
-
-The name of the column that will be used as input for the Embedding Function.
-
-#### Defined in
-
-[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/embedding_function.ts#L22)

docs/src/javascript/interfaces/Table.md

@@ -1,195 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / Table
-
-# Interface: Table<T\>
-
-A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
-
-## Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-## Implemented by
-
-- [`LocalTable`](../classes/LocalTable.md)
-
-## Table of contents
-
-### Properties
-
-- [add](Table.md#add)
-- [countRows](Table.md#countrows)
-- [createIndex](Table.md#createindex)
-- [delete](Table.md#delete)
-- [name](Table.md#name)
-- [overwrite](Table.md#overwrite)
-- [search](Table.md#search)
-
-## Properties
-
-### add
-
-• **add**: (`data`: `Record`<`string`, `unknown`\>[]) => `Promise`<`number`\>
-
-#### Type declaration
-
-▸ (`data`): `Promise`<`number`\>
-
-Insert records into this Table.
-
-##### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
-
-##### Returns
-
-`Promise`<`number`\>
-
-The number of rows added to the table
-
-#### Defined in
-
-[index.ts:95](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L95)
-
-___
-
-### countRows
-
-• **countRows**: () => `Promise`<`number`\>
-
-#### Type declaration
-
-▸ (): `Promise`<`number`\>
-
-Returns the number of rows in this table.
-
-##### Returns
-
-`Promise`<`number`\>
-
-#### Defined in
-
-[index.ts:115](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L115)
-
-___
-
-### createIndex
-
-• **createIndex**: (`indexParams`: `IvfPQIndexConfig`) => `Promise`<`any`\>
-
-#### Type declaration
-
-▸ (`indexParams`): `Promise`<`any`\>
-
-Create an ANN index on this Table vector index.
-
-**`See`**
-
-VectorIndexParams.
-
-##### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `indexParams` | `IvfPQIndexConfig` | The parameters of this Index, |
-
-##### Returns
-
-`Promise`<`any`\>
-
-#### Defined in
-
-[index.ts:110](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L110)
-
-___
-
-### delete
-
-• **delete**: (`filter`: `string`) => `Promise`<`void`\>
-
-#### Type declaration
-
-▸ (`filter`): `Promise`<`void`\>
-
-Delete rows from this table.
-
-##### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `filter` | `string` | A filter in the same format used by a sql WHERE clause. |
-
-##### Returns
-
-`Promise`<`void`\>
-
-#### Defined in
-
-[index.ts:122](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L122)
-
-___
-
-### name
-
-• **name**: `string`
-
-#### Defined in
-
-[index.ts:81](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L81)
-
-___
-
-### overwrite
-
-• **overwrite**: (`data`: `Record`<`string`, `unknown`\>[]) => `Promise`<`number`\>
-
-#### Type declaration
-
-▸ (`data`): `Promise`<`number`\>
-
-Insert records into this Table, replacing its contents.
-
-##### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
-
-##### Returns
-
-`Promise`<`number`\>
-
-The number of rows added to the table
-
-#### Defined in
-
-[index.ts:103](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L103)
-
-___
-
-### search
-
-• **search**: (`query`: `T`) => [`Query`](../classes/Query.md)<`T`\>
-
-#### Type declaration
-
-▸ (`query`): [`Query`](../classes/Query.md)<`T`\>
-
-Creates a search query to find the nearest neighbors of the given search term
-
-##### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `query` | `T` | The query search term |
-
-##### Returns
-
-[`Query`](../classes/Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:87](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L87)

docs/src/javascript/modules.md

@@ -1,63 +0,0 @@
-[vectordb](README.md) / Exports
-
-# vectordb
-
-## Table of contents
-
-### Enumerations
-
-- [MetricType](enums/MetricType.md)
-- [WriteMode](enums/WriteMode.md)
-
-### Classes
-
-- [LocalConnection](classes/LocalConnection.md)
-- [LocalTable](classes/LocalTable.md)
-- [OpenAIEmbeddingFunction](classes/OpenAIEmbeddingFunction.md)
-- [Query](classes/Query.md)
-
-### Interfaces
-
-- [Connection](interfaces/Connection.md)
-- [EmbeddingFunction](interfaces/EmbeddingFunction.md)
-- [Table](interfaces/Table.md)
-
-### Type Aliases
-
-- [VectorIndexParams](modules.md#vectorindexparams)
-
-### Functions
-
-- [connect](modules.md#connect)
-
-## Type Aliases
-
-### VectorIndexParams
-
-Ƭ **VectorIndexParams**: `IvfPQIndexConfig`
-
-#### Defined in
-
-[index.ts:345](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L345)
-
-## Functions
-
-### connect
-
-▸ **connect**(`uri`): `Promise`<[`Connection`](interfaces/Connection.md)\>
-
-Connect to a LanceDB instance at the given URI
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `uri` | `string` | The uri of the database. |
-
-#### Returns
-
-`Promise`<[`Connection`](interfaces/Connection.md)\>
-
-#### Defined in
-
-[index.ts:34](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L34)

docs/src/notebooks/diffusiondb/datagen.py

@@ -1,109 +0,0 @@
-#!/usr/bin/env python
-#
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-"""Dataset hf://poloclub/diffusiondb
-"""
-
-import io
-from argparse import ArgumentParser
-from multiprocessing import Pool
-
-import lance
-import pyarrow as pa
-from datasets import load_dataset
-from PIL import Image
-from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast
-
-import lancedb
-
-MODEL_ID = "openai/clip-vit-base-patch32"
-
-device = "cuda"
-
-tokenizer = CLIPTokenizerFast.from_pretrained(MODEL_ID)
-model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32").to(device)
-processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
-
-schema = pa.schema(
-    [
-        pa.field("prompt", pa.string()),
-        pa.field("seed", pa.uint32()),
-        pa.field("step", pa.uint16()),
-        pa.field("cfg", pa.float32()),
-        pa.field("sampler", pa.string()),
-        pa.field("width", pa.uint16()),
-        pa.field("height", pa.uint16()),
-        pa.field("timestamp", pa.timestamp("s")),
-        pa.field("image_nsfw", pa.float32()),
-        pa.field("prompt_nsfw", pa.float32()),
-        pa.field("vector", pa.list_(pa.float32(), 512)),
-        pa.field("image", pa.binary()),
-    ]
-)
-
-
-def pil_to_bytes(img) -> list[bytes]:
-    buf = io.BytesIO()
-    img.save(buf, format="PNG")
-    return buf.getvalue()
-
-
-def generate_clip_embeddings(batch) -> pa.RecordBatch:
-    image = processor(text=None, images=batch["image"], return_tensors="pt")[
-        "pixel_values"
-    ].to(device)
-    img_emb = model.get_image_features(image)
-    batch["vector"] = img_emb.cpu().tolist()
-
-    with Pool() as p:
-        batch["image_bytes"] = p.map(pil_to_bytes, batch["image"])
-    return batch
-
-
-def datagen(args):
-    """Generate DiffusionDB dataset, and use CLIP model to generate image embeddings."""
-    dataset = load_dataset("poloclub/diffusiondb", args.subset)
-    data = []
-    for b in dataset.map(
-        generate_clip_embeddings, batched=True, batch_size=256, remove_columns=["image"]
-    )["train"]:
-        b["image"] = b["image_bytes"]
-        del b["image_bytes"]
-        data.append(b)
-    tbl = pa.Table.from_pylist(data, schema=schema)
-    return tbl
-
-
-def main():
-    parser = ArgumentParser()
-    parser.add_argument(
-        "-o", "--output", metavar="DIR", help="Output lance directory", required=True
-    )
-    parser.add_argument(
-        "-s",
-        "--subset",
-        choices=["2m_all", "2m_first_10k", "2m_first_100k"],
-        default="2m_first_10k",
-        help="subset of the hg dataset",
-    )
-
-    args = parser.parse_args()
-
-    batches = datagen(args)
-    lance.write_dataset(batches, args.output)
-
-
-if __name__ == "__main__":
-    main()

docs/src/notebooks/diffusiondb/requirements.txt

@@ -1,9 +0,0 @@
-datasets
-Pillow
-lancedb
-isort
-black
-transformers
---index-url https://download.pytorch.org/whl/cu118
-torch
-torchvision

docs/src/notebooks/multimodal_search.ipynb

@@ -1,269 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
-      "\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
-     ]
-    }
-   ],
-   "source": [
-    "!pip install --quiet -U lancedb\n",
-    "!pip install --quiet gradio transformers torch torchvision"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import io\n",
-    "import PIL\n",
-    "import duckdb\n",
-    "import lancedb"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## First run setup: Download data and pre-process"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 30,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "<lance.dataset.LanceDataset at 0x3045db590>"
-      ]
-     },
-     "execution_count": 30,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "# remove null prompts\n",
-    "import lance\n",
-    "import pyarrow.compute as pc\n",
-    "\n",
-    "# download s3://eto-public/datasets/diffusiondb/small_10k.lance to this uri\n",
-    "data = lance.dataset(\"~/datasets/rawdata.lance\").to_table()\n",
-    "\n",
-    "# First data processing and full-text-search index\n",
-    "db = lancedb.connect(\"~/datasets/demo\")\n",
-    "tbl = db.create_table(\"diffusiondb\", data.filter(~pc.field(\"prompt\").is_null()))\n",
-    "tbl = tbl.create_fts_index([\"prompt\"])"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Create / Open LanceDB Table"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "db = lancedb.connect(\"~/datasets/demo\")\n",
-    "tbl = db.open_table(\"diffusiondb\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Create CLIP embedding function for the text"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast\n",
-    "\n",
-    "MODEL_ID = \"openai/clip-vit-base-patch32\"\n",
-    "\n",
-    "tokenizer = CLIPTokenizerFast.from_pretrained(MODEL_ID)\n",
-    "model = CLIPModel.from_pretrained(MODEL_ID)\n",
-    "processor = CLIPProcessor.from_pretrained(MODEL_ID)\n",
-    "\n",
-    "def embed_func(query):\n",
-    "    inputs = tokenizer([query], padding=True, return_tensors=\"pt\")\n",
-    "    text_features = model.get_text_features(**inputs)\n",
-    "    return text_features.detach().numpy()[0]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Search functions for Gradio"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def find_image_vectors(query):\n",
-    "    emb = embed_func(query)\n",
-    "    code = (\n",
-    "        \"import lancedb\\n\"\n",
-    "        \"db = lancedb.connect('~/datasets/demo')\\n\"\n",
-    "        \"tbl = db.open_table('diffusiondb')\\n\\n\"\n",
-    "        f\"embedding = embed_func('{query}')\\n\"\n",
-    "        \"tbl.search(embedding).limit(9).to_df()\"\n",
-    "    )\n",
-    "    return (_extract(tbl.search(emb).limit(9).to_df()), code)\n",
-    "\n",
-    "def find_image_keywords(query):\n",
-    "    code = (\n",
-    "        \"import lancedb\\n\"\n",
-    "        \"db = lancedb.connect('~/datasets/demo')\\n\"\n",
-    "        \"tbl = db.open_table('diffusiondb')\\n\\n\"\n",
-    "        f\"tbl.search('{query}').limit(9).to_df()\"\n",
-    "    )\n",
-    "    return (_extract(tbl.search(query).limit(9).to_df()), code)\n",
-    "\n",
-    "def find_image_sql(query):\n",
-    "    code = (\n",
-    "        \"import lancedb\\n\"\n",
-    "        \"import duckdb\\n\"\n",
-    "        \"db = lancedb.connect('~/datasets/demo')\\n\"\n",
-    "        \"tbl = db.open_table('diffusiondb')\\n\\n\"\n",
-    "        \"diffusiondb = tbl.to_lance()\\n\"\n",
-    "        f\"duckdb.sql('{query}').to_df()\"\n",
-    "    )    \n",
-    "    diffusiondb = tbl.to_lance()\n",
-    "    return (_extract(duckdb.sql(query).to_df()), code)\n",
-    "\n",
-    "def _extract(df):\n",
-    "    image_col = \"image\"\n",
-    "    return [(PIL.Image.open(io.BytesIO(row[image_col])), row[\"prompt\"]) for _, row in df.iterrows()]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Setup Gradio interface"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 28,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Running on local URL:  http://127.0.0.1:7881\n",
-      "\n",
-      "To create a public link, set `share=True` in `launch()`.\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<div><iframe src=\"http://127.0.0.1:7881/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
-      ],
-      "text/plain": [
-       "<IPython.core.display.HTML object>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/plain": []
-     },
-     "execution_count": 28,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import gradio as gr\n",
-    "\n",
-    "\n",
-    "with gr.Blocks() as demo:\n",
-    "    with gr.Row():\n",
-    "        with gr.Tab(\"Embeddings\"):\n",
-    "            vector_query = gr.Textbox(value=\"portraits of a person\", show_label=False)\n",
-    "            b1 = gr.Button(\"Submit\")\n",
-    "        with gr.Tab(\"Keywords\"):\n",
-    "            keyword_query = gr.Textbox(value=\"ninja turtle\", show_label=False)\n",
-    "            b2 = gr.Button(\"Submit\")\n",
-    "        with gr.Tab(\"SQL\"):\n",
-    "            sql_query = gr.Textbox(value=\"SELECT * from diffusiondb WHERE image_nsfw >= 2 LIMIT 9\", show_label=False)\n",
-    "            b3 = gr.Button(\"Submit\")\n",
-    "    with gr.Row():\n",
-    "        code = gr.Code(label=\"Code\", language=\"python\")\n",
-    "    with gr.Row():\n",
-    "        gallery = gr.Gallery(\n",
-    "                label=\"Found images\", show_label=False, elem_id=\"gallery\"\n",
-    "            ).style(columns=[3], rows=[3], object_fit=\"contain\", height=\"auto\")   \n",
-    "        \n",
-    "    b1.click(find_image_vectors, inputs=vector_query, outputs=[gallery, code])\n",
-    "    b2.click(find_image_keywords, inputs=keyword_query, outputs=[gallery, code])\n",
-    "    b3.click(find_image_sql, inputs=sql_query, outputs=[gallery, code])\n",
-    "    \n",
-    "demo.launch()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.3"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 1
-}

docs/src/python.md

@@ -0,0 +1,14 @@
+# LanceDB Python API Reference
+
+## Installation
+
+```shell
+pip install lancedb
+```
+
+## ::: lancedb
+## ::: lancedb.db
+## ::: lancedb.table
+## ::: lancedb.query
+## ::: lancedb.embeddings
+## ::: lancedb.context

docs/src/python/arrow.md

@@ -1,101 +0,0 @@
-# Pandas and PyArrow
-
-
-Built on top of [Apache Arrow](https://arrow.apache.org/),
-`LanceDB` is easy to integrate with the Python ecosystem, including [Pandas](https://pandas.pydata.org/)
-and PyArrow.
-
-## Create dataset
-
-First, we need to connect to a `LanceDB` database.
-
-```py
-
-import lancedb
-
-db = lancedb.connect("data/sample-lancedb")
-```
-
-Afterwards, we write a `Pandas DataFrame` to LanceDB directly.
-
-```py
-import pandas as pd
-
-data = pd.DataFrame({
-    "vector": [[3.1, 4.1], [5.9, 26.5]],
-    "item": ["foo", "bar"],
-    "price": [10.0, 20.0]
-})
-table = db.create_table("pd_table", data=data)
-```
-
-Similar to [`pyarrow.write_dataset()`](https://arrow.apache.org/docs/python/generated/pyarrow.dataset.write_dataset.html),
-[db.create_table()](../python/#lancedb.db.DBConnection.create_table) accepts a wide-range of forms of data.
-
-For example, if you have a dataset that is larger than memory size, you can create table with `Iterator[pyarrow.RecordBatch]`,
-to lazily generate data:
-
-```py
-
-from typing import Iterable
-import pyarrow as pa
-import lancedb
-
-def make_batches() -> Iterable[pa.RecordBatch]:
-    for i in range(5):
-        yield pa.RecordBatch.from_arrays(
-            [
-                pa.array([[3.1, 4.1], [5.9, 26.5]]),
-                pa.array(["foo", "bar"]),
-                pa.array([10.0, 20.0]),
-            ],
-            ["vector", "item", "price"])
-
-schema=pa.schema([
-    pa.field("vector", pa.list_(pa.float32())),
-    pa.field("item", pa.utf8()),
-    pa.field("price", pa.float32()),
-])
-
-table = db.create_table("iterable_table", data=make_batches(), schema=schema)
-```
-
-You will find detailed instructions of creating dataset in
-[Basic Operations](../basic.md) and [API](../python/#lancedb.db.DBConnection.create_table)
-sections.
-
-## Vector Search
-
-We can now perform similarity search via `LanceDB` Python API.
-
-```py
-# Open the table previously created.
-table = db.open_table("pd_table")
-
-query_vector = [100, 100]
-# Pandas DataFrame
-df = table.search(query_vector).limit(1).to_df()
-print(df)
-```
-
-```
-    vector     item  price        score
-0  [5.9, 26.5]  bar   20.0  14257.05957
-```
-
-If you have a simple filter, it's faster to provide a `where clause` to `LanceDB`'s search query.
-If you have more complex criteria, you can always apply the filter to the resulting Pandas `DataFrame`.
-
-```python
-
-# Apply the filter via LanceDB
-results = table.search([100, 100]).where("price < 15").to_df()
-assert len(results) == 1
-assert results["item"].iloc[0] == "foo"
-
-# Apply the filter via Pandas
-df = results = table.search([100, 100]).to_df()
-results = df[df.price < 15]
-assert len(results) == 1
-assert results["item"].iloc[0] == "foo"
-```

docs/src/python/duckdb.md

@@ -1,56 +0,0 @@
-# DuckDB
-
-`LanceDB` works with `DuckDB` via [PyArrow integration](https://duckdb.org/docs/guides/python/sql_on_arrow).
-
-Let us start with installing `duckdb` and `lancedb`.
-
-```shell
-pip install duckdb lancedb
-```
-
-We will re-use [the dataset created previously](./arrow.md):
-
-```python
-import pandas as pd
-import lancedb
-
-db = lancedb.connect("data/sample-lancedb")
-data = pd.DataFrame({
-    "vector": [[3.1, 4.1], [5.9, 26.5]],
-    "item": ["foo", "bar"],
-    "price": [10.0, 20.0]
-})
-table = db.create_table("pd_table", data=data)
-arrow_table = table.to_arrow()
-```
-
-`DuckDB` can directly query the `arrow_table`:
-
-```python
-import duckdb
-
-duckdb.query("SELECT * FROM arrow_table")
-```
-
-```
-┌─────────────┬─────────┬────────┐
-│   vector    │  item   │ price  │
-│   float[]   │ varchar │ double │
-├─────────────┼─────────┼────────┤
-│ [3.1, 4.1]  │ foo     │   10.0 │
-│ [5.9, 26.5] │ bar     │   20.0 │
-└─────────────┴─────────┴────────┘
-```
-
-```py
-duckdb.query("SELECT mean(price) FROM arrow_table")
-```
-
-```
-┌─────────────┐
-│ mean(price) │
-│   double    │
-├─────────────┤
-│        15.0 │
-└─────────────┘
-```

docs/src/python/integration.md

@@ -1,7 +0,0 @@
-# Integration
-
-Built on top of [Apache Arrow](https://arrow.apache.org/),
-`LanceDB` is very easy to be integrate with Python ecosystems.
-
-* [Pandas and Arrow Integration](./arrow.md)
-* [DuckDB Integration](./duckdb.md)

docs/src/python/pydantic.md

@@ -1,35 +0,0 @@
-# Pydantic
-
-[Pydantic](https://docs.pydantic.dev/latest/) is a data validation library in Python.
-
-## Schema
-
-LanceDB supports to create Apache Arrow Schema from a
-[Pydantic BaseModel](https://docs.pydantic.dev/latest/api/main/#pydantic.main.BaseModel)
-via [pydantic_to_schema()](python.md##lancedb.pydantic.pydantic_to_schema) method.
-
-::: lancedb.pydantic.pydantic_to_schema
-
-## Vector Field
-
-LanceDB provides a [`vector(dim)`](python.md#lancedb.pydantic.vector) method to define a
-vector Field in a Pydantic Model.
-
-::: lancedb.pydantic.vector
-
-## Type Conversion
-
-LanceDB automatically convert Pydantic fields to
-[Apache Arrow DataType](https://arrow.apache.org/docs/python/generated/pyarrow.DataType.html#pyarrow.DataType).
-
-Current supported type conversions:
-
-| Pydantic Field Type | PyArrow Data Type |
-| ------------------- | ----------------- |
-| `int`               | `pyarrow.int64`   |
-| `float`              | `pyarrow.float64`  |
-| `bool`              | `pyarrow.bool`    |
-| `str`               | `pyarrow.utf8()`    |
-| `list`              | `pyarrow.List`    |
-| `BaseModel`         | `pyarrow.Struct`    |
-| `vector(n)`         | `pyarrow.FixedSizeList(float32, n)` |

docs/src/python/python.md

@@ -1,59 +0,0 @@
-# LanceDB Python API Reference
-
-## Installation
-
-```shell
-pip install lancedb
-```
-
-## Connection
-
-::: lancedb.connect
-
-::: lancedb.db.DBConnection
-
-## Table
-
-::: lancedb.table.Table
-
-## Querying
-
-::: lancedb.query.Query
-
-::: lancedb.query.LanceQueryBuilder
-
-::: lancedb.query.LanceFtsQueryBuilder
-
-## Embeddings
-
-::: lancedb.embeddings.with_embeddings
-
-::: lancedb.embeddings.EmbeddingFunction
-
-## Context
-
-::: lancedb.context.contextualize
-
-::: lancedb.context.Contextualizer
-
-## Full text search
-
-::: lancedb.fts.create_index
-
-::: lancedb.fts.populate_index
-
-::: lancedb.fts.search_index
-
-## Utilities
-
-::: lancedb.vector
-
-## Integrations
-
-### Pydantic
-
-::: lancedb.pydantic.pydantic_to_schema
-
-::: lancedb.pydantic.vector
-
-

docs/src/search.md

@@ -1,121 +0,0 @@
-# Vector Search
-
-`Vector Search` finds the nearest vectors from the database.
-In a recommendation system or search engine, you can find similar products from
-the one you searched.
-In LLM and other AI applications,
-each data point can be [presented by the embeddings generated from some models](embedding.md),
-it returns the most relevant features.
-
-A search in high-dimensional vector space, is to find `K-Nearest-Neighbors (KNN)` of the query vector.
-
-## Metric
-
-In LanceDB, a `Metric` is the way to describe the distance between a pair of vectors.
-Currently, we support the following metrics:
-
-| Metric      | Description                          |
-| ----------- | ------------------------------------ |
-| `L2`        | [Euclidean / L2 distance](https://en.wikipedia.org/wiki/Euclidean_distance) |
-| `Cosine`    | [Cosine Similarity](https://en.wikipedia.org/wiki/Cosine_similarity)|
-| `Dot`       | [Dot Production](https://en.wikipedia.org/wiki/Dot_product) |
-
-
-## Search
-
-### Flat Search
-
-If LanceDB does not create a vector index, LanceDB would need to scan (`Flat Search`) the entire vector column
-and compute the distance for each vector in order to find the closest matches.
-
-
-<!-- Setup Code
-```python
-import lancedb
-import numpy as np
-uri = "data/sample-lancedb"
-db = lancedb.connect(uri)
-
-data = [{"vector": row, "item": f"item {i}"}
-     for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]
-
-db.create_table("my_vectors", data=data)
-```
--->
-<!-- Setup Code
-```javascript
-const vectordb_setup = require('vectordb')
-const db_setup = await vectordb_setup.connect('data/sample-lancedb')
-
-let data = []
-for (let i = 0; i < 10_000; i++) {
-     data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
-}
-await db_setup.createTable('my_vectors', data)
-```
--->
-=== "Python"
-
-
-    ```python
-    import lancedb
-    import numpy as np
-
-    db = lancedb.connect("data/sample-lancedb")
-
-    tbl = db.open_table("my_vectors")
-
-    df = tbl.search(np.random.random((1536))) \
-        .limit(10) \
-        .to_df()
-    ```
-
-=== "JavaScript"
-
-    ```javascript
-    const vectordb = require('vectordb')
-    const db = await vectordb.connect('data/sample-lancedb')
-
-    const tbl = await db.openTable("my_vectors")
-
-    const results_1 = await tbl.search(Array(1536).fill(1.2))
-        .limit(10)
-        .execute()
-    ```
-
-By default, `l2` will be used as `Metric` type. You can customize the metric type
-as well.
-
-=== "Python"
-
-    ```python
-    df = tbl.search(np.random.random((1536))) \
-        .metric("cosine") \
-        .limit(10) \
-        .to_df()
-    ```
-
-
-=== "JavaScript"
-
-    ```javascript
-    const results_2 = await tbl.search(Array(1536).fill(1.2))
-        .metricType("cosine")
-        .limit(10)
-        .execute()
-    ```
-
-
-### Approximate Nearest Neighbor (ANN) Search with Vector Index.
-
-To accelerate vector retrievals, it is common to build vector indices.
-A vector index is a data structure specifically designed to efficiently organize and
-search vector data based on their similarity or distance metrics.
-By constructing a vector index, you can reduce the search space and avoid the need
-for brute-force scanning of the entire vector column.
-
-However, fast vector search using indices often entails making a trade-off with accuracy to some extent.
-This is why it is often called **Approximate Nearest Neighbors (ANN)** search, while the Flat Search (KNN)
-always returns 100% recall.
-
-See [ANN Index](ann_indexes.md) for more details.

docs/src/sql.md

@@ -1,120 +0,0 @@
-# SQL filters
-
-LanceDB embraces the utilization of standard SQL expressions as predicates for hybrid
-filters. It can be used during hybrid vector search and deletion operations.
-
-Currently, Lance supports a growing list of expressions.
-
-* ``>``, ``>=``, ``<``, ``<=``, ``=``
-* ``AND``, ``OR``, ``NOT``
-* ``IS NULL``, ``IS NOT NULL``
-* ``IS TRUE``, ``IS NOT TRUE``, ``IS FALSE``, ``IS NOT FALSE``
-* ``IN``
-* ``LIKE``, ``NOT LIKE``
-* ``CAST``
-* ``regexp_match(column, pattern)``
-
-For example, the following filter string is acceptable:
-<!-- Setup Code
-```python 
-import lancedb
-import numpy as np
-uri = "data/sample-lancedb"
-db = lancedb.connect(uri)
-
-data = [{"vector": row, "item": f"item {i}"}
-     for i, row in enumerate(np.random.random((10_000, 2)).astype('int'))]
-
-tbl = db.create_table("my_vectors", data=data)
-```
--->
-<!-- Setup Code
-```javascript 
-const vectordb = require('vectordb')
-const db = await vectordb.connect('data/sample-lancedb')
-
-let data = []
-for (let i = 0; i < 10_000; i++) {
-     data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
-}
-const tbl = await db.createTable('my_vectors', data)
-```
--->
-=== "Python"
-
-    ```python
-    tbl.search([100, 102]) \
-       .where("""(
-        (label IN [10, 20])
-        AND
-        (note.email IS NOT NULL)
-        ) OR NOT note.created
-        """)
-
-    ```
-=== "Javascript"
-
-    ```javascript
-    tbl.search([100, 102])
-       .where(`(
-            (label IN [10, 20])
-            AND
-            (note.email IS NOT NULL)
-        ) OR NOT note.created
-       `)
-    ```
-
-
-If your column name contains special characters or is a [SQL Keyword](https://docs.rs/sqlparser/latest/sqlparser/keywords/index.html),
-you can use backtick (`` ` ``) to escape it. For nested fields, each segment of the
-path must be wrapped in backticks.
-
-=== "SQL"
-    ```sql
-    `CUBE` = 10 AND `column name with space` IS NOT NULL
-      AND `nested with space`.`inner with space` < 2
-    ```
-
-!!! warning
-    Field names containing periods (``.``) are not supported.
-
-Literals for dates, timestamps, and decimals can be written by writing the string
-value after the type name. For example
-
-=== "SQL"
-    ```sql
-    date_col = date '2021-01-01'
-    and timestamp_col = timestamp '2021-01-01 00:00:00'
-    and decimal_col = decimal(8,3) '1.000'
-    ```
-
-For timestamp columns, the precision can be specified as a number in the type
-parameter. Microsecond precision (6) is the default.
-
-| SQL              | Time unit    |
-|------------------|--------------|
-| ``timestamp(0)`` | Seconds      |
-| ``timestamp(3)`` | Milliseconds |
-| ``timestamp(6)`` | Microseconds |
-| ``timestamp(9)`` | Nanoseconds  |
-
-LanceDB internally stores data in [Apache Arrow](https://arrow.apache.org/) format.
-The mapping from SQL types to Arrow types is:
-
-| SQL type | Arrow type |
-|----------|------------|
-| ``boolean`` | ``Boolean`` |
-| ``tinyint`` / ``tinyint unsigned`` | ``Int8`` / ``UInt8`` |
-| ``smallint`` / ``smallint unsigned`` | ``Int16`` / ``UInt16`` |
-| ``int`` or ``integer`` / ``int unsigned`` or ``integer unsigned`` | ``Int32`` / ``UInt32`` |
-| ``bigint`` / ``bigint unsigned`` | ``Int64`` / ``UInt64`` |
-| ``float`` | ``Float32`` |
-| ``double`` | ``Float64`` |
-| ``decimal(precision, scale)`` | ``Decimal128`` |
-| ``date`` | ``Date32`` |
-| ``timestamp`` | ``Timestamp`` [^1] |
-| ``string`` | ``Utf8`` |
-| ``binary`` | ``Binary`` |
-
-[^1]: See precision mapping in previous table.
-

docs/src/styles/global.css

@@ -1,6 +0,0 @@
-:root {
-    --md-primary-fg-color:        #625eff;
-    --md-primary-fg-color--dark:  #4338ca;
-    --md-text-font: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
-    --md-code-font: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
-}

docs/test/md_testing.js

@@ -1,52 +0,0 @@
-const glob = require("glob");
-const fs = require("fs");
-const path = require("path");
-
-const excludedFiles = [
-  "../src/fts.md",
-  "../src/embedding.md",
-  "../src/examples/serverless_lancedb_with_s3_and_lambda.md",
-  "../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
-  "../src/examples/transformerjs_embedding_search_nodejs.md",
-  "../src/examples/youtube_transcript_bot_with_nodejs.md",
-];
-const nodePrefix = "javascript";
-const nodeFile = ".js";
-const nodeFolder = "node";
-const globString = "../src/**/*.md";
-const asyncPrefix = "(async () => {\n";
-const asyncSuffix = "})();";
-
-function* yieldLines(lines, prefix, suffix) {
-  let inCodeBlock = false;
-  for (const line of lines) {
-    if (line.trim().startsWith(prefix + nodePrefix)) {
-      inCodeBlock = true;
-    } else if (inCodeBlock && line.trim().startsWith(suffix)) {
-      inCodeBlock = false;
-      yield "\n";
-    } else if (inCodeBlock) {
-      yield line;
-    }
-  }
-}
-
-const files = glob.sync(globString, { recursive: true });
-
-for (const file of files.filter((file) => !excludedFiles.includes(file))) {
-  const lines = [];
-  const data = fs.readFileSync(file, "utf-8");
-  const fileLines = data.split("\n");
-
-  for (const line of yieldLines(fileLines, "```", "```")) {
-    lines.push(line);
-  }
-
-  if (lines.length > 0) {
-    const fileName = path.basename(file, ".md");
-    const outPath = path.join(nodeFolder, fileName, `${fileName}${nodeFile}`);
-    console.log(outPath)
-    fs.mkdirSync(path.dirname(outPath), { recursive: true });
-    fs.writeFileSync(outPath, asyncPrefix + "\n" + lines.join("\n") + asyncSuffix);
-  }
-}

docs/test/md_testing.py

@@ -1,41 +0,0 @@
-import glob
-from typing import Iterator
-from pathlib import Path
-
-excluded_files = [
-    "../src/fts.md",
-    "../src/embedding.md",
-    "../src/examples/serverless_lancedb_with_s3_and_lambda.md",
-    "../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
-    "../src/examples/youtube_transcript_bot_with_nodejs.md"
-]
-
-python_prefix = "py"
-python_file = ".py"
-python_folder = "python"
-glob_string = "../src/**/*.md"
-
-def yield_lines(lines: Iterator[str], prefix: str, suffix: str):
-    in_code_block = False
-    # Python code has strict indentation
-    strip_length = 0
-    for line in lines:
-        if line.strip().startswith(prefix + python_prefix):
-            in_code_block = True
-            strip_length = len(line) - len(line.lstrip())
-        elif in_code_block and line.strip().startswith(suffix):
-            in_code_block = False
-            yield "\n"
-        elif in_code_block:
-            yield line[strip_length:]
-
-for file in filter(lambda file: file not in excluded_files, glob.glob(glob_string, recursive=True)):
-    with open(file, "r") as f:
-        lines = list(yield_lines(iter(f), "```", "```"))
-
-    if len(lines) > 0:
-        out_path = Path(python_folder) / Path(file).name.strip(".md") / (Path(file).name.strip(".md") + python_file)
-        print(out_path)
-        out_path.parent.mkdir(exist_ok=True, parents=True)
-        with open(out_path, "w") as out:
-            out.writelines(lines)

docs/test/package.json

@@ -1,13 +0,0 @@
-{
-  "name": "lancedb-docs-test",
-  "version": "1.0.0",
-  "description": "",
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "fs": "^0.0.1-security",
-    "glob": "^10.2.7",
-    "path": "^0.12.7",
-    "vectordb": "https://gitpkg.now.sh/lancedb/lancedb/node?main"
-  }
-}

docs/test/requirements.txt

@@ -1,5 +0,0 @@
-lancedb @ git+https://github.com/lancedb/lancedb.git#egg=subdir&subdirectory=python
-numpy
-pandas
-pylance
-duckdb

node/.eslintrc.js

@@ -12,6 +12,5 @@ module.exports = {
     sourceType: 'module'
   },
   rules: {
-    "@typescript-eslint/method-signature-style": "off",
   }
 }

node/.npmignore

@@ -1,4 +1,2 @@
 gen_test_data.py
 index.node
-dist/lancedb*.tgz
-vectordb*.tgz

node/CHANGELOG.md

@@ -1,64 +0,0 @@
-# Changelog
-
-All notable changes to this project will be documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-## [0.1.5] - 2023-06-00
-
-### Added
-
-- Support for macOS X86
-
-## [0.1.4] - 2023-06-03
-
-### Added
-
-- Select / Project query API
-
-### Changed
-
--  Deprecated created_index in favor of createIndex
-
-## [0.1.3] - 2023-06-01
-
-### Added
-
-- Support S3 and Google Cloud Storage
-- Embedding functions support
-- OpenAI embedding function
-
-## [0.1.2] - 2023-05-27
-
-### Added
-
-- Append records API
-- Extra query params to to nodejs client
-- Create_index API
- 
-### Fixed
-
-- bugfix: string columns should be converted to Utf8Array (#94)
-
-## [0.1.1] - 2023-05-16
-
-### Added
-
-- create_table API
-- limit parameter for queries
-- Typescript / JavaScript examples
-- Linux support
-
-## [0.1.0] - 2023-05-16
-
-### Added
-
-- Initial  JavaScript / Node.js library for LanceDB
-- Read-only api to query LanceDB datasets
-- Supports macOS arm only
-
-## [pre-0.1.0]
-
-- Various prototypes / test builds
-

node/README.md

@@ -9,8 +9,7 @@ npm install vectordb
 ```
 
 This will download the appropriate native library for your platform. We currently
-support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
-yet support Windows or musl-based Linux (such as Alpine Linux).
+support x86_64 Linux, Intel MacOS, and ARM (M1/M2) MacOS.
 
 ## Usage
 
@@ -18,11 +17,9 @@ yet support Windows or musl-based Linux (such as Alpine Linux).
 
 ```javascript
 const lancedb = require('vectordb');
-const db = await lancedb.connect('data/sample-lancedb');
-const table = await db.createTable("my_table",
-      [{ id: 1, vector: [0.1, 1.0], item: "foo", price: 10.0 },
-      { id: 2, vector: [3.9, 0.5], item: "bar", price: 20.0 }])
-const results = await table.search([0.1, 0.3]).limit(20).execute();
+const db = lancedb.connect('<PATH_TO_LANCEDB_DATASET>');
+const table = await db.openTable('my_table');
+const query = await table.search([0.1, 0.3]).setLimit(20).execute();
 console.log(results);
 ```
 
@@ -30,42 +27,33 @@ The [examples](./examples) folder contains complete examples.
 
 ## Development
 
-To build everything fresh:
+Build and install the rust library with:
 
 ```bash
-npm install
-npm run tsc
 npm run build
+npm run pack-build
+npm install --no-save ./dist/vectordb-*.tgz
 ```
 
-Then you should be able to run the tests with:
+`npm run build` builds the Rust library, `npm run pack-build` packages the Rust
+binary into an npm module called `@vectordb/<platform>` (for example, 
+`@vectordb/darwin-arm64.node`), and then `npm run install ...` installs that
+module.
+
+The LanceDB javascript is built with npm:
+
+```bash
+npm run tsc
+```
+
+Run the tests with
 
 ```bash
 npm test
 ```
 
-### Rebuilding Rust library
-
-```bash
-npm run build
-```
-
-### Rebuilding Typescript
-
-```bash
-npm run tsc
-```
-
-### Fix lints
-
 To run the linter and have it automatically fix all errors
 
 ```bash
 npm run lint -- --fix
 ```
-
-To build documentation
-
-```bash
-npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
-```

node/examples/js-openai/index.js

@@ -1,41 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-'use strict'
-
-async function example () {
-  const lancedb = require('vectordb')
-  // You need to provide an OpenAI API key, here we read it from the OPENAI_API_KEY environment variable
-  const apiKey = process.env.OPENAI_API_KEY
-  // The embedding function will create embeddings for the 'text' column(text in this case)
-  const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
-
-  const db = await lancedb.connect('data/sample-lancedb')
-
-  const data = [
-    { id: 1, text: 'Black T-Shirt', price: 10 },
-    { id: 2, text: 'Leather Jacket', price: 50 }
-  ]
-
-  const table = await db.createTable('vectors', data, embedding)
-  console.log(await db.tableNames())
-
-  const results = await table
-    .search('keeps me warm')
-    .limit(1)
-    .execute()
-  console.log(results[0].text)
-}
-
-example().then(_ => { console.log('All done!') })

node/examples/js-openai/package.json

@@ -1,15 +0,0 @@
-{
-  "name": "vectordb-example-js-openai",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "author": "Lance Devs",
-  "license": "Apache-2.0",
-  "dependencies": {
-    "vectordb": "file:../..",
-    "openai": "^3.2.1"
-  }
-}

node/examples/js-transformers/index.js

@@ -1,66 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-'use strict'
-
-
-async function example() {
-
-    const lancedb = require('vectordb')
-
-    // Import transformers and the all-MiniLM-L6-v2 model (https://huggingface.co/Xenova/all-MiniLM-L6-v2)
-    const { pipeline } = await import('@xenova/transformers')
-    const pipe = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
-
-
-    // Create embedding function from pipeline which returns a list of vectors from batch
-    // sourceColumn is the name of the column in the data to be embedded
-    //
-    // Output of pipe is a Tensor { data: Float32Array(384) }, so filter for the vector
-    const embed_fun = {}
-    embed_fun.sourceColumn = 'text'
-    embed_fun.embed = async function (batch) {
-        let result = []
-        for (let text of batch) {
-            const res = await pipe(text, { pooling: 'mean', normalize: true })
-            result.push(Array.from(res['data']))
-        }
-        return (result)
-    }
-
-    // Link a folder and create a table with data
-    const db = await lancedb.connect('data/sample-lancedb')
-
-    const data = [
-        { id: 1, text: 'Cherry', type: 'fruit' },
-        { id: 2, text: 'Carrot', type: 'vegetable' },
-        { id: 3, text: 'Potato', type: 'vegetable' },
-        { id: 4, text: 'Apple', type: 'fruit' },
-        { id: 5, text: 'Banana', type: 'fruit' }
-    ]
-
-    const table = await db.createTable('food_table', data, "create", embed_fun)
-
-
-    // Query the table
-    const results = await table
-        .search("a sweet fruit to eat")
-        .metricType("cosine")
-        .limit(2)
-        .execute()
-    console.log(results.map(r => r.text))
-
-}
-
-example().then(_ => { console.log("Done!") })

node/examples/js-transformers/package.json

@@ -1,16 +0,0 @@
-{
-  "name": "vectordb-example-js-transformers",
-  "version": "1.0.0",
-  "description": "Example for using transformers.js with lancedb",
-  "main": "index.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "author": "Lance Devs",
-  "license": "Apache-2.0",
-  "dependencies": {
-    "@xenova/transformers": "^2.4.1",
-    "vectordb": "^0.1.12"
-  }
-
-}

node/examples/js-youtube-transcripts/index.js

@@ -1,122 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-'use strict'
-
-const lancedb = require('vectordb')
-const fs = require('fs/promises')
-const readline = require('readline/promises')
-const { stdin: input, stdout: output } = require('process')
-const { Configuration, OpenAIApi } = require('openai')
-
-// Download file from XYZ
-const INPUT_FILE_NAME = 'data/youtube-transcriptions_sample.jsonl';
-
-(async () => {
-  // You need to provide an OpenAI API key, here we read it from the OPENAI_API_KEY environment variable
-  const apiKey = process.env.OPENAI_API_KEY
-  // The embedding function will create embeddings for the 'context' column
-  const embedFunction = new lancedb.OpenAIEmbeddingFunction('context', apiKey)
-
-  // Connects to LanceDB
-  const db = await lancedb.connect('data/youtube-lancedb')
-
-  // Open the vectors table or create one if it does not exist
-  let tbl
-  if ((await db.tableNames()).includes('vectors')) {
-    tbl = await db.openTable('vectors', embedFunction)
-  } else {
-    tbl = await createEmbeddingsTable(db, embedFunction)
-  }
-
-  // Use OpenAI Completion API to generate and answer based on the context that LanceDB provides
-  const configuration = new Configuration({ apiKey })
-  const openai = new OpenAIApi(configuration)
-  const rl = readline.createInterface({ input, output })
-  try {
-    while (true) {
-      const query = await rl.question('Prompt: ')
-      const results = await tbl
-        .search(query)
-        .select(['title', 'text', 'context'])
-        .limit(3)
-        .execute()
-
-      // console.table(results)
-
-      const response = await openai.createCompletion({
-        model: 'text-davinci-003',
-        prompt: createPrompt(query, results),
-        max_tokens: 400,
-        temperature: 0,
-        top_p: 1,
-        frequency_penalty: 0,
-        presence_penalty: 0
-      })
-      console.log(response.data.choices[0].text)
-    }
-  } catch (err) {
-    console.log('Error: ', err)
-  } finally {
-    rl.close()
-  }
-  process.exit(1)
-})()
-
-async function createEmbeddingsTable (db, embedFunction) {
-  console.log(`Creating embeddings from ${INPUT_FILE_NAME}`)
-  // read the input file into a JSON array, skipping empty lines
-  const lines = (await fs.readFile(INPUT_FILE_NAME, 'utf-8'))
-    .toString()
-    .split('\n')
-    .filter(line => line.length > 0)
-    .map(line => JSON.parse(line))
-
-  const data = contextualize(lines, 20, 'video_id')
-  return await db.createTable('vectors', data, embedFunction)
-}
-
-// Each transcript has a small text column, we include previous transcripts in order to
-// have more context information when creating embeddings
-function contextualize (rows, contextSize, groupColumn) {
-  const grouped = []
-  rows.forEach(row => {
-    if (!grouped[row[groupColumn]]) {
-      grouped[row[groupColumn]] = []
-    }
-    grouped[row[groupColumn]].push(row)
-  })
-
-  const data = []
-  Object.keys(grouped).forEach(key => {
-    for (let i = 0; i < grouped[key].length; i++) {
-      const start = i - contextSize > 0 ? i - contextSize : 0
-      grouped[key][i].context = grouped[key].slice(start, i + 1).map(r => r.text).join(' ')
-    }
-    data.push(...grouped[key])
-  })
-  return data
-}
-
-// Creates a prompt by aggregating all relevant contexts
-function createPrompt (query, context) {
-  let prompt =
-      'Answer the question based on the context below.\n\n' +
-      'Context:\n'
-
-  // need to make sure our prompt is not larger than max size
-  prompt = prompt + context.map(c => c.context).join('\n\n---\n\n').substring(0, 3750)
-  prompt = prompt + `\n\nQuestion: ${query}\nAnswer:`
-  return prompt
-}

node/examples/js-youtube-transcripts/package.json

@@ -1,15 +0,0 @@
-{
-  "name": "vectordb-example-js-openai",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "author": "Lance Devs",
-  "license": "Apache-2.0",
-  "dependencies": {
-    "vectordb": "file:../..",
-    "openai": "^3.2.1"
-  }
-}

node/examples/js/package.json

@@ -9,6 +9,6 @@
   "author": "Lance Devs",
   "license": "Apache-2.0",
   "dependencies": {
-    "vectordb": "file:../.."
+    "vectordb": "^0.1.0"
   }
 }

node/examples/ts/package.json

@@ -17,6 +17,6 @@
     "typescript": "*"
   },
   "dependencies": {
-    "vectordb": "file:../.."
+    "vectordb": "^0.1.0"
   }
 }

node/gen_test_data.py

@@ -0,0 +1,8 @@
+import lancedb
+
+uri = "sample-lancedb"
+db = lancedb.connect(uri)
+table = db.create_table("my_table",
+                         data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                               {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+

node/native.js

@@ -17,20 +17,14 @@ const { currentTarget } = require('@neon-rs/load');
 let nativeLib;
 
 try {
-    nativeLib = require(`vectordb-${currentTarget()}`);
+    nativeLib = require(`@vectordb/${currentTarget()}`);
 } catch (e) {
-    try {
-        // Might be developing locally, so try that. But don't expose that error
-        // to the user.
-        nativeLib = require("./index.node");
-    } catch {
-        throw new Error(`vectordb: failed to load native library.
-  You may need to run \`npm install vectordb-${currentTarget()}\`.
-      
+    throw new Error(`vectordb: failed to load native library.
+  You may need to run \`npm install @vectordb/${currentTarget()}\`.
+
   If that does not work, please file a bug report at https://github.com/lancedb/lancedb/issues
-      
+
   Source error: ${e}`);
-    }   
 }
 
 // Dynamic require for runtime.

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.1.13",
+  "version": "0.1.2",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -8,11 +8,10 @@
     "tsc": "tsc -b",
     "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json",
     "build-release": "npm run build -- --release",
-    "test": "npm run tsc && mocha -recursive dist/test",
+    "cross-release": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cross build --message-format=json --release -p vectordb-node",
+    "test": "mocha -recursive dist/test",
     "lint": "eslint src --ext .js,.ts",
-    "clean": "rm -rf node_modules *.node dist/",
-    "pack-build": "neon pack-build",
-    "check-npm": "printenv && which node && which npm && npm --version"
+    "pack-build": "neon pack-build"
   },
   "repository": {
     "type": "git",
@@ -29,28 +28,21 @@
   "devDependencies": {
     "@neon-rs/cli": "^0.0.74",
     "@types/chai": "^4.3.4",
-    "@types/chai-as-promised": "^7.1.5",
     "@types/mocha": "^10.0.1",
     "@types/node": "^18.16.2",
-    "@types/sinon": "^10.0.15",
     "@types/temp": "^0.9.1",
     "@typescript-eslint/eslint-plugin": "^5.59.1",
     "cargo-cp-artifact": "^0.1",
     "chai": "^4.3.7",
-    "chai-as-promised": "^7.1.1",
     "eslint": "^8.39.0",
     "eslint-config-standard-with-typescript": "^34.0.1",
-    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-import": "^2.27.5",
     "eslint-plugin-n": "^15.7.0",
     "eslint-plugin-promise": "^6.1.1",
     "mocha": "^10.2.0",
-    "openai": "^3.2.1",
-    "sinon": "^15.1.0",
     "temp": "^0.9.4",
     "ts-node": "^10.9.1",
     "ts-node-dev": "^2.0.0",
-    "typedoc": "^0.24.7",
-    "typedoc-plugin-markdown": "^3.15.3",
     "typescript": "*"
   },
   "dependencies": {
@@ -60,8 +52,7 @@
   },
   "os": [
     "darwin",
-    "linux",
-    "win32"
+    "linux"
   ],
   "cpu": [
     "x64",
@@ -69,18 +60,20 @@
   ],
   "neon": {
     "targets": {
-      "x86_64-apple-darwin": "vectordb-darwin-x64",
-      "aarch64-apple-darwin": "vectordb-darwin-arm64",
-      "x86_64-unknown-linux-gnu": "vectordb-linux-x64-gnu",
-      "aarch64-unknown-linux-gnu": "vectordb-linux-arm64-gnu",
-      "x86_64-pc-windows-msvc": "vectordb-win32-x64-msvc"
+      "x86_64-apple-darwin": "@vectordb/darwin-x64",
+      "aarch64-apple-darwin": "@vectordb/darwin-arm64",
+      "x86_64-unknown-linux-gnu": "@vectordb/linux-x64-gnu",
+      "x86_64-unknown-linux-musl": "@vectordb/linux-x64-musl",
+      "aarch64-unknown-linux-gnu": "@vectordb/linux-arm64-gnu",
+      "aarch64-unknown-linux-musl": "@vectordb/linux-arm64-musl"
     }
   },
   "optionalDependencies": {
-    "vectordb-darwin-arm64": "0.1.13",
-    "vectordb-darwin-x64": "0.1.13",
-    "vectordb-linux-arm64-gnu": "0.1.13",
-    "vectordb-linux-x64-gnu": "0.1.13",
-    "vectordb-win32-x64-msvc": "0.1.13"
+    "@vectordb/darwin-arm64": "0.1.2",
+    "@vectordb/darwin-x64": "0.1.2",
+    "@vectordb/linux-x64-gnu": "0.1.2",
+    "@vectordb/linux-x64-musl": "0.1.2",
+    "@vectordb/linux-arm64-gnu": "0.1.2",
+    "@vectordb/linux-arm64-musl": "0.1.2"
   }
 }

node/src/arrow.ts

@@ -15,16 +15,15 @@
 import {
   Field,
   Float32,
-  List, type ListBuilder,
+  List,
   makeBuilder,
   RecordBatchFileWriter,
   Table, Utf8,
   type Vector,
   vectorFromArray
 } from 'apache-arrow'
-import { type EmbeddingFunction } from './index'
 
-export async function convertToTable<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Table> {
+export function convertToTable (data: Array<Record<string, unknown>>): Table {
   if (data.length === 0) {
     throw new Error('At least one record needs to be provided')
   }
@@ -34,7 +33,11 @@ export async function convertToTable<T> (data: Array<Record<string, unknown>>, e
 
   for (const columnsKey of columns) {
     if (columnsKey === 'vector') {
-      const listBuilder = newVectorListBuilder()
+      const children = new Field<Float32>('item', new Float32())
+      const list = new List(children)
+      const listBuilder = makeBuilder({
+        type: list
+      })
       const vectorSize = (data[0].vector as any[]).length
       for (const datum of data) {
         if ((datum[columnsKey] as any[]).length !== vectorSize) {
@@ -49,14 +52,6 @@ export async function convertToTable<T> (data: Array<Record<string, unknown>>, e
       for (const datum of data) {
         values.push(datum[columnsKey])
       }
-
-      if (columnsKey === embeddings?.sourceColumn) {
-        const vectors = await embeddings.embed(values as T[])
-        const listBuilder = newVectorListBuilder()
-        vectors.map(v => listBuilder.append(v))
-        records.vector = listBuilder.finish().toVector()
-      }
-
       if (typeof values[0] === 'string') {
         // `vectorFromArray` converts strings into dictionary vectors, forcing it back to a string column
         records[columnsKey] = vectorFromArray(values, new Utf8())
@@ -69,17 +64,8 @@ export async function convertToTable<T> (data: Array<Record<string, unknown>>, e
   return new Table(records)
 }
 
-// Creates a new Arrow ListBuilder that stores a Vector column
-function newVectorListBuilder (): ListBuilder<Float32, any> {
-  const children = new Field<Float32>('item', new Float32())
-  const list = new List(children)
-  return makeBuilder({
-    type: list
-  })
-}
-
-export async function fromRecordsToBuffer<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Buffer> {
-  const table = await convertToTable(data, embeddings)
+export async function fromRecordsToBuffer (data: Array<Record<string, unknown>>): Promise<Buffer> {
+  const table = convertToTable(data)
   const writer = RecordBatchFileWriter.writeAll(table)
   return Buffer.from(await writer.toUint8Array())
 }

node/src/embedding/embedding_function.ts

@@ -1,28 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-/**
- * An embedding function that automatically creates vector representation for a given column.
- */
-export interface EmbeddingFunction<T> {
-  /**
-     * The name of the column that will be used as input for the Embedding Function.
-     */
-  sourceColumn: string
-
-  /**
-     * Creates a vector representation for the given values.
-     */
-  embed: (data: T[]) => Promise<number[][]>
-}

node/src/embedding/openai.ts

@@ -1,51 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { type EmbeddingFunction } from '../index'
-
-export class OpenAIEmbeddingFunction implements EmbeddingFunction<string> {
-  private readonly _openai: any
-  private readonly _modelName: string
-
-  constructor (sourceColumn: string, openAIKey: string, modelName: string = 'text-embedding-ada-002') {
-    let openai
-    try {
-      // eslint-disable-next-line @typescript-eslint/no-var-requires
-      openai = require('openai')
-    } catch {
-      throw new Error('please install openai using npm install openai')
-    }
-
-    this.sourceColumn = sourceColumn
-    const configuration = new openai.Configuration({
-      apiKey: openAIKey
-    })
-    this._openai = new openai.OpenAIApi(configuration)
-    this._modelName = modelName
-  }
-
-  async embed (data: string[]): Promise<number[][]> {
-    const response = await this._openai.createEmbedding({
-      model: this._modelName,
-      input: data
-    })
-    const embeddings: number[][] = []
-    for (let i = 0; i < response.data.data.length; i++) {
-      embeddings.push(response.data.data[i].embedding as number[])
-    }
-    return embeddings
-  }
-
-  sourceColumn: string
-}

node/src/index.ts

@@ -19,148 +19,32 @@ import {
   Vector
 } from 'apache-arrow'
 import { fromRecordsToBuffer } from './arrow'
-import type { EmbeddingFunction } from './embedding/embedding_function'
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableSearch, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete } = require('../native.js')
-
-export type { EmbeddingFunction }
-export { OpenAIEmbeddingFunction } from './embedding/openai'
-
-export interface AwsCredentials {
-  accessKeyId: string
-
-  secretKey: string
-
-  sessionToken?: string
-}
-
-export interface ConnectionOptions {
-  uri: string
-  awsCredentials?: AwsCredentials
-}
+const { databaseNew, databaseTableNames, databaseOpenTable, tableCreate, tableSearch, tableAdd, tableCreateVectorIndex } = require('../native.js')
 
 /**
  * Connect to a LanceDB instance at the given URI
  * @param uri The uri of the database.
  */
-export async function connect (uri: string): Promise<Connection>
-export async function connect (opts: Partial<ConnectionOptions>): Promise<Connection>
-export async function connect (arg: string | Partial<ConnectionOptions>): Promise<Connection> {
-  let opts: ConnectionOptions
-  if (typeof arg === 'string') {
-    opts = { uri: arg }
-  } else {
-    // opts = { uri: arg.uri, awsCredentials = arg.awsCredentials }
-    opts = Object.assign({
-      uri: '',
-      awsCredentials: undefined
-    }, arg)
-  }
-  const db = await databaseNew(opts.uri)
-  return new LocalConnection(db, opts)
-}
-
-/**
- * A LanceDB Connection that allows you to open tables and create new ones.
- *
- * Connection could be local against filesystem or remote against a server.
- */
-export interface Connection {
-  uri: string
-
-  tableNames(): Promise<string[]>
-
-  /**
-   * Open a table in the database.
-   *
-   * @param name The name of the table.
-   * @param embeddings An embedding function to use on this table
-   */
-  openTable<T>(name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
-
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param {string} name - The name of the table.
-   * @param data - Non-empty Array of Records to be inserted into the table
-   * @param {WriteMode} mode - The write mode to use when creating the table.
-   * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
-   */
-  createTable<T>(name: string, data: Array<Record<string, unknown>>, mode?: WriteMode, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
-
-  createTableArrow(name: string, table: ArrowTable): Promise<Table>
-
-  /**
-   * Drop an existing table.
-   * @param name The name of the table to drop.
-   */
-  dropTable(name: string): Promise<void>
-
-}
-
-/**
- * A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
- */
-export interface Table<T = number[]> {
-  name: string
-
-  /**
-   * Creates a search query to find the nearest neighbors of the given search term
-   * @param query The query search term
-   */
-  search: (query: T) => Query<T>
-
-  /**
-   * Insert records into this Table.
-   *
-   * @param data Records to be inserted into the Table
-   * @return The number of rows added to the table
-   */
-  add: (data: Array<Record<string, unknown>>) => Promise<number>
-
-  /**
-   * Insert records into this Table, replacing its contents.
-   *
-   * @param data Records to be inserted into the Table
-   * @return The number of rows added to the table
-   */
-  overwrite: (data: Array<Record<string, unknown>>) => Promise<number>
-
-  /**
-   * Create an ANN index on this Table vector index.
-   *
-   * @param indexParams The parameters of this Index, @see VectorIndexParams.
-   */
-  createIndex: (indexParams: VectorIndexParams) => Promise<any>
-
-  /**
-   * Returns the number of rows in this table.
-   */
-  countRows: () => Promise<number>
-
-  /**
-   * Delete rows from this table.
-   *
-   * @param filter  A filter in the same format used by a sql WHERE clause.
-   */
-  delete: (filter: string) => Promise<void>
+export async function connect (uri: string): Promise<Connection> {
+  return new Connection(uri)
 }
 
 /**
  * A connection to a LanceDB database.
  */
-export class LocalConnection implements Connection {
-  private readonly _options: ConnectionOptions
+export class Connection {
+  private readonly _uri: string
   private readonly _db: any
 
-  constructor (db: any, options: ConnectionOptions) {
-    this._options = options
-    this._db = db
+  constructor (uri: string) {
+    this._uri = uri
+    this._db = databaseNew(uri)
   }
 
   get uri (): string {
-    return this._options.uri
+    return this._uri
   }
 
   /**
@@ -171,69 +55,17 @@ export class LocalConnection implements Connection {
   }
 
   /**
-   * Open a table in the database.
-   *
-   * @param name The name of the table.
-   */
-  async openTable (name: string): Promise<Table>
-  /**
-   * Open a table in the database.
-   *
-   * @param name The name of the table.
-   * @param embeddings An embedding function to use on this Table
-   */
-  async openTable<T> (name: string, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
-  async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
-  async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
+     * Open a table in the database.
+     * @param name The name of the table.
+     */
+  async openTable (name: string): Promise<Table> {
     const tbl = await databaseOpenTable.call(this._db, name)
-    if (embeddings !== undefined) {
-      return new LocalTable(tbl, name, this._options, embeddings)
-    } else {
-      return new LocalTable(tbl, name, this._options)
-    }
+    return new Table(tbl, name)
   }
 
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param name The name of the table.
-   * @param data Non-empty Array of Records to be inserted into the Table
-   * @param mode The write mode to use when creating the table.
-   */
-  async createTable (name: string, data: Array<Record<string, unknown>>, mode?: WriteMode): Promise<Table>
-  async createTable (name: string, data: Array<Record<string, unknown>>, mode: WriteMode): Promise<Table>
-
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param name The name of the table.
-   * @param data Non-empty Array of Records to be inserted into the Table
-   * @param mode The write mode to use when creating the table.
-   * @param embeddings An embedding function to use on this Table
-   */
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, mode: WriteMode, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, mode: WriteMode, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, mode: WriteMode, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
-    if (mode === undefined) {
-      mode = WriteMode.Create
-    }
-
-    const createArgs = [this._db, name, await fromRecordsToBuffer(data, embeddings), mode.toLowerCase()]
-    if (this._options.awsCredentials !== undefined) {
-      createArgs.push(this._options.awsCredentials.accessKeyId)
-      createArgs.push(this._options.awsCredentials.secretKey)
-      if (this._options.awsCredentials.sessionToken !== undefined) {
-        createArgs.push(this._options.awsCredentials.sessionToken)
-      }
-    }
-
-    const tbl = await tableCreate.call(...createArgs)
-
-    if (embeddings !== undefined) {
-      return new LocalTable(tbl, name, this._options, embeddings)
-    } else {
-      return new LocalTable(tbl, name, this._options)
-    }
+  async createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table> {
+    await tableCreate.call(this._db, name, await fromRecordsToBuffer(data))
+    return await this.openTable(name)
   }
 
   async createTableArrow (name: string, table: ArrowTable): Promise<Table> {
@@ -241,35 +73,18 @@ export class LocalConnection implements Connection {
     await tableCreate.call(this._db, name, Buffer.from(await writer.toUint8Array()))
     return await this.openTable(name)
   }
-
-  /**
-   * Drop an existing table.
-   * @param name The name of the table to drop.
-   */
-  async dropTable (name: string): Promise<void> {
-    await databaseDropTable.call(this._db, name)
-  }
 }
 
-export class LocalTable<T = number[]> implements Table<T> {
+/**
+ * A table in a LanceDB database.
+ */
+export class Table {
   private readonly _tbl: any
   private readonly _name: string
-  private readonly _embeddings?: EmbeddingFunction<T>
-  private readonly _options: ConnectionOptions
 
-  constructor (tbl: any, name: string, options: ConnectionOptions)
-  /**
-   * @param tbl
-   * @param name
-   * @param options
-   * @param embeddings An embedding function to use when interacting with this table
-   */
-  constructor (tbl: any, name: string, options: ConnectionOptions, embeddings: EmbeddingFunction<T>)
-  constructor (tbl: any, name: string, options: ConnectionOptions, embeddings?: EmbeddingFunction<T>) {
+  constructor (tbl: any, name: string) {
     this._tbl = tbl
     this._name = name
-    this._embeddings = embeddings
-    this._options = options
   }
 
   get name (): string {
@@ -277,11 +92,11 @@ export class LocalTable<T = number[]> implements Table<T> {
   }
 
   /**
-   * Creates a search query to find the nearest neighbors of the given search term
-   * @param query The query search term
-   */
-  search (query: T): Query<T> {
-    return new Query(this._tbl, query, this._embeddings)
+     * Create a search query to find the nearest neighbors of the given query vector.
+     * @param queryVector The query vector.
+     */
+  search (queryVector: number[]): Query {
+    return new Query(this._tbl, queryVector)
   }
 
   /**
@@ -291,15 +106,7 @@ export class LocalTable<T = number[]> implements Table<T> {
    * @return The number of rows added to the table
    */
   async add (data: Array<Record<string, unknown>>): Promise<number> {
-    const callArgs = [this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Append.toString()]
-    if (this._options.awsCredentials !== undefined) {
-      callArgs.push(this._options.awsCredentials.accessKeyId)
-      callArgs.push(this._options.awsCredentials.secretKey)
-      if (this._options.awsCredentials.sessionToken !== undefined) {
-        callArgs.push(this._options.awsCredentials.sessionToken)
-      }
-    }
-    return tableAdd.call(...callArgs)
+    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data), WriteMode.Append.toString())
   }
 
   /**
@@ -309,46 +116,15 @@ export class LocalTable<T = number[]> implements Table<T> {
    * @return The number of rows added to the table
    */
   async overwrite (data: Array<Record<string, unknown>>): Promise<number> {
-    const callArgs = [this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Overwrite.toString()]
-    if (this._options.awsCredentials !== undefined) {
-      callArgs.push(this._options.awsCredentials.accessKeyId)
-      callArgs.push(this._options.awsCredentials.secretKey)
-      if (this._options.awsCredentials.sessionToken !== undefined) {
-        callArgs.push(this._options.awsCredentials.sessionToken)
-      }
-    }
-    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Overwrite.toString())
+    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data), WriteMode.Overwrite.toString())
   }
 
-  /**
-   * Create an ANN index on this Table vector index.
-   *
-   * @param indexParams The parameters of this Index, @see VectorIndexParams.
-   */
-  async createIndex (indexParams: VectorIndexParams): Promise<any> {
+  async create_index (indexParams: VectorIndexParams): Promise<any> {
     return tableCreateVectorIndex.call(this._tbl, indexParams)
   }
-
-  /**
-   * Returns the number of rows in this table.
-   */
-  async countRows (): Promise<number> {
-    return tableCountRows.call(this._tbl)
-  }
-
-  /**
-   * Delete rows from this table.
-   *
-   * @param filter A filter in the same format used by a sql WHERE clause.
-   */
-  async delete (filter: string): Promise<void> {
-    return tableDelete.call(this._tbl, filter)
-  }
 }
 
-/// Config to build IVF_PQ index.
-///
-export interface IvfPQIndexConfig {
+interface IvfPQIndexConfig {
   /**
    * The column to be indexed
    */
@@ -393,11 +169,6 @@ export interface IvfPQIndexConfig {
    */
   max_opq_iters?: number
 
-  /**
-   * Replace an existing index with the same name if it exists.
-   */
-  replace?: boolean
-
   type: 'ivf_pq'
 }
 
@@ -406,35 +177,32 @@ export type VectorIndexParams = IvfPQIndexConfig
 /**
  * A builder for nearest neighbor queries for LanceDB.
  */
-export class Query<T = number[]> {
+export class Query {
   private readonly _tbl: any
-  private readonly _query: T
-  private _queryVector?: number[]
+  private readonly _queryVector: number[]
   private _limit: number
   private _refineFactor?: number
   private _nprobes: number
-  private _select?: string[]
+  private readonly _columns?: string[]
   private _filter?: string
   private _metricType?: MetricType
-  private readonly _embeddings?: EmbeddingFunction<T>
 
-  constructor (tbl: any, query: T, embeddings?: EmbeddingFunction<T>) {
+  constructor (tbl: any, queryVector: number[]) {
     this._tbl = tbl
-    this._query = query
+    this._queryVector = queryVector
     this._limit = 10
     this._nprobes = 20
     this._refineFactor = undefined
-    this._select = undefined
+    this._columns = undefined
     this._filter = undefined
     this._metricType = undefined
-    this._embeddings = embeddings
   }
 
   /***
    * Sets the number of results that will be returned
    * @param value number of results
    */
-  limit (value: number): Query<T> {
+  limit (value: number): Query {
     this._limit = value
     return this
   }
@@ -443,7 +211,7 @@ export class Query<T = number[]> {
    * Refine the results by reading extra elements and re-ranking them in memory.
    * @param value refine factor to use in this query.
    */
-  refineFactor (value: number): Query<T> {
+  refineFactor (value: number): Query {
     this._refineFactor = value
     return this
   }
@@ -452,7 +220,7 @@ export class Query<T = number[]> {
    * The number of probes used. A higher number makes search more accurate but also slower.
    * @param value The number of probes used.
    */
-  nprobes (value: number): Query<T> {
+  nprobes (value: number): Query {
     this._nprobes = value
     return this
   }
@@ -461,27 +229,16 @@ export class Query<T = number[]> {
    * A filter statement to be applied to this query.
    * @param value A filter in the same format used by a sql WHERE clause.
    */
-  filter (value: string): Query<T> {
+  filter (value: string): Query {
     this._filter = value
     return this
   }
 
-  where = this.filter
-
-  /** Return only the specified columns.
-   *
-   * @param value Only select the specified columns. If not specified, all columns will be returned.
-   */
-  select (value: string[]): Query<T> {
-    this._select = value
-    return this
-  }
-
   /**
    * The MetricType used for this Query.
    * @param value The metric to the. @see MetricType for the different options
    */
-  metricType (value: MetricType): Query<T> {
+  metricType (value: MetricType): Query {
     this._metricType = value
     return this
   }
@@ -490,15 +247,8 @@ export class Query<T = number[]> {
    * Execute the query and return the results as an Array of Objects
    */
   async execute<T = Record<string, unknown>> (): Promise<T[]> {
-    if (this._embeddings !== undefined) {
-      this._queryVector = (await this._embeddings.embed([this._query]))[0]
-    } else {
-      this._queryVector = this._query as number[]
-    }
-
     const buffer = await tableSearch.call(this._tbl, this)
     const data = tableFromIPC(buffer)
-
     return data.toArray().map((entry: Record<string, unknown>) => {
       const newObject: Record<string, unknown> = {}
       Object.keys(entry).forEach((key: string) => {
@@ -513,15 +263,8 @@ export class Query<T = number[]> {
   }
 }
 
-/**
- * Write mode for writing a table.
- */
 export enum WriteMode {
-  /** Create a new {@link Table}. */
-  Create = 'create',
-  /** Overwrite the existing {@link Table} if presented. */
   Overwrite = 'overwrite',
-  /** Append new data to the table. */
   Append = 'append'
 }
 
@@ -537,10 +280,5 @@ export enum MetricType {
   /**
    * Cosine distance
    */
-  Cosine = 'cosine',
-
-  /**
-   * Dot product
-   */
-  Dot = 'dot'
+  Cosine = 'cosine'
 }

node/src/test/embedding/openai.ts

@@ -1,50 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { describe } from 'mocha'
-import { assert } from 'chai'
-
-import { OpenAIEmbeddingFunction } from '../../embedding/openai'
-
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const { OpenAIApi } = require('openai')
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const { stub } = require('sinon')
-
-describe('OpenAPIEmbeddings', function () {
-  const stubValue = {
-    data: {
-      data: [
-        {
-          embedding: Array(1536).fill(1.0)
-        },
-        {
-          embedding: Array(1536).fill(2.0)
-        }
-      ]
-    }
-  }
-
-  describe('#embed', function () {
-    it('should create vector embeddings', async function () {
-      const openAIStub = stub(OpenAIApi.prototype, 'createEmbedding').returns(stubValue)
-      const f = new OpenAIEmbeddingFunction('text', 'sk-key')
-      const vectors = await f.embed(['abc', 'def'])
-      assert.isTrue(openAIStub.calledOnce)
-      assert.equal(vectors.length, 2)
-      assert.deepEqual(vectors[0], stubValue.data.data[0].embedding)
-      assert.deepEqual(vectors[1], stubValue.data.data[1].embedding)
-    })
-  })
-})

node/src/test/io.ts

@@ -1,74 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// IO tests
-
-import { describe } from 'mocha'
-import { assert } from 'chai'
-
-import * as lancedb from '../index'
-import { type ConnectionOptions } from '../index'
-
-describe('LanceDB S3 client', function () {
-  if (process.env.TEST_S3_BASE_URL != null) {
-    const baseUri = process.env.TEST_S3_BASE_URL
-    it('should have a valid url', async function () {
-      const opts = { uri: `${baseUri}/valid_url` }
-      const table = await createTestDB(opts, 2, 20)
-      const con = await lancedb.connect(opts)
-      assert.equal(con.uri, opts.uri)
-
-      const results = await table.search([0.1, 0.3]).limit(5).execute()
-      assert.equal(results.length, 5)
-    }).timeout(10_000)
-  } else {
-    describe.skip('Skip S3 test', function () {})
-  }
-
-  if (process.env.TEST_S3_BASE_URL != null && process.env.TEST_AWS_ACCESS_KEY_ID != null && process.env.TEST_AWS_SECRET_ACCESS_KEY != null) {
-    const baseUri = process.env.TEST_S3_BASE_URL
-    it('use custom credentials', async function () {
-      const opts: ConnectionOptions = {
-        uri: `${baseUri}/custom_credentials`,
-        awsCredentials: {
-          accessKeyId: process.env.TEST_AWS_ACCESS_KEY_ID as string,
-          secretKey: process.env.TEST_AWS_SECRET_ACCESS_KEY as string
-        }
-      }
-      const table = await createTestDB(opts, 2, 20)
-      const con = await lancedb.connect(opts)
-      assert.equal(con.uri, opts.uri)
-
-      const results = await table.search([0.1, 0.3]).limit(5).execute()
-      assert.equal(results.length, 5)
-    }).timeout(10_000)
-  } else {
-    describe.skip('Skip S3 test', function () {})
-  }
-})
-
-async function createTestDB (opts: ConnectionOptions, numDimensions: number = 2, numRows: number = 2): Promise<lancedb.Table> {
-  const con = await lancedb.connect(opts)
-
-  const data = []
-  for (let i = 0; i < numRows; i++) {
-    const vector = []
-    for (let j = 0; j < numDimensions; j++) {
-      vector.push(i + (j * 0.1))
-    }
-    data.push({ id: i + 1, name: `name_${i}`, price: i + 10, is_active: (i % 2 === 0), vector })
-  }
-
-  return await con.createTable('vectors', data)
-}

node/src/test/test.ts

@@ -1,4 +1,4 @@
-// Copyright 2023 LanceDB Developers.
+// Copyright 2023 Lance Developers.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,16 +13,11 @@
 // limitations under the License.
 
 import { describe } from 'mocha'
+import { assert } from 'chai'
 import { track } from 'temp'
-import * as chai from 'chai'
-import * as chaiAsPromised from 'chai-as-promised'
 
 import * as lancedb from '../index'
-import { type AwsCredentials, type EmbeddingFunction, MetricType, Query, WriteMode } from '../index'
-
-const expect = chai.expect
-const assert = chai.assert
-chai.use(chaiAsPromised)
+import { MetricType, Query } from '../index'
 
 describe('LanceDB client', function () {
   describe('when creating a connection to lancedb', function () {
@@ -32,22 +27,6 @@ describe('LanceDB client', function () {
       assert.equal(con.uri, uri)
     })
 
-    it('should accept an options object', async function () {
-      const uri = await createTestDB()
-      const con = await lancedb.connect({ uri })
-      assert.equal(con.uri, uri)
-    })
-
-    it('should accept custom aws credentials', async function () {
-      const uri = await createTestDB()
-      const awsCredentials: AwsCredentials = {
-        accessKeyId: '',
-        secretKey: ''
-      }
-      const con = await lancedb.connect({ uri, awsCredentials })
-      assert.equal(con.uri, uri)
-    })
-
     it('should return the existing table names', async function () {
       const uri = await createTestDB()
       const con = await lancedb.connect(uri)
@@ -85,36 +64,13 @@ describe('LanceDB client', function () {
       assert.equal(results[0].id, 1)
     })
 
-    it('uses a filter / where clause', async function () {
-      // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
-      const assertResults = (results: Array<Record<string, unknown>>) => {
-        assert.equal(results.length, 1)
-        assert.equal(results[0].id, 2)
-      }
-
+    it('uses a filter', async function () {
       const uri = await createTestDB()
       const con = await lancedb.connect(uri)
       const table = await con.openTable('vectors')
-      let results = await table.search([0.1, 0.1]).filter('id == 2').execute()
-      assertResults(results)
-      results = await table.search([0.1, 0.1]).where('id == 2').execute()
-      assertResults(results)
-    })
-
-    it('select only a subset of columns', async function () {
-      const uri = await createTestDB()
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-      const results = await table.search([0.1, 0.1]).select(['is_active']).execute()
-      assert.equal(results.length, 2)
-      // vector and score are always returned
-      assert.isDefined(results[0].vector)
-      assert.isDefined(results[0].score)
-      assert.isDefined(results[0].is_active)
-
-      assert.isUndefined(results[0].id)
-      assert.isUndefined(results[0].name)
-      assert.isUndefined(results[0].price)
+      const results = await table.search([0.1, 0.1]).filter('id == 2').execute()
+      assert.equal(results.length, 1)
+      assert.equal(results[0].id, 2)
     })
   })
 
@@ -131,32 +87,9 @@ describe('LanceDB client', function () {
       const tableName = `vectors_${Math.floor(Math.random() * 100)}`
       const table = await con.createTable(tableName, data)
       assert.equal(table.name, tableName)
-      assert.equal(await table.countRows(), 2)
-    })
 
-    it('use overwrite flag to overwrite existing table', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-
-      const data = [
-        { id: 1, vector: [0.1, 0.2], price: 10 },
-        { id: 2, vector: [1.1, 1.2], price: 50 }
-      ]
-
-      const tableName = 'overwrite'
-      await con.createTable(tableName, data, WriteMode.Create)
-
-      const newData = [
-        { id: 1, vector: [0.1, 0.2], price: 10 },
-        { id: 2, vector: [1.1, 1.2], price: 50 },
-        { id: 3, vector: [1.1, 1.2], price: 50 }
-      ]
-
-      await expect(con.createTable(tableName, newData)).to.be.rejectedWith(Error, 'already exists')
-
-      const table = await con.createTable(tableName, newData, WriteMode.Overwrite)
-      assert.equal(table.name, tableName)
-      assert.equal(await table.countRows(), 3)
+      const results = await table.search([0.1, 0.3]).execute()
+      assert.equal(results.length, 2)
     })
 
     it('appends records to an existing table ', async function () {
@@ -169,14 +102,16 @@ describe('LanceDB client', function () {
       ]
 
       const table = await con.createTable('vectors', data)
-      assert.equal(await table.countRows(), 2)
+      const results = await table.search([0.1, 0.3]).execute()
+      assert.equal(results.length, 2)
 
       const dataAdd = [
         { id: 3, vector: [2.1, 2.2], price: 10, name: 'c' },
         { id: 4, vector: [3.1, 3.2], price: 50, name: 'd' }
       ]
       await table.add(dataAdd)
-      assert.equal(await table.countRows(), 4)
+      const resultsAdd = await table.search([0.1, 0.3]).execute()
+      assert.equal(resultsAdd.length, 4)
     })
 
     it('overwrite all records in a table', async function () {
@@ -184,25 +119,16 @@ describe('LanceDB client', function () {
       const con = await lancedb.connect(uri)
 
       const table = await con.openTable('vectors')
-      assert.equal(await table.countRows(), 2)
+      const results = await table.search([0.1, 0.3]).execute()
+      assert.equal(results.length, 2)
 
       const dataOver = [
         { vector: [2.1, 2.2], price: 10, name: 'foo' },
         { vector: [3.1, 3.2], price: 50, name: 'bar' }
       ]
       await table.overwrite(dataOver)
-      assert.equal(await table.countRows(), 2)
-    })
-
-    it('can delete records from a table', async function () {
-      const uri = await createTestDB()
-      const con = await lancedb.connect(uri)
-
-      const table = await con.openTable('vectors')
-      assert.equal(await table.countRows(), 2)
-
-      await table.delete('price = 10')
-      assert.equal(await table.countRows(), 1)
+      const resultsAdd = await table.search([0.1, 0.3]).execute()
+      assert.equal(resultsAdd.length, 2)
     })
   })
 
@@ -211,58 +137,8 @@ describe('LanceDB client', function () {
       const uri = await createTestDB(32, 300)
       const con = await lancedb.connect(uri)
       const table = await con.openTable('vectors')
-      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+      await table.create_index({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2 })
     }).timeout(10_000) // Timeout is high partially because GH macos runner is pretty slow
-
-    it('replace an existing index', async function () {
-      const uri = await createTestDB(16, 300)
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-
-      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
-
-      // Replace should fail if the index already exists
-      await expect(table.createIndex({
-        type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2, replace: false
-      })
-      ).to.be.rejectedWith('LanceError(Index)')
-
-      // Default replace = true
-      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
-    }).timeout(50_000)
-  })
-
-  describe('when using a custom embedding function', function () {
-    class TextEmbedding implements EmbeddingFunction<string> {
-      sourceColumn: string
-
-      constructor (targetColumn: string) {
-        this.sourceColumn = targetColumn
-      }
-
-      _embedding_map = new Map<string, number[]>([
-        ['foo', [2.1, 2.2]],
-        ['bar', [3.1, 3.2]]
-      ])
-
-      async embed (data: string[]): Promise<number[][]> {
-        return data.map(datum => this._embedding_map.get(datum) ?? [0.0, 0.0])
-      }
-    }
-
-    it('should encode the original data into embeddings', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-      const embeddings = new TextEmbedding('name')
-
-      const data = [
-        { price: 10, name: 'foo' },
-        { price: 50, name: 'bar' }
-      ]
-      const table = await con.createTable('vectors', data, WriteMode.Create, embeddings)
-      const results = await table.search('foo').execute()
-      assert.equal(results.length, 2)
-    })
   })
 })
 
@@ -272,13 +148,11 @@ describe('Query object', function () {
       .limit(1)
       .metricType(MetricType.Cosine)
       .refineFactor(100)
-      .select(['a', 'b'])
       .nprobes(20) as Record<string, any>
     assert.equal(query._limit, 1)
     assert.equal(query._metricType, MetricType.Cosine)
     assert.equal(query._refineFactor, 100)
     assert.equal(query._nprobes, 20)
-    assert.deepEqual(query._select, ['a', 'b'])
   })
 })
 
@@ -298,22 +172,3 @@ async function createTestDB (numDimensions: number = 2, numRows: number = 2): Pr
   await con.createTable('vectors', data)
   return dir
 }
-
-describe('Drop table', function () {
-  it('drop a table', async function () {
-    const dir = await track().mkdir('lancejs')
-    const con = await lancedb.connect(dir)
-
-    const data = [
-      { price: 10, name: 'foo', vector: [1, 2, 3] },
-      { price: 50, name: 'bar', vector: [4, 5, 6] }
-    ]
-    await con.createTable('t1', data)
-    await con.createTable('t2', data)
-
-    assert.deepEqual(await con.tableNames(), ['t1', 't2'])
-
-    await con.dropTable('t1')
-    assert.deepEqual(await con.tableNames(), ['t2'])
-  })
-})

notebooks/code_qa_bot.ipynb

@@ -72,8 +72,6 @@
     "import lancedb\n",
     "import re\n",
     "import pickle\n",
-    "import requests\n",
-    "import zipfile\n",
     "from pathlib import Path\n",
     "\n",
     "from langchain.document_loaders import UnstructuredHTMLLoader\n",
@@ -87,25 +85,10 @@
   {
    "attachments": {},
    "cell_type": "markdown",
-   "id": "56cc6d50",
+   "id": "6ccf9b2b",
    "metadata": {},
    "source": [
-    "To make this easier, we've downloaded Pandas documentation and stored the raw HTML files for you to download. We'll download them and then use LangChain's HTML document readers to parse them and store them in LanceDB as a vector store, along with relevant metadata."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7da77e75",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "pandas_docs = requests.get(\"https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip\")\n",
-    "with open('/tmp/pandas.documentation.zip', 'wb') as f:\n",
-    "    f.write(pandas_docs.content)\n",
-    "\n",
-    "file = zipfile.ZipFile(\"/tmp/pandas.documentation.zip\")\n",
-    "file.extractall(path=\"/tmp/pandas_docs\")"
+    "You can download the Pandas documentation from https://pandas.pydata.org/docs/. To make sure we're not littering our repo with docs, we won't include it in the LanceDB repo, so download this and store it locally first."
    ]
   },
   {
@@ -154,8 +137,7 @@
     "docs = []\n",
     "\n",
     "if not docs_path.exists():\n",
-    "    for p in Path(\"/tmp/pandas_docs/pandas.documentation\").rglob(\"*.html\"):\n",
-    "        print(p)\n",
+    "    for p in Path(\"./pandas.documentation\").rglob(\"*.html\"):\n",
     "        if p.is_dir():\n",
     "            continue\n",
     "        loader = UnstructuredHTMLLoader(p)\n",

notebooks/youtube_transcript_search.ipynb

@@ -1,12 +1,11 @@
 {
  "cells": [
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "42bf01fb",
    "metadata": {},
    "source": [
-    "# Youtube Transcript Search QA Bot\n",
+    "# We're going to build question and answer bot\n",
     "\n",
     "This Q&A bot will allow you to search through youtube transcripts using natural language! By going through this notebook, we'll introduce how you can use LanceDB to store and manage your data easily."
    ]
@@ -36,7 +35,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "22e570f4",
    "metadata": {},
@@ -89,7 +87,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "5ac2b6a3",
    "metadata": {},
@@ -184,7 +181,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "3044e0b0",
    "metadata": {},
@@ -213,7 +209,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "db586267",
    "metadata": {},
@@ -234,7 +229,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "2106b5bb",
    "metadata": {},
@@ -344,7 +338,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "53e4bff1",
    "metadata": {},
@@ -378,7 +371,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "8ef34fca",
    "metadata": {},
@@ -467,7 +459,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "23afc2f9",
    "metadata": {},
@@ -550,7 +541,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "28705959",
    "metadata": {},
@@ -581,7 +571,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "559a095b",
    "metadata": {},

python/.bumpversion.cfg

@@ -1,8 +0,0 @@
-[bumpversion]
-current_version = 0.1.8
-commit = True
-message = [python] Bump version: {current_version} → {new_version}
-tag = True
-tag_name = python-v{new_version}
-
-[bumpversion:file:pyproject.toml]

python/README.md

@@ -1,85 +0,0 @@
-# LanceDB
-
-A Python library for [LanceDB](https://github.com/lancedb/lancedb).
-
-## Installation
-
-```bash
-pip install lancedb
-```
-
-## Usage
-
-### Basic Example
-
-```python
-import lancedb
-db = lancedb.connect('<PATH_TO_LANCEDB_DATASET>')
-table = db.open_table('my_table')
-results = table.search([0.1, 0.3]).limit(20).to_df()
-print(results)
-```
-
-
-## Development
-
-Create a virtual environment and activate it:
-
-```bash
-python -m venv venv
-. ./venv/bin/activate
-```
-
-Install the necessary packages:
-
-```bash
-python -m pip install .
-```
-
-To run the unit tests:
-
-```bash
-pytest
-```
-
-To run linter and automatically fix all errors:
-
-```bash
-black .
-isort .
-```
-
-If any packages are missing, install them with:
-
-```bash
-pip install <PACKAGE_NAME>
-```
-
-
-___
-For **Windows** users, there may be errors when installing packages, so these commands may be helpful:
-
-Activate the virtual environment:
-```bash
-. .\venv\Scripts\activate
-```
-
-You may need to run the installs separately:
-```bash
-pip install -e .[tests]
-pip install -e .[dev]
-```
-
-
-`tantivy` requires `rust` to be installed, so install it with `conda`, as it doesn't support windows installation:
-```bash
-pip install wheel
-pip install cargo
-conda install rust
-pip install tantivy
-```
-
-To run the unit tests:
-```bash
-pytest
-```

python/lancedb/__init__.py

@@ -11,49 +11,19 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from typing import Optional
-
-from .db import URI, DBConnection, LanceDBConnection
-from .remote.db import RemoteDBConnection
-from .schema import vector
+from .db import URI, LanceDBConnection
 
 
-def connect(
-    uri: URI, *, api_key: Optional[str] = None, region: str = "us-west-2"
-) -> DBConnection:
-    """Connect to a LanceDB database.
+def connect(uri: URI) -> LanceDBConnection:
+    """Connect to a LanceDB instance at the given URI
 
     Parameters
     ----------
     uri: str or Path
         The uri of the database.
-    api_token: str, optional
-        If presented, connect to LanceDB cloud.
-        Otherwise, connect to a database on file system or cloud storage.
-
-    Examples
-    --------
-
-    For a local directory, provide a path for the database:
-
-    >>> import lancedb
-    >>> db = lancedb.connect("~/.lancedb")
-
-    For object storage, use a URI prefix:
-
-    >>> db = lancedb.connect("s3://my-bucket/lancedb")
-
-    Connect to LancdDB cloud:
-
-    >>> db = lancedb.connect("db://my_database", api_key="ldb_...")
 
     Returns
     -------
-    conn : DBConnection
-        A connection to a LanceDB database.
+    A connection to a LanceDB database.
     """
-    if isinstance(uri, str) and uri.startswith("db://"):
-        if api_key is None:
-            raise ValueError(f"api_key is required to connected LanceDB cloud: {uri}")
-        return RemoteDBConnection(uri, api_key, region)
     return LanceDBConnection(uri)

python/lancedb/common.py

@@ -23,13 +23,3 @@ URI = Union[str, Path]
 # TODO support generator
 DATA = Union[List[dict], dict, pd.DataFrame]
 VECTOR_COLUMN_NAME = "vector"
-
-
-class Credential(str):
-    """Credential field"""
-
-    def __repr__(self) -> str:
-        return "********"
-
-    def __str__(self) -> str:
-        return "********"