Bump to 0.1.11

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.0
+current_version = 0.1.13
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True

.github/workflows/make-release-commit.yml

@@ -52,8 +52,4 @@ jobs:
         github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
         branch: main
         tags: true
-      - uses: ./.github/workflows/update_package_lock
-        if: ${{ inputs.dry_run }} == "false"
-        with:
-          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
 

.github/workflows/node.yml

@@ -9,7 +9,6 @@ on:
       - node/**
       - rust/ffi/node/**
       - .github/workflows/node.yml
-      - docker-compose.yml
 
 env:
   # Disable full debug symbol generation to speed up CI build and keep memory down
@@ -71,7 +70,7 @@ jobs:
         npm run tsc
         npm run build
         npm run pack-build
-        npm install --no-save ./dist/lancedb-vectordb-*.tgz
+        npm install --no-save ./dist/vectordb-*.tgz
         # Remove index.node to test with dependency installed
         rm index.node
     - name: Test
@@ -102,62 +101,9 @@ jobs:
         npm run tsc
         npm run build
         npm run pack-build
-        npm install --no-save ./dist/lancedb-vectordb-*.tgz
+        npm install --no-save ./dist/vectordb-*.tgz
         # Remove index.node to test with dependency installed
         rm index.node
     - name: Test
       run: |
         npm run test
-  aws-integtest:
-    timeout-minutes: 45
-    runs-on: "ubuntu-22.04"
-    defaults:
-      run:
-        shell: bash
-        working-directory: node
-    env:
-      AWS_ACCESS_KEY_ID: ACCESSKEY
-      AWS_SECRET_ACCESS_KEY: SECRETKEY
-      AWS_DEFAULT_REGION: us-west-2
-      # this one is for s3
-      AWS_ENDPOINT: http://localhost:4566
-      # this one is for dynamodb
-      DYNAMODB_ENDPOINT: http://localhost:4566
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        lfs: true
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 18
-        cache: 'npm'
-        cache-dependency-path: node/package-lock.json
-    - name: start local stack
-      run: docker compose -f ../docker-compose.yml up -d --wait
-    - name: create s3
-      run: aws s3 mb s3://lancedb-integtest --endpoint $AWS_ENDPOINT
-    - name: create ddb
-      run: |
-        aws dynamodb create-table \
-          --table-name lancedb-integtest \
-          --attribute-definitions '[{"AttributeName": "base_uri", "AttributeType": "S"}, {"AttributeName": "version", "AttributeType": "N"}]' \
-          --key-schema '[{"AttributeName": "base_uri", "KeyType": "HASH"}, {"AttributeName": "version", "KeyType": "RANGE"}]' \
-          --provisioned-throughput '{"ReadCapacityUnits": 10, "WriteCapacityUnits": 10}' \
-          --endpoint-url $DYNAMODB_ENDPOINT
-    - uses: Swatinem/rust-cache@v2
-    - name: Install dependencies
-      run: |
-        sudo apt update
-        sudo apt install -y protobuf-compiler libssl-dev
-    - name: Build
-      run: |
-        npm ci
-        npm run tsc
-        npm run build
-        npm run pack-build
-        npm install --no-save ./dist/lancedb-vectordb-*.tgz
-        # Remove index.node to test with dependency installed
-        rm index.node
-    - name: Test
-      run: npm run integration-test

.github/workflows/npm-publish.yml

@@ -62,35 +62,59 @@ jobs:
     - name: Upload Darwin Artifacts
       uses: actions/upload-artifact@v3
       with:
-          name: native-darwin
+        name: darwin-native
         path: |
-            node/dist/lancedb-vectordb-darwin*.tgz
+          node/dist/vectordb-darwin*.tgz
 
   node-linux:
-    name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu
-    runs-on: ${{ matrix.config.runner }}
+    name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }})
+    runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
     strategy:
       fail-fast: false
       matrix:
-        config:
-          - arch: x86_64
-            runner: ubuntu-latest
-          - arch: aarch64
-            runner: buildjet-4vcpu-ubuntu-2204-arm
+        libc:
+          - gnu
+          # TODO: re-enable musl once we have refactored to pre-built containers
+          # Right now we have to build node from source which is too expensive.
+          # - musl
+        arch:
+          - x86_64
+          # Building on aarch64 is too slow for now
+          # - aarch64
     steps:
     - name: Checkout
       uses: actions/checkout@v3
-      - name: Build Linux Artifacts
+    - name: Change owner to root (for npm)
+      # The docker container is run as root, so we need the files to be owned by root
+      # Otherwise npm is a nightmare: https://github.com/npm/cli/issues/3773
+      run: sudo chown -R root:root .
+    - name: Set up QEMU
+      if: ${{ matrix.arch == 'aarch64' }}
+      uses: docker/setup-qemu-action@v2
+      with:
+        platforms: arm64
+    - name: Build Linux GNU native node modules
+      if: ${{ matrix.libc == 'gnu' }}
       run: |
-          bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}
+        docker run \
+          -v $(pwd):/io -w /io \
+          rust:1.70-bookworm \
+          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu
+    - name: Build musl Linux native node modules
+      if: ${{ matrix.libc == 'musl' }}
+      run: |
+        docker run --platform linux/arm64/v8 \
+          -v $(pwd):/io -w /io \
+          quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
+          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
     - name: Upload Linux Artifacts
       uses: actions/upload-artifact@v3
       with:
-          name: native-linux
+        name: linux-native
         path: |
-            node/dist/lancedb-vectordb-linux*.tgz
+          node/dist/vectordb-linux*.tgz
 
   node-windows:
     runs-on: windows-2022
@@ -121,12 +145,12 @@ jobs:
       - name: Upload Windows Artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: native-windows
+          name: windows-native
           path: |
-            node/dist/lancedb-vectordb-win32*.tgz
+            node/dist/vectordb-win32*.tgz
 
   release:
-    needs: [node, node-macos, node-linux, node-windows]
+    needs: [node, node-macos, node-linux]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -137,7 +161,6 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: 20
-          registry-url: 'https://registry.npmjs.org'
       - name: Publish to NPM
         env:
           NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
@@ -146,18 +169,3 @@ jobs:
           for filename in *.tgz; do
             npm publish $filename
           done
-
-  update-package-lock:
-    needs: [release]
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          ref: main
-          persist-credentials: false
-          fetch-depth: 0
-          lfs: true
-      - uses: ./.github/workflows/update_package_lock
-        with:
-          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.github/workflows/python.yml

@@ -30,7 +30,7 @@ jobs:
         python-version: 3.${{ matrix.python-minor-version }}
     - name: Install lancedb
       run: |
-        pip install -e .[tests]
+        pip install -e .
         pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
         pip install pytest pytest-mock black isort
     - name: Black
@@ -38,7 +38,7 @@ jobs:
     - name: isort
       run: isort --check --diff --quiet .
     - name: Run tests
-      run: pytest -m "not slow" -x -v --durations=30 tests
+      run: pytest -x -v --durations=30 tests
     - name: doctest
       run: pytest --doctest-modules lancedb
   mac:
@@ -59,40 +59,10 @@ jobs:
         python-version: "3.11"
     - name: Install lancedb
       run: |
-        pip install -e .[tests]
+        pip install -e .
         pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
         pip install pytest pytest-mock black
     - name: Black
       run: black --check --diff --no-color --quiet .
     - name: Run tests
-      run: pytest -m "not slow" -x -v --durations=30 tests
-  pydantic1x:
-    timeout-minutes: 30
-    runs-on: "ubuntu-22.04"
-    defaults:
-      run:
-        shell: bash
-        working-directory: python
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.9
-    - name: Install lancedb
-      run: |
-        pip install "pydantic<2"
-        pip install -e .[tests]
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock black isort
-    - name: Black
-      run: black --check --diff --no-color --quiet .
-    - name: isort
-      run: isort --check --diff --quiet .
-    - name: Run tests
-      run: pytest -m "not slow" -x -v --durations=30 tests
-    - name: doctest
-      run: pytest --doctest-modules lancedb
+      run: pytest -x -v --durations=30 tests

.github/workflows/trigger-vectordb-recipes.yml

@@ -1,26 +0,0 @@
-name: Trigger vectordb-recipers workflow
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    paths:
-      - .github/workflows/trigger-vectordb-recipes.yml
-  workflow_dispatch:
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Trigger vectordb-recipes workflow
-        uses: actions/github-script@v6
-        with:
-          github-token: ${{ secrets.VECTORDB_RECIPES_ACTION_TOKEN }}
-          script: |
-            const result = await github.rest.actions.createWorkflowDispatch({
-                owner: 'lancedb',
-                repo: 'vectordb-recipes',
-                workflow_id: 'examples-test.yml',
-                ref: 'main'
-            });
-            console.log(result);

.github/workflows/update_package_lock/action.yml

@@ -1,33 +0,0 @@
-name: update_package_lock
-description: "Update node's package.lock"
-
-inputs:
-  github_token:
-    required: true
-    description: "github token for the repo"
-
-runs:
-  using: "composite"
-  steps:
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 20
-    - name: Set git configs
-      shell: bash
-      run: |
-        git config user.name 'Lance Release'
-        git config user.email 'lance-dev@lancedb.com'
-    - name: Update package-lock.json file
-      working-directory: ./node
-      run: |
-        npm install
-        git add package-lock.json
-        git commit -m "Updating package-lock.json"
-      shell: bash
-    - name: Push changes
-      if: ${{ inputs.dry_run }} == "false"
-      uses: ad-m/github-push-action@master
-      with:
-        github_token: ${{ inputs.github_token }}
-        branch: main
-        tags: true

.github/workflows/update_package_lock_run.yml

@@ -1,19 +0,0 @@
-name: Update package-lock.json
-
-on:
-  workflow_dispatch:
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          ref: main
-          persist-credentials: false
-          fetch-depth: 0
-          lfs: true
-      - uses: ./.github/workflows/update_package_lock
-        with:
-          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

Cargo.toml

@@ -1,27 +1,16 @@
 [workspace]
-members = ["rust/ffi/node", "rust/vectordb"]
-# Python package needs to be built by maturin.
-exclude = ["python"]
+members = [
+    "rust/vectordb",
+    "rust/ffi/node"
+]
 resolver = "2"
 
 [workspace.dependencies]
-lance = { "version" = "=0.8.3", "features" = ["dynamodb"] }
-lance-linalg = { "version" = "=0.8.3" }
-lance-testing = { "version" = "=0.8.3" }
-# Note that this one does not include pyarrow
-arrow = { version = "43.0.0", optional = false }
-arrow-array = "43.0"
-arrow-data = "43.0"
-arrow-ipc = "43.0"
-arrow-ord = "43.0"
-arrow-schema = "43.0"
-arrow-arith = "43.0"
-arrow-cast = "43.0"
-chrono = "0.4.23"
-half = { "version" = "=2.2.1", default-features = false, features = [
-    "num-traits"
-] }
-log = "0.4"
+lance = "=0.5.8"
+arrow-array = "42.0"
+arrow-data = "42.0"
+arrow-schema = "42.0"
+arrow-ipc = "42.0"
+half = { "version" = "2.2.1", default-features = false }
 object_store = "0.6.1"
-snafu = "0.7.4"
-url = "2"
+

README.md

@@ -33,8 +33,6 @@ The key features of LanceDB include:
 
 * Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure.
 
-* GPU support in building vector index(*).
-
 * Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lanecdb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
 
 LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
@@ -54,7 +52,8 @@ const table = await db.createTable('vectors',
       [{ id: 1, vector: [0.1, 0.2], item: "foo", price: 10 },
        { id: 2, vector: [1.1, 1.2], item: "bar", price: 50 }])
 
-const query = table.search([0.1, 0.3]).limit(2);
+const query = table.search([0.1, 0.3]);
+query.limit = 20;
 const results = await query.execute();
 ```
 
@@ -71,7 +70,7 @@ db = lancedb.connect(uri)
 table = db.create_table("my_table",
                          data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
                                {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-result = table.search([100, 100]).limit(2).to_pandas()
+result = table.search([100, 100]).limit(2).to_df()
 ```
 
 ## Blogs, Tutorials & Videos

ci/build_linux_artifacts.sh

@@ -1,19 +1,72 @@
 #!/bin/bash
+# Builds the Linux artifacts (node binaries).
+# Usage: ./build_linux_artifacts.sh [target]
+# Targets supported: 
+# - x86_64-unknown-linux-gnu:centos
+# - aarch64-unknown-linux-gnu:centos
+# - aarch64-unknown-linux-musl
+# - x86_64-unknown-linux-musl
+
+# TODO: refactor this into a Docker container we can pull
+
 set -e
-ARCH=${1:-x86_64}
 
-# We pass down the current user so that when we later mount the local files 
-# into the container, the files are accessible by the current user.
-pushd ci/manylinux_node
-docker build \
-    -t lancedb-node-manylinux \
-    --build-arg="ARCH=$ARCH" \
-    --build-arg="DOCKER_USER=$(id -u)" \
-    --progress=plain \
-    .
-popd
+setup_dependencies() {
+    echo "Installing system dependencies..."
+    if [[ $1 == *musl ]]; then
+        # musllinux
+        apk add openssl-dev
+    else
+        # rust / debian
+        apt update
+        apt install -y libssl-dev protobuf-compiler
+    fi
+}
 
-docker run \
-    -v $(pwd):/io -w /io \
-    lancedb-node-manylinux \
-    bash ci/manylinux_node/build.sh $ARCH
+install_node() {
+    echo "Installing node..."
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+    source "$HOME"/.bashrc
+
+    if [[ $1 == *musl ]]; then
+        # This node version is 15, we need 16 or higher:
+        # apk add nodejs-current npm 
+        # So instead we install from source (nvm doesn't provide binaries for musl):
+        nvm install -s --no-progress 17
+    else
+        nvm install --no-progress 17 # latest that supports glibc 2.17
+    fi
+}
+
+build_node_binary() {
+    echo "Building node library for $1..."
+    pushd node
+
+    npm ci
+    
+    if [[ $1 == *musl ]]; then
+        # This is needed for cargo to allow build cdylibs with musl
+        export RUSTFLAGS="-C target-feature=-crt-static"
+    fi
+
+    # Cargo can run out of memory while pulling dependencies, especially when running
+    # in QEMU. This is a workaround for that.
+    export CARGO_NET_GIT_FETCH_WITH_CLI=true
+
+    # We don't pass in target, since the native target here already matches
+    # We need to pass OPENSSL_LIB_DIR and OPENSSL_INCLUDE_DIR for static build to work https://github.com/sfackler/rust-openssl/issues/877
+    OPENSSL_STATIC=1 OPENSSL_LIB_DIR=/usr/lib/x86_64-linux-gnu OPENSSL_INCLUDE_DIR=/usr/include/openssl/ npm run build-release
+    npm run pack-build
+
+    popd
+}
+
+TARGET=${1:-x86_64-unknown-linux-gnu}
+# Others:
+# aarch64-unknown-linux-gnu
+# x86_64-unknown-linux-musl
+# aarch64-unknown-linux-musl
+
+setup_dependencies $TARGET
+install_node $TARGET
+build_node_binary $TARGET

ci/manylinux_node/Dockerfile

@@ -1,31 +0,0 @@
-# Many linux dockerfile with Rust, Node, and Lance dependencies installed.
-# This container allows building the node modules native libraries in an 
-# environment with a very old glibc, so that we are compatible with a wide
-# range of linux distributions.
-ARG ARCH=x86_64
-
-FROM quay.io/pypa/manylinux2014_${ARCH}
-
-ARG ARCH=x86_64
-ARG DOCKER_USER=default_user
-
-# Install static openssl
-COPY install_openssl.sh install_openssl.sh
-RUN ./install_openssl.sh ${ARCH} > /dev/null
-
-# Protobuf is also installed as root.
-COPY install_protobuf.sh install_protobuf.sh
-RUN ./install_protobuf.sh ${ARCH}
-
-ENV DOCKER_USER=${DOCKER_USER}
-# Create a group and user
-RUN echo ${ARCH} && adduser --user-group --create-home --uid ${DOCKER_USER} build_user
-
-# We switch to the user to install Rust and Node, since those like to be 
-# installed at the user level.
-USER ${DOCKER_USER}
-
-COPY prepare_manylinux_node.sh prepare_manylinux_node.sh
-RUN cp /prepare_manylinux_node.sh $HOME/ && \
-    cd $HOME && \
-    ./prepare_manylinux_node.sh ${ARCH}

ci/manylinux_node/build.sh

@@ -1,19 +0,0 @@
-#!/bin/bash
-# Builds the node module for manylinux. Invoked by ci/build_linux_artifacts.sh.
-set -e
-ARCH=${1:-x86_64}
-
-if [ "$ARCH" = "x86_64" ]; then
-    export OPENSSL_LIB_DIR=/usr/local/lib64/
-else 
-    export OPENSSL_LIB_DIR=/usr/local/lib/
-fi
-export OPENSSL_STATIC=1
-export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
-
-source $HOME/.bashrc
-
-cd node
-npm ci
-npm run build-release
-npm run pack-build

ci/manylinux_node/install_openssl.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-# Builds openssl from source so we can statically link to it
-
-# this is to avoid the error we get with the system installation:
-# /usr/bin/ld: <library>: version node not found for symbol SSLeay@@OPENSSL_1.0.1
-# /usr/bin/ld: failed to set dynamic section sizes: Bad value
-set -e
-
-git clone -b OpenSSL_1_1_1u \
-    --single-branch \
-    https://github.com/openssl/openssl.git
-
-pushd openssl
-
-if [[ $1 == x86_64* ]]; then
-    ARCH=linux-x86_64
-else
-    # gnu target
-    ARCH=linux-aarch64
-fi
-
-./Configure no-shared $ARCH
-
-make
-
-make install

ci/manylinux_node/install_protobuf.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-# Installs protobuf compiler. Should be run as root.
-set -e
-
-if [[ $1 == x86_64* ]]; then
-    ARCH=x86_64
-else
-    # gnu target
-    ARCH=aarch_64
-fi
-
-PB_REL=https://github.com/protocolbuffers/protobuf/releases
-PB_VERSION=23.1
-curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
-unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local

ci/manylinux_node/prepare_manylinux_node.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-set -e
-
-install_node() {
-    echo "Installing node..."
-
-    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
-
-    source "$HOME"/.bashrc
-
-    nvm install --no-progress 16
-}
-
-install_rust() {
-    echo "Installing rust..."
-    curl https://sh.rustup.rs -sSf | bash -s -- -y
-    export PATH="$PATH:/root/.cargo/bin"
-}
-
-install_node
-install_rust

docker-compose.yml

@@ -1,18 +0,0 @@
-version: "3.9"
-services:
-  localstack:
-    image: localstack/localstack:0.14
-    ports:
-      - 4566:4566
-    environment:
-      - SERVICES=s3,dynamodb
-      - DEBUG=1
-      - LS_LOG=trace
-      - DOCKER_HOST=unix:///var/run/docker.sock
-      - AWS_ACCESS_KEY_ID=ACCESSKEY
-      - AWS_SECRET_ACCESS_KEY=SECRETKEY
-    healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:4566/health" ]
-      interval: 5s
-      retries: 3
-      start_period: 10s

docs/mkdocs.yml

@@ -1,6 +1,5 @@
 site_name: LanceDB Docs
 repo_url: https://github.com/lancedb/lancedb
-edit_uri: https://github.com/lancedb/lancedb/tree/main/docs/src
 repo_name: lancedb/lancedb
 docs_dir: src
 
@@ -11,17 +10,6 @@ theme:
   features:
     - content.code.copy
     - content.tabs.link
-    - content.action.edit
-    - toc.follow
-    - toc.integrate
-    - navigation.top
-    - navigation.tabs
-    - navigation.tabs.sticky
-    - navigation.footer
-    - navigation.tracking
-    - navigation.instant
-    - navigation.indexes
-    - navigation.expand
   icon:
     repo: fontawesome/brands/github
   custom_dir: overrides
@@ -65,79 +53,32 @@ markdown_extensions:
 - md_in_html
 
 nav:
-- Home:
-  - 🏢 Home: index.md
-  - 💡 Basics: basic.md
-  - 📚 Guides:
-    - Create Ingest Update Delete: guides/tables.md
-    - Vector Search: search.md
-    - SQL filters: sql.md
-    - Indexing: ann_indexes.md
-  - 🧬 Embeddings: embedding.md
-  - 🔍 Python full-text search: fts.md
-  - 🔌 Integrations:
-    - integrations/index.md
-    - Pandas and PyArrow: python/arrow.md
-    - DuckDB: python/duckdb.md
-    - LangChain 🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
-    - LangChain JS/TS 🔗: https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb
-    - LlamaIndex 🦙: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
-    - Pydantic: python/pydantic.md
-    - Voxel51: integrations/voxel51.md
-    - PromptTools: integrations/prompttools.md
-  - 🐍 Python examples:
-    - examples/index.md
-    - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
-    - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
-    - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
-    - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
-    - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
-  - 🌐 Javascript examples:
-    - Examples: examples/index_js.md
-    - Serverless Website Chatbot: examples/serverless_website_chatbot.md
-    - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
-    - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
-  - ⚙️ CLI & Config: cli_config.md
-
+- Home: index.md
 - Basics: basic.md
-- Guides:
-  - Create Ingest Update Delete: guides/tables.md
-  - Vector Search: search.md
-  - SQL filters: sql.md
-  - Indexing: ann_indexes.md
 - Embeddings: embedding.md
 - Python full-text search: fts.md
-- Integrations:
-  - integrations/index.md
+- Python integrations:
   - Pandas and PyArrow: python/arrow.md
   - DuckDB: python/duckdb.md
   - LangChain 🦜️🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
-  - LangChain JS/TS 🦜️🔗: https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb
   - LlamaIndex 🦙: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
   - Pydantic: python/pydantic.md
-  - Voxel51: integrations/voxel51.md
-  - PromptTools: integrations/prompttools.md
 - Python examples:
-  - examples/index.md
   - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
   - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
   - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
   - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
   - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
 - Javascript examples:
-  - examples/index_js.md
   - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
-  - Serverless Chatbot from any website: examples/serverless_website_chatbot.md
   - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
+- References:
+  - Vector Search: search.md
+  - SQL filters: sql.md
+  - Indexing: ann_indexes.md
 - API references:
   - Python API: python/python.md
   - Javascript API: javascript/modules.md
-- LanceDB Cloud↗: https://noteforms.com/forms/lancedb-mailing-list-cloud-kty1o5?notionforms=1&utm_source=notionforms
 
 extra_css:
   - styles/global.css
-
-extra:
-  analytics:
-    provider: google
-    property: G-B7NFM40W74

docs/src/ann_indexes.md

@@ -6,7 +6,7 @@ LanceDB provides many parameters to fine-tune the index's size, the speed of que
 
 Currently, LanceDB does *not* automatically create the ANN index.
 LanceDB has optimized code for KNN as well. For many use-cases, datasets under 100K vectors won't require index creation at all.
-If you can live with < 100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
+If you can live with <100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
 
 In the future we will look to automatically create and configure the ANN index.
 
@@ -68,12 +68,6 @@ a single PQ code.
   <figcaption>IVF_PQ index with <code>num_partitions=2, num_sub_vectors=4</code></figcaption>
 </figure>
 
-### Use GPU to build vector index
-
-Lance Python SDK has experimental GPU support for creating IVF index.
-You can specify the GPU device to train IVF partitions via
-
-- **accelerator**: Specify to `"cuda"`` to enable GPU training.
 
 ## Querying an ANN Index
 
@@ -97,10 +91,10 @@ There are a couple of parameters that can be used to fine-tune the search:
          .limit(2) \
          .nprobes(20) \
          .refine_factor(10) \
-         .to_pandas()
+         .to_df()
      ```
      ```
-                                              vector       item       _distance
+                                              vector       item       score
      0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
      1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
      ```
@@ -115,8 +109,9 @@ There are a couple of parameters that can be used to fine-tune the search:
          .execute()
      ```
 
-The search will return the data requested in addition to the distance of each item.
+The search will return the data requested in addition to the score of each item.
 
+**Note:** The score is the distance between the query vector and the element. A lower number means that the result is more relevant.
 
 ### Filtering (where clause)
 
@@ -124,7 +119,7 @@ You can further filter the elements returned by a search using a where clause.
 
 === "Python"
      ```python
-     tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
+     tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_df()
      ```
 
 === "Javascript"
@@ -141,10 +136,10 @@ You can select the columns returned by the query using a select clause.
 
 === "Python"
      ```python
-     tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
+     tbl.search(np.random.random((1536))).select(["vector"]).to_df()
      ```
      ```
-        vector                                             _distance
+     vector      score
      0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
      1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
      ...
@@ -160,28 +155,28 @@ You can select the columns returned by the query using a select clause.
 
 ## FAQ
 
-### When is it necessary to create an ANN vector index?
+### When is it necessary to create an ANN vector index.
 
-`LanceDB` has manually-tuned SIMD code for computing vector distances.
-In our benchmarks, computing 100K pairs of 1K dimension vectors takes **less than 20ms**.
-For small datasets (< 100K rows) or applications that can accept 100ms latency, vector indices are usually not necessary.
+`LanceDB` has manually tuned SIMD code for computing vector distances.
+In our benchmarks, computing 100K pairs of 1K dimension vectors only take less than 20ms.
+For small dataset (<100K rows) or the applications which can accept 100ms latency, vector indices are usually not necessary.
 
 For large-scale or higher dimension vectors, it is beneficial to create vector index.
 
-### How big is my index, and how many memory will it take?
+### How big is my index, and how many memory will it take.
 
-In LanceDB, all vector indices are **disk-based**, meaning that when responding to a vector query, only the relevant pages from the index file are loaded from disk and cached in memory. Additionally, each sub-vector is usually encoded into 1 byte PQ code.
+In LanceDB, all vector indices are disk-based, meaning that when responding to a vector query, only the relevant pages from the index file are loaded from disk and cached in memory. Additionally, each sub-vector is usually encoded into 1 byte PQ code.
 
 For example, with a 1024-dimension dataset, if we choose `num_sub_vectors=64`, each sub-vector has `1024 / 64 = 16` float32 numbers.
 Product quantization can lead to approximately `16 * sizeof(float32) / 1 = 64` times of space reduction.
 
-### How to choose `num_partitions` and `num_sub_vectors` for `IVF_PQ` index?
+### How to choose `num_partitions` and `num_sub_vectors` for `IVF_PQ` index.
 
 `num_partitions` is used to decide how many partitions the first level `IVF` index uses.
 Higher number of partitions could lead to more efficient I/O during queries and better accuracy, but it takes much more time to train.
 On `SIFT-1M` dataset, our benchmark shows that keeping each partition 1K-4K rows lead to a good latency / recall.
 
-`num_sub_vectors` specifies how many Product Quantization (PQ) short codes to generate on each vector. Because
-PQ is a lossy compression of the original vector, a higher `num_sub_vectors` usually results in
-less space distortion, and thus yields better accuracy. However, a higher `num_sub_vectors` also causes heavier I/O and
-more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.
+`num_sub_vectors` decides how many Product Quantization code to generate on each vector. Because
+Product Quantization is a lossy compression of the original vector, the more `num_sub_vectors` usually results to
+less space distortion, and thus yield better accuracy. However, similarly, more `num_sub_vectors` causes heavier I/O and
+more PQ computation, thus, higher latency. `dimension / num_sub_vectors` should be aligned with 8 for better SIMD efficiency.

docs/src/basic.md

@@ -79,18 +79,6 @@ We'll cover the basics of using LanceDB on your local machine in this section.
 
 ??? info "Under the hood, LanceDB is converting the input data into an Apache Arrow table and persisting it to disk in [Lance format](https://www.github.com/lancedb/lance)."
 
-### Creating an empty table
-
-Sometimes you may not have the data to insert into the table at creation time.
-In this case, you can create an empty table and specify the schema.
-
-=== "Python"
-      ```python
-      import pyarrow as pa
-      schema = pa.schema([pa.field("vector", pa.list_(pa.float32(), list_size=2))])
-      tbl = db.create_table("empty_table", schema=schema)
-      ```
-
 ## How to open an existing table
 
 Once created, you can open a table using the following code:
@@ -123,15 +111,9 @@ After a table has been created, you can always add more data to it using
 
 === "Python"
       ```python
-
-      # Option 1: Add a list of dicts to a table
-      data = [{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
-            {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}]
-      tbl.add(data)
-
-      # Option 2: Add a pandas DataFrame to a table
-      df = pd.DataFrame(data)
-      tbl.add(data)
+      df = pd.DataFrame([{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
+                        {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}])
+      tbl.add(df)
       ```
 
 === "Javascript"
@@ -146,7 +128,7 @@ Once you've embedded the query, you can find its nearest neighbors using the fol
 
 === "Python"
       ```python
-      tbl.search([100, 100]).limit(2).to_pandas()
+      tbl.search([100, 100]).limit(2).to_df()
       ```
 
       This returns a pandas DataFrame with the results.
@@ -156,63 +138,8 @@ Once you've embedded the query, you can find its nearest neighbors using the fol
       const query = await tbl.search([100, 100]).limit(2).execute();
       ```
 
-## How to delete rows from a table
-
-Use the `delete()` method on tables to delete rows from a table. To choose
-which rows to delete, provide a filter that matches on the metadata columns.
-This can delete any number of rows that match the filter.
-
-=== "Python"
-      ```python
-      tbl.delete('item = "fizz"')
-      ```
-
-=== "Javascript"
-      ```javascript
-      await tbl.delete('item = "fizz"')
-      ```
-
-The deletion predicate is a SQL expression that supports the same expressions
-as the `where()` clause on a search. They can be as simple or complex as needed.
-To see what expressions are supported, see the [SQL filters](sql.md) section.
-
-
-=== "Python"
-
-      Read more: [lancedb.table.Table.delete][]
-
-=== "Javascript"
-
-      Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
-
-## How to remove a table
-
-Use the `drop_table()` method on the database to remove a table.
-
-=== "Python"
-      ```python
-      db.drop_table("my_table")
-      ```
-
-This permanently removes the table and is not recoverable, unlike deleting rows.
-By default, if the table does not exist an exception is raised. To suppress this,
-you can pass in `ignore_missing=True`.
-
-
 ## What's next
 
 This section covered the very basics of the LanceDB API.
 LanceDB supports many additional features when creating indices to speed up search and options for search.
 These are contained in the next section of the documentation.
-
-## Note: Bundling vectorDB apps with webpack
-Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying on Vercel.
-```javascript
-/** @type {import('next').NextConfig} */
-module.exports = ({
-  webpack(config) {
-    config.externals.push({ vectordb: 'vectordb' })
-    return config;
-  }
-})
-```

docs/src/cli_config.md

@@ -1,37 +0,0 @@
-
-## LanceDB CLI
-Once lanceDB is installed, you can access the CLI using `lancedb` command on the console
-```
-lancedb
-```
-This lists out all the various command-line options available. You can get the usage or help for a particular command
-```
-lancedb {command} --help
-```
-
-## LanceDB config
-LanceDB uses a global config file to store certain settings. These settings are configurable using the lanceDB cli.
-To view your config settings, you can use:
-```
-lancedb config
-```
-These config parameters can be tuned using the cli.
-```
-lancedb {config_name} --{argument}
-```
-
-## LanceDB Opt-in Diagnostics
-When enabled, LanceDB will send anonymous events to help us improve LanceDB. These diagnostics are used only for error reporting and no data is collected. Error & stats allow us to automate certain aspects of bug reporting, prioritization of fixes and feature requests.
-These diagnostics are opt-in and can be enabled or disabled using the `lancedb diagnostics` command. These are enabled by default.
-Get usage help.
-```
-lancedb diagnostics --help
-```
-Disable diagnostics
-```
-lancedb diagnostics --disabled
-```
-Enable diagnostics
-```
-lancedb diagnostics --enabled
-```

docs/src/embedding.md

@@ -66,7 +66,7 @@ You can also use an external API like OpenAI to generate embeddings
     to generate embeddings for each row.
 
     Say if you have a pandas DataFrame with a `text` column that you want to be embedded,
-    you can use the [with_embeddings](https://lancedb.github.io/lancedb/python/python/#lancedb.embeddings.with_embeddings)
+    you can use the [with_embeddings](https://lancedb.github.io/lancedb/python/#lancedb.embeddings.with_embeddings)
     function to generate embeddings and add create a combined pyarrow table:
 
 
@@ -118,7 +118,7 @@ belong in the same latent space and your results will be nonsensical.
      ```python
      query = "What's the best pizza topping?"
      query_vector = embed_func([query])[0]
-     tbl.search(query_vector).limit(10).to_pandas()
+     tbl.search(query_vector).limit(10).to_df()
      ```
 
      The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.

docs/src/examples/index.md

@@ -1,23 +0,0 @@
-# Examples
-
-Here are some of the examples, projects and applications using LanceDB python library. Some examples are covered in detail in the next sections. You can find more on [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes)
-
-| Example | Interactive Envs | Scripts  |
-|-------- | ---------------- | ------   |
-| | | |
-| [Youtube transcript search bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/youtube_bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/main.py)|
-| [Langchain: Code Docs QA bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Code-Documentation-QA-Bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/main.py) |
-| [AI Agents: Reducing Hallucination](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/reducing_hallucinations_ai_agents/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/main.py)|
-| [Multimodal CLIP: DiffusionDB](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_clip/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_clip/main.py)  |
-| [Multimodal CLIP: Youtube videos](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_video_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_video_search/main.py)  |
-| [Movie Recommender](https://github.com/lancedb/vectordb-recipes/tree/main/examples/movie-recommender/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/movie-recommender/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/movie-recommender/main.py)  |
-| [Audio Search](https://github.com/lancedb/vectordb-recipes/tree/main/examples/audio_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/audio_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/audio_search/main.py)  |
-| [Multimodal Image + Text Search](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_search/main.py)  |
-| [Evaluating Prompts with Prompttools](https://github.com/lancedb/vectordb-recipes/tree/main/examples/prompttools-eval-prompts/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/prompttools-eval-prompts/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> |  |
-
-## Projects & Applications powered by LanceDB
-
-| Project Name                                        | Description                                                                                                          | Screenshot                                |
-|-----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------|-------------------------------------------|
-| [YOLOExplorer](https://github.com/lancedb/yoloexplorer) | Iterate on your YOLO / CV datasets using SQL, Vector semantic search, and more within seconds                  | ![YOLOExplorer](https://github.com/lancedb/vectordb-recipes/assets/15766192/ae513a29-8f15-4e0b-99a1-ccd8272b6131) |
-| [Website Chatbot (Deployable Vercel Template)](https://github.com/lancedb/lancedb-vercel-chatbot) | Create a chatbot from the sitemap of any website/docs of your choice. Built using vectorDB serverless native javascript package. | ![Chatbot](../assets/vercel-template.gif)    |

docs/src/examples/index_js.md

@@ -1,19 +0,0 @@
-# Examples
-
-Here are some of the examples, projects and applications using vectordb native javascript library.
-Some examples are covered in detail in the next sections. You can find more on [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes)
-
-| Example | Scripts  |
-|-------- | ------   |
-| | |
-| [Youtube transcript search bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/index.js)|
-| [Langchain: Code Docs QA bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/index.js)|
-| [AI Agents: Reducing Hallucination](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/index.js)|
-| [TransformersJS Embedding example](https://github.com/lancedb/vectordb-recipes/tree/main/examples/js-transformers/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/js-transformers/index.js)  |
-
-## Projects & Applications
-
-| Project Name                                        | Description                                                                                                          | Screenshot                                |
-|-----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------|-------------------------------------------|
-| [YOLOExplorer](https://github.com/lancedb/yoloexplorer) | Iterate on your YOLO / CV datasets using SQL, Vector semantic search, and more within seconds                  | ![YOLOExplorer](https://github.com/lancedb/vectordb-recipes/assets/15766192/ae513a29-8f15-4e0b-99a1-ccd8272b6131) |
-| [Website Chatbot (Deployable Vercel Template)](https://github.com/lancedb/lancedb-vercel-chatbot) | Create a chatbot from the sitemap of any website/docs of your choice. Built using vectorDB serverless native javascript package. | ![Chatbot](../assets/vercel-template.gif)    |

docs/src/examples/serverless_lancedb_with_s3_and_lambda.md

@@ -80,14 +80,14 @@ def handler(event, context):
     # Shape of SIFT is (128,1M), d=float32
     query_vector = np.array(event['query_vector'], dtype=np.float32)
 
-    rs = table.search(query_vector).limit(2).to_list()
+    rs = table.search(query_vector).limit(2).to_df()
 
     return {
         "statusCode": status_code,
         "headers": {
             "Content-Type": "application/json"
         },
-        "body": json.dumps(rs)
+        "body": rs.to_json()
     }
 ``` 
 

docs/src/examples/serverless_website_chatbot.md

@@ -1,61 +0,0 @@
-# LanceDB Chatbot - Vercel Next.js Template
-Use an AI chatbot with website context retrieved from a vector store like LanceDB. LanceDB is lightweight and can be embedded directly into Next.js, with data stored on-prem.
-
-## One click deploy on Vercel
-[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Flancedb%2Flancedb-vercel-chatbot&env=OPENAI_API_KEY&envDescription=OpenAI%20API%20Key%20for%20chat%20completion.&project-name=lancedb-vercel-chatbot&repository-name=lancedb-vercel-chatbot&demo-title=LanceDB%20Chatbot%20Demo&demo-description=Demo%20website%20chatbot%20with%20LanceDB.&demo-url=https%3A%2F%2Flancedb.vercel.app&demo-image=https%3A%2F%2Fi.imgur.com%2FazVJtvr.png)
-
-![Demo website landing page](../assets/vercel-template.gif)
-
-## Development
-
-First, rename `.env.example` to `.env.local`, and fill out `OPENAI_API_KEY` with your OpenAI API key. You can get one [here](https://openai.com/blog/openai-api).
-
-Run the development server:
-
-```bash
-npm run dev
-# or
-yarn dev
-# or
-pnpm dev
-```
-
-Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
-
-This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
-
-## Learn More
-
-To learn more about LanceDB or Next.js, take a look at the following resources:
-
-- [LanceDB Documentation](https://lancedb.github.io/lancedb/) - learn about LanceDB, the developer-friendly serverless vector database.
-- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
-- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
-
-## LanceDB on Next.js and Vercel
-
-FYI: these configurations have been pre-implemented in this template.
-
-Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying on Vercel.
-```js
-/** @type {import('next').NextConfig} */
-module.exports = ({
-  webpack(config) {
-    config.externals.push({ vectordb: 'vectordb' })
-    return config;
-  }
-})
-```
-
-To deploy on Vercel, we need to make sure that the NodeJS runtime static file analysis for Vercel can find the binary, since LanceDB uses dynamic imports by default. We can do this by modifying `package.json` in the `scripts` section.
-```json
-{
-  ...
-  "scripts": {
-    ...
-    "vercel-build": "sed -i 's/nativeLib = require(`@lancedb\\/vectordb-\\${currentTarget()}`);/nativeLib = require(`@lancedb\\/vectordb-linux-x64-gnu`);/' node_modules/vectordb/native.js && next build",
-    ...
-  },
-  ...
-}
-```

docs/src/examples/transformerjs_embedding_search_nodejs.md

@@ -1,6 +1,6 @@
 # Vector embedding search using TransformersJS
 
-## Embed and query data from LanceDB using TransformersJS
+## Embed and query data from LacneDB using TransformersJS
 
 <img id="splash" width="400" alt="transformersjs" src="https://github.com/lancedb/lancedb/assets/43097991/88a31e30-3d6f-4eef-9216-4b7c688f1b4f">
 
@@ -99,7 +99,7 @@ Output of `results`:
     id: 5,
     text: 'Banana',
     type: 'fruit',
-    _distance: 0.4919965863227844
+    score: 0.4919965863227844
   },
   {
     vector: Float32Array(384) [
@@ -111,7 +111,7 @@ Output of `results`:
     id: 1,
     text: 'Cherry',
     type: 'fruit',
-    _distance: 0.5540297031402588
+    score: 0.5540297031402588
   }
 ]
 ```

docs/src/examples/youtube_transcript_bot.md

@@ -4,10 +4,4 @@
 
 <img id="splash" width="400" alt="youtube transcript search" src="https://user-images.githubusercontent.com/917119/236965568-def7394d-171c-45f2-939d-8edfeaadd88c.png">
 
-
-<a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/youtube_bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab">
-
-Scripts - [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipesexamples/youtube_bot/main.py)  [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/examples/youtube_bot/index.js)
-
-
 This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb)

docs/src/fts.md

@@ -6,19 +6,17 @@ to make this available for JS as well.
 
 ## Installation
 
-To use full text search, you must install the dependency `tantivy-py`:
+To use full text search, you must install optional dependency tantivy-py:
 
-# tantivy 0.20.1
-```sh
-pip install tantivy==0.20.1
-```
+# tantivy 0.19.2
+pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
 
 
 ## Quickstart
 
 Assume:
 1. `table` is a LanceDB Table
-2. `text` is the name of the `Table` column that we want to index
+2. `text` is the name of the Table column that we want to index
 
 For example,
 
@@ -43,13 +41,7 @@ table.create_fts_index("text")
 To search:
 
 ```python
-table.search("puppy").limit(10).select(["text"]).to_list()
-```
-
-Which returns a list of dictionaries:
-
-```python
-[{'text': 'Frodo was a happy puppy', 'score': 0.6931471824645996}]
+df = table.search("puppy").limit(10).select(["text"]).to_df()
 ```
 
 LanceDB automatically looks for an FTS index if the input is str.

docs/src/guides/tables.md

@@ -1,411 +0,0 @@
- <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/tables_guide.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
-A Table is a collection of Records in a LanceDB Database. You can follow along on colab!
-
-## Creating a LanceDB Table
-
-=== "Python"
-    ### LanceDB Connection
-
-    ```python
-    import lancedb
-    db = lancedb.connect("./.lancedb")
-    ```
-
-    LanceDB allows ingesting data from various sources - `dict`, `list[dict]`, `pd.DataFrame`, `pa.Table` or a `Iterator[pa.RecordBatch]`. Let's take a look at some of the these.
-
-    ### From list of tuples or dictionaries
-
-    ```python
-    import lancedb
-
-    db = lancedb.connect("./.lancedb")
-
-    data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
-            {"vector": [0.2, 1.8], "lat": 40.1, "long": -74.1}]
-
-    db.create_table("my_table", data)
-
-    db["my_table"].head()
-    ```
-
-    !!! info "Note"
-        If the table already exists, LanceDB will raise an error by default. If you want to overwrite the table, you can pass in mode="overwrite" to the createTable function.
-
-        ```python
-        db.create_table("name", data, mode="overwrite")
-        ```
-
-
-    ### From pandas DataFrame
-
-    ```python
-    import pandas as pd
-
-    data = pd.DataFrame({
-        "vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],
-        "lat": [45.5, 40.1],
-        "long": [-122.7, -74.1]
-    })
-
-    db.create_table("table2", data)
-
-    db["table2"].head()
-    ```
-    !!! info "Note"
-        Data is converted to Arrow before being written to disk. For maximum control over how data is saved, either provide the PyArrow schema to convert to or else provide a PyArrow Table directly.
-
-    ```python
-    custom_schema = pa.schema([
-    pa.field("vector", pa.list_(pa.float32(), 4)),
-    pa.field("lat", pa.float32()),
-    pa.field("long", pa.float32())
-    ])
-
-    table = db.create_table("table3", data, schema=custom_schema)
-    ```
-
-    ### From PyArrow Tables
-    You can also create LanceDB tables directly from pyarrow tables
-
-    ```python
-    table = pa.Table.from_arrays(
-            [
-                pa.array([[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],
-                        pa.list_(pa.float32(), 4)),
-                pa.array(["foo", "bar"]),
-                pa.array([10.0, 20.0]),
-            ],
-            ["vector", "item", "price"],
-        )
-
-    db = lancedb.connect("db")
-
-    tbl = db.create_table("test1", table)
-    ```
-
-    ### From Pydantic Models
-    When you create an empty table without data, you must specify the table schema.
-    LanceDB supports creating tables by specifying a pyarrow schema or a specialized
-    pydantic model called `LanceModel`.
-
-    For example, the following Content model specifies a table with 5 columns:
-    movie_id, vector, genres, title, and imdb_id. When you create a table, you can
-    pass the class as the value of the `schema` parameter to `create_table`.
-    The `vector` column is a `Vector` type, which is a specialized pydantic type that
-    can be configured with the vector dimensions. It is also important to note that
-    LanceDB only understands subclasses of `lancedb.pydantic.LanceModel`
-    (which itself derives from `pydantic.BaseModel`).
-
-    ```python
-    from lancedb.pydantic import Vector, LanceModel
-
-    class Content(LanceModel):
-        movie_id: int
-        vector: Vector(128)
-        genres: str
-        title: str
-        imdb_id: int
-
-        @property
-        def imdb_url(self) -> str:
-            return f"https://www.imdb.com/title/tt{self.imdb_id}"
-
-    import pyarrow as pa
-    db = lancedb.connect("~/.lancedb")
-    table_name = "movielens_small"
-    table = db.create_table(table_name, schema=Content)
-    ```
-
-    ### Using Iterators / Writing Large Datasets
-
-    It is recommended to use itertators to add large datasets in batches when creating your table in one go. This does not create multiple versions of your dataset unlike manually adding batches using `table.add()`
-
-    LanceDB additionally supports pyarrow's `RecordBatch` Iterators or other generators producing supported data types.
-
-    Here's an example using using `RecordBatch` iterator for creating tables.
-
-    ```python
-    import pyarrow as pa
-
-    def make_batches():
-        for i in range(5):
-            yield pa.RecordBatch.from_arrays(
-                [
-                    pa.array([[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],
-                            pa.list_(pa.float32(), 4)),
-                    pa.array(["foo", "bar"]),
-                    pa.array([10.0, 20.0]),
-                ],
-                ["vector", "item", "price"],
-            )
-
-    schema = pa.schema([
-        pa.field("vector", pa.list_(pa.float32(), 4)),
-        pa.field("item", pa.utf8()),
-        pa.field("price", pa.float32()),
-    ])
-
-    db.create_table("table4", make_batches(), schema=schema)
-    ```
-
-    You can also use iterators of other types like Pandas dataframe or Pylists directly in the above example.
-
-    ## Creating Empty Table
-    You can also create empty tables in python. Initialize it with schema and later ingest data into it.
-
-    ```python
-    import lancedb
-    import pyarrow as pa
-
-    schema = pa.schema(
-      [
-          pa.field("vector", pa.list_(pa.float32(), 2)),
-          pa.field("item", pa.string()),
-          pa.field("price", pa.float32()),
-      ])
-    tbl = db.create_table("table5", schema=schema)
-    data = [
-        {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0},
-    ]
-    tbl.add(data=data)
-    ```
-
-    You can also use Pydantic to specify the schema
-
-    ```python
-    import lancedb
-    from lancedb.pydantic import LanceModel, vector
-
-    class Model(LanceModel):
-          vector: Vector(2)
-
-    tbl = db.create_table("table5", schema=Model.to_arrow_schema())
-    ```
-
-=== "Javascript/Typescript"
-
-    ### VectorDB Connection
-
-    ```javascript
-    const lancedb = require("vectordb");
-
-    const uri = "data/sample-lancedb";
-    const db = await lancedb.connect(uri);
-    ```
-
-    ### Creating a Table
-
-    You can create a LanceDB table in javascript using an array of records.
-
-    ```javascript
-    data
-    const tb = await db.createTable("my_table",
-                  data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-    ```
-
-    !!! info "Note"
-    If the table already exists, LanceDB will raise an error by default. If you want to overwrite the table, you need to specify the `WriteMode` in the createTable function.
-
-    ```javascript
-    const table = await con.createTable(tableName, data, { writeMode: WriteMode.Overwrite })
-    ```
-
-## Open existing tables
-
-If you forget the name of your table, you can always get a listing of all table names:
-
-
-=== "Python"
-    ### Get a list of existing Tables
-
-    ```python
-    print(db.table_names())
-    ```
-=== "Javascript/Typescript"
-
-    ```javascript
-    console.log(await db.tableNames());
-    ```
-
-Then, you can open any existing tables
-
-=== "Python"
-
-    ```python
-    tbl = db.open_table("my_table")
-    ```
-=== "Javascript/Typescript"
-
-    ```javascript
-    const tbl = await db.openTable("my_table");
-    ```
-
-## Adding to a Table
-After a table has been created, you can always add more data to it using
-
-=== "Python"
-    You can add any of the valid data structures accepted by LanceDB table, i.e, `dict`, `list[dict]`, `pd.DataFrame`, or a `Iterator[pa.RecordBatch]`. Here are some examples.
-
-    ### Adding Pandas DataFrame
-
-    ```python
-    df = pd.DataFrame([{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
-                  {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}])
-    tbl.add(df)
-    ```
-
-    You can also add a large dataset batch in one go using Iterator of any supported data types.
-
-    ### Adding to table using Iterator
-
-    ```python
-    import pandas as pd
-
-    def make_batches():
-        for i in range(5):
-            yield pd.DataFrame(
-                {
-                    "vector": [[3.1, 4.1], [1, 1]],
-                    "item": ["foo", "bar"],
-                    "price": [10.0, 20.0],
-                })
-
-    tbl.add(make_batches())
-    ```
-
-    The other arguments accepted:
-
-    | Name | Type | Description | Default |
-    |---|---|---|---|
-    | data | DATA | The data to insert into the table. | required |
-    | mode | str | The mode to use when writing the data. Valid values are "append" and "overwrite". | append |
-    | on_bad_vectors | str | What to do if any of the vectors are not the same size or contains NaNs. One of "error", "drop", "fill". | drop |
-    | fill value | float | The value to use when filling vectors: Only used if on_bad_vectors="fill". | 0.0 |
-
-
-=== "Javascript/Typescript"
-
-    ```javascript
-    await tbl.add([{vector: [1.3, 1.4], item: "fizz", price: 100.0},
-        {vector: [9.5, 56.2], item: "buzz", price: 200.0}])
-    ```
-
-## Deleting from a Table
-
-Use the `delete()` method on tables to delete rows from a table. To choose which rows to delete, provide a filter that matches on the metadata columns. This can delete any number of rows that match the filter.
-
-=== "Python"
-
-    ```python
-    tbl.delete('item = "fizz"')
-    ```
-
-    ### Deleting row with specific column value
-
-    ```python
-    import lancedb
-    import pandas as pd
-
-    data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
-    db = lancedb.connect("./.lancedb")
-    table = db.create_table("my_table", data)
-    table.to_pandas()
-    #   x      vector
-    # 0  1  [1.0, 2.0]
-    # 1  2  [3.0, 4.0]
-    # 2  3  [5.0, 6.0]
-
-    table.delete("x = 2")
-    table.to_pandas()
-    #   x      vector
-    # 0  1  [1.0, 2.0]
-    # 1  3  [5.0, 6.0]
-    ```
-
-    ### Delete from a list of values
-
-    ```python
-    to_remove = [1, 5]
-    to_remove = ", ".join(str(v) for v in to_remove)
-
-    table.delete(f"x IN ({to_remove})")
-    table.to_pandas()
-    #   x      vector
-    # 0  3  [5.0, 6.0]
-    ```
-
-=== "Javascript/Typescript"
-
-    ```javascript
-    await tbl.delete('item = "fizz"')
-    ```
-
-    ### Deleting row with specific column value
-
-    ```javascript
-    const con = await lancedb.connect("./.lancedb")
-    const data = [
-      {id: 1, vector: [1, 2]},
-      {id: 2, vector: [3, 4]},
-      {id: 3, vector: [5, 6]},
-    ];
-    const tbl = await con.createTable("my_table", data)
-    await tbl.delete("id = 2")
-    await tbl.countRows() // Returns 2
-    ```
-
-    ### Delete from a list of values
-
-    ```javascript
-    const to_remove = [1, 5];
-    await tbl.delete(`id IN (${to_remove.join(",")})`)
-    await tbl.countRows() // Returns 1
-    ```
-
-### Updating a Table [Experimental]
-EXPERIMENTAL: Update rows in the table (not threadsafe).
-
-This can be used to update zero to all rows depending on how many rows match the where clause.
-
-| Parameter | Type | Description |
-|---|---|---|
-| `where` | `str` | The SQL where clause to use when updating rows. For example, `'x = 2'` or `'x IN (1, 2, 3)'`. The filter must not be empty, or it will error. |
-| `values` | `dict` | The values to update. The keys are the column names and the values are the values to set. |
-
-
-=== "Python"
-
-    ```python
-    import lancedb
-    import pandas as pd
-
-    # Create a lancedb connection
-    db = lancedb.connect("./.lancedb")
-
-    # Create a table from a pandas DataFrame
-    data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
-    table = db.create_table("my_table", data)
-
-    # Update the table where x = 2
-    table.update(where="x = 2", values={"vector": [10, 10]})
-
-    # Get the updated table as a pandas DataFrame
-    df = table.to_pandas()
-
-    # Print the DataFrame
-    print(df)
-    ```
-
-    Output
-    ```shell
-        x  vector
-    0  1  [1.0, 2.0]
-    1  3  [5.0, 6.0]
-    2  2  [10.0, 10.0]
-    ```
-
-## What's Next?
-
-Learn how to Query your tables and create indices

docs/src/index.md

@@ -1,23 +1,20 @@
-# LanceDB
+# Welcome to LanceDB's Documentation
 
-LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrieval, filtering and management of embeddings.
-
-![Illustration](/lancedb/assets/ecosystem-illustration.png)
+LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrevial, filtering and management of embeddings.
 
 The key features of LanceDB include:
 
+* Production-scale vector search with no servers to manage.
+
 * Store, query and filter vectors, metadata and multi-modal data (text, images, videos, point clouds, and more).
 
-* Support for production-scale vector similarity search, full-text search and SQL, with no servers to manage.
+* Support for vector similarity search, full-text search and SQL.
 
 * Native Python and Javascript/Typescript support.
 
 * Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure.
 
-* Persisted on HDD, allowing scalability without breaking the bank.
-
-* Ingest your favorite data formats directly, like pandas DataFrames, Pydantic objects and more.
-
+* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
 
 LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
 
@@ -36,7 +33,7 @@ LanceDB's core is written in Rust 🦀 and is built using <a href="https://githu
       table = db.create_table("my_table",
                               data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
                                     {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      result = table.search([100, 100]).limit(2).to_list()
+      result = table.search([100, 100]).limit(2).to_df()
       ```
 
 === "Javascript"
@@ -72,4 +69,4 @@ LanceDB's core is written in Rust 🦀 and is built using <a href="https://githu
 * [`Full text search`](fts.md) - [EXPERIMENTAL] full-text search API
 * [`Ecosystem Integrations`](python/integration.md) - integrating LanceDB with python data tooling ecosystem.
 * [`Python API Reference`](python/python.md) - detailed documentation for the LanceDB Python SDK.
-* [`Node API Reference`](javascript/modules.md) - detailed documentation for the LanceDB Node SDK.
+* [`Node API Reference`](javascript/modules.md) - detailed documentation for the LanceDB Python SDK.

docs/src/integrations/index.md

@@ -1,21 +0,0 @@
-# Integrations
-
-## Data Formats
-
-LanceDB supports ingesting from your favorite data tools.
-
-![Illustration](/lancedb/assets/ecosystem-illustration.png)
-
-
-## Tools
-
-LanceDB is integrated with most of the popular AI tools, with more coming soon.
-Get started using these examples and quick links.
-
-| Integrations | |
-|---|---:|
-| <h3> LlamaIndex </h3>LlamaIndex is a simple, flexible data framework for connecting custom data sources to large language models. Llama index integrates with LanceDB as the serverless VectorDB. <h3>[Lean More](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html) </h3> |<img src="../assets/llama-index.jpg" alt="image" width="150" height="auto">|
-| <h3>Langchain</h3>Langchain allows building applications with LLMs through composability <h3>[Lean More](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html) | <img src="../assets/langchain.png" alt="image" width="150" height="auto">|
-| <h3>Langchain TS</h3> Javascript bindings for Langchain. It integrates with LanceDB's serverless vectordb allowing you to build powerful AI applications through composibility using only serverless functions. <h3>[Learn More]( https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb) | <img src="../assets/langchain.png" alt="image" width="150" height="auto">|
-| <h3>Voxel51</h3>  It is an open source toolkit that enables you to build better computer vision workflows by improving the quality of your datasets and delivering insights about your models.<h3>[Learn More](./voxel51.md) | <img src="../assets/voxel.gif" alt="image" width="150" height="auto">|
-| <h3>PromptTools</h3>  Offers a set of free, open-source tools for testing and experimenting with models, prompts, and configurations. The core idea is to enable developers to evaluate prompts using familiar interfaces like code and notebooks. You can use it to experiment with different configurations of LanceDB, and test how LanceDB integrates with the LLM of your choice.<h3>[Learn More](./prompttools.md) | <img src="../assets/prompttools.jpeg" alt="image" width="150" height="auto">|

docs/src/integrations/prompttools.md

@@ -1,7 +0,0 @@
-
-[PromptTools](https://github.com/hegelai/prompttools) offers a set of free, open-source tools for testing and experimenting with models, prompts, and configurations. The core idea is to enable developers to evaluate prompts using familiar interfaces like code and notebooks. You can use it to experiment with different configurations of LanceDB, and test how LanceDB integrates with the LLM of your choice.
-
-[Evaluating Prompts with PromptTools](./examples/prompttools-eval-prompts/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/prompttools-eval-prompts/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>
-
-![Alt text](https://prompttools.readthedocs.io/en/latest/_images/demo.gif "a title")
-

docs/src/integrations/voxel51.md

@@ -1,71 +0,0 @@
-![example](/assets/voxel.gif)
-
-Basic recipe
-____________
-
-The basic workflow to use LanceDB to create a similarity index on your FiftyOne
-datasets and use this to query your data is as follows:
-
-1)  Load a dataset into FiftyOne
-
-2)  Compute embedding vectors for samples or patches in your dataset, or select
-    a model to use to generate embeddings
-
-3)  Use the `compute_similarity()`
-    method to generate a LanceDB table for the samples or object
-    patches embeddings in a dataset by setting the parameter `backend="lancedb"` and
-    specifying a `brain_key` of your choice
-
-4)  Use this LanceDB table to query your data with
-    `sort_by_similarity()`
-
-5) If desired, delete the table
-
-The example below demonstrates this workflow.
-
-!!! Note
-
-    You must install the LanceDB Python client to run this
-    ```
-    pip install lancedb
-    ```
-
-```python
-
-import fiftyone as fo
-import fiftyone.brain as fob
-import fiftyone.zoo as foz
-
-# Step 1: Load your data into FiftyOne
-dataset = foz.load_zoo_dataset("quickstart")
-
-# Steps 2 and 3: Compute embeddings and create a similarity index
-lancedb_index = fob.compute_similarity(
-    dataset, 
-    model="clip-vit-base32-torch",
-    brain_key="lancedb_index",
-    backend="lancedb",
-)
-```
-Once the similarity index has been generated, we can query our data in FiftyOne
-by specifying the `brain_key`:
-
-```python
-# Step 4: Query your data
-query = dataset.first().id  # query by sample ID
-view = dataset.sort_by_similarity(
-    query, 
-    brain_key="lancedb_index",
-    k=10,  # limit to 10 most similar samples
-)
-
-# Step 5 (optional): Cleanup
-
-# Delete the LanceDB table
-lancedb_index.cleanup()
-
-# Delete run record from FiftyOne
-dataset.delete_brain_run("lancedb_index")
-```
-
-More in depth walkthrough of the integration, visit the LanceDB guide on Voxel51 - [LaceDB x Voxel51](https://docs.voxel51.com/integrations/lancedb.html)

docs/src/javascript/README.md

@@ -10,10 +10,6 @@ A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb)
 npm install vectordb
 ```
 
-This will download the appropriate native library for your platform. We currently
-support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
-yet support Windows or musl-based Linux (such as Alpine Linux).
-
 ## Usage
 
 ### Basic Example
@@ -32,34 +28,12 @@ The [examples](./examples) folder contains complete examples.
 
 ## Development
 
-To build everything fresh:
-
-```bash
-npm install
-npm run tsc
-npm run build
-```
-
-Then you should be able to run the tests with:
+Run the tests with
 
 ```bash
 npm test
 ```
 
-### Rebuilding Rust library
-
-```bash
-npm run build
-```
-
-### Rebuilding Typescript
-
-```bash
-npm run tsc
-```
-
-### Fix lints
-
 To run the linter and have it automatically fix all errors
 
 ```bash

docs/src/javascript/classes/LocalConnection.md

@@ -17,7 +17,7 @@ A connection to a LanceDB database.
 ### Properties
 
 - [\_db](LocalConnection.md#_db)
-- [\_options](LocalConnection.md#_options)
+- [\_uri](LocalConnection.md#_uri)
 
 ### Accessors
 
@@ -35,18 +35,18 @@ A connection to a LanceDB database.
 
 ### constructor
 
-• **new LocalConnection**(`db`, `options`)
+• **new LocalConnection**(`db`, `uri`)
 
 #### Parameters
 
 | Name | Type |
 | :------ | :------ |
 | `db` | `any` |
-| `options` | [`ConnectionOptions`](../interfaces/ConnectionOptions.md) |
+| `uri` | `string` |
 
 #### Defined in
 
-[index.ts:184](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L184)
+[index.ts:132](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L132)
 
 ## Properties
 
@@ -56,17 +56,17 @@ A connection to a LanceDB database.
 
 #### Defined in
 
-[index.ts:182](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L182)
+[index.ts:130](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L130)
 
 ___
 
-### \_options
+### \_uri
 
-• `Private` `Readonly` **\_options**: [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
+• `Private` `Readonly` **\_uri**: `string`
 
 #### Defined in
 
-[index.ts:181](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L181)
+[index.ts:129](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L129)
 
 ## Accessors
 
@@ -84,7 +84,7 @@ ___
 
 #### Defined in
 
-[index.ts:189](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L189)
+[index.ts:137](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L137)
 
 ## Methods
 
@@ -112,7 +112,7 @@ Creates a new Table and initialize it with new data.
 
 #### Defined in
 
-[index.ts:230](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L230)
+[index.ts:177](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L177)
 
 ▸ **createTable**(`name`, `data`, `mode`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
 
@@ -134,7 +134,7 @@ Connection.createTable
 
 #### Defined in
 
-[index.ts:231](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L231)
+[index.ts:178](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L178)
 
 ▸ **createTable**<`T`\>(`name`, `data`, `mode`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
 
@@ -165,36 +165,7 @@ Connection.createTable
 
 #### Defined in
 
-[index.ts:241](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L241)
-
-▸ **createTable**<`T`\>(`name`, `data`, `mode`, `embeddings?`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` |
-| `data` | `Record`<`string`, `unknown`\>[] |
-| `mode` | [`WriteMode`](../enums/WriteMode.md) |
-| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-#### Implementation of
-
-Connection.createTable
-
-#### Defined in
-
-[index.ts:242](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L242)
+[index.ts:188](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L188)
 
 ___
 
@@ -219,7 +190,7 @@ ___
 
 #### Defined in
 
-[index.ts:266](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L266)
+[index.ts:201](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L201)
 
 ___
 
@@ -245,7 +216,7 @@ Drop an existing table.
 
 #### Defined in
 
-[index.ts:276](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L276)
+[index.ts:211](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L211)
 
 ___
 
@@ -271,7 +242,7 @@ Open a table in the database.
 
 #### Defined in
 
-[index.ts:205](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L205)
+[index.ts:153](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L153)
 
 ▸ **openTable**<`T`\>(`name`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
 
@@ -300,34 +271,7 @@ Connection.openTable
 
 #### Defined in
 
-[index.ts:212](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L212)
-
-▸ **openTable**<`T`\>(`name`, `embeddings?`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` |
-| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-#### Implementation of
-
-Connection.openTable
-
-#### Defined in
-
-[index.ts:213](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L213)
+[index.ts:160](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L160)
 
 ___
 
@@ -347,4 +291,4 @@ Get the names of all tables in the database.
 
 #### Defined in
 
-[index.ts:196](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L196)
+[index.ts:144](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L144)

docs/src/javascript/classes/LocalTable.md

@@ -24,7 +24,6 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 - [\_embeddings](LocalTable.md#_embeddings)
 - [\_name](LocalTable.md#_name)
-- [\_options](LocalTable.md#_options)
 - [\_tbl](LocalTable.md#_tbl)
 
 ### Accessors
@@ -44,7 +43,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 ### constructor
 
-• **new LocalTable**<`T`\>(`tbl`, `name`, `options`)
+• **new LocalTable**<`T`\>(`tbl`, `name`)
 
 #### Type parameters
 
@@ -58,13 +57,12 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 | :------ | :------ |
 | `tbl` | `any` |
 | `name` | `string` |
-| `options` | [`ConnectionOptions`](../interfaces/ConnectionOptions.md) |
 
 #### Defined in
 
-[index.ts:287](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L287)
+[index.ts:221](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L221)
 
-• **new LocalTable**<`T`\>(`tbl`, `name`, `options`, `embeddings`)
+• **new LocalTable**<`T`\>(`tbl`, `name`, `embeddings`)
 
 #### Type parameters
 
@@ -78,12 +76,11 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 | :------ | :------ | :------ |
 | `tbl` | `any` |  |
 | `name` | `string` |  |
-| `options` | [`ConnectionOptions`](../interfaces/ConnectionOptions.md) |  |
 | `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use when interacting with this table |
 
 #### Defined in
 
-[index.ts:294](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L294)
+[index.ts:227](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L227)
 
 ## Properties
 
@@ -93,7 +90,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 #### Defined in
 
-[index.ts:284](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L284)
+[index.ts:219](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L219)
 
 ___
 
@@ -103,17 +100,7 @@ ___
 
 #### Defined in
 
-[index.ts:283](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L283)
-
-___
-
-### \_options
-
-• `Private` `Readonly` **\_options**: [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
-
-#### Defined in
-
-[index.ts:285](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L285)
+[index.ts:218](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L218)
 
 ___
 
@@ -123,7 +110,7 @@ ___
 
 #### Defined in
 
-[index.ts:282](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L282)
+[index.ts:217](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L217)
 
 ## Accessors
 
@@ -141,7 +128,7 @@ ___
 
 #### Defined in
 
-[index.ts:302](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L302)
+[index.ts:234](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L234)
 
 ## Methods
 
@@ -169,7 +156,7 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:320](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L320)
+[index.ts:252](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L252)
 
 ___
 
@@ -189,7 +176,7 @@ Returns the number of rows in this table.
 
 #### Defined in
 
-[index.ts:362](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L362)
+[index.ts:278](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L278)
 
 ___
 
@@ -207,7 +194,7 @@ VectorIndexParams.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `indexParams` | [`IvfPQIndexConfig`](../interfaces/IvfPQIndexConfig.md) | The parameters of this Index, |
+| `indexParams` | `IvfPQIndexConfig` | The parameters of this Index, |
 
 #### Returns
 
@@ -219,7 +206,7 @@ VectorIndexParams.
 
 #### Defined in
 
-[index.ts:355](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L355)
+[index.ts:271](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L271)
 
 ___
 
@@ -245,7 +232,7 @@ Delete rows from this table.
 
 #### Defined in
 
-[index.ts:371](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L371)
+[index.ts:287](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L287)
 
 ___
 
@@ -273,7 +260,7 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:338](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L338)
+[index.ts:262](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L262)
 
 ___
 
@@ -299,4 +286,4 @@ Creates a search query to find the nearest neighbors of the given search term
 
 #### Defined in
 
-[index.ts:310](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L310)
+[index.ts:242](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L242)

docs/src/javascript/classes/OpenAIEmbeddingFunction.md

@@ -40,7 +40,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L21)
+[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L21)
 
 ## Properties
 
@@ -50,7 +50,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L19)
+[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L19)
 
 ___
 
@@ -60,7 +60,7 @@ ___
 
 #### Defined in
 
-[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L18)
+[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L18)
 
 ___
 
@@ -76,7 +76,7 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L50)
+[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L50)
 
 ## Methods
 
@@ -102,4 +102,4 @@ Creates a vector representation for the given values.
 
 #### Defined in
 
-[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L38)
+[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L38)

docs/src/javascript/classes/Query.md

@@ -62,7 +62,7 @@ A builder for nearest neighbor queries for LanceDB.
 
 #### Defined in
 
-[index.ts:448](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L448)
+[index.ts:362](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L362)
 
 ## Properties
 
@@ -72,7 +72,7 @@ A builder for nearest neighbor queries for LanceDB.
 
 #### Defined in
 
-[index.ts:446](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L446)
+[index.ts:360](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L360)
 
 ___
 
@@ -82,7 +82,7 @@ ___
 
 #### Defined in
 
-[index.ts:444](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L444)
+[index.ts:358](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L358)
 
 ___
 
@@ -92,7 +92,7 @@ ___
 
 #### Defined in
 
-[index.ts:440](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L440)
+[index.ts:354](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L354)
 
 ___
 
@@ -102,7 +102,7 @@ ___
 
 #### Defined in
 
-[index.ts:445](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L445)
+[index.ts:359](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L359)
 
 ___
 
@@ -112,7 +112,7 @@ ___
 
 #### Defined in
 
-[index.ts:442](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L442)
+[index.ts:356](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L356)
 
 ___
 
@@ -122,7 +122,7 @@ ___
 
 #### Defined in
 
-[index.ts:438](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L438)
+[index.ts:352](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L352)
 
 ___
 
@@ -132,7 +132,7 @@ ___
 
 #### Defined in
 
-[index.ts:439](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L439)
+[index.ts:353](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L353)
 
 ___
 
@@ -142,7 +142,7 @@ ___
 
 #### Defined in
 
-[index.ts:441](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L441)
+[index.ts:355](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L355)
 
 ___
 
@@ -152,7 +152,7 @@ ___
 
 #### Defined in
 
-[index.ts:443](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L443)
+[index.ts:357](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L357)
 
 ___
 
@@ -162,7 +162,7 @@ ___
 
 #### Defined in
 
-[index.ts:437](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L437)
+[index.ts:351](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L351)
 
 ___
 
@@ -188,7 +188,7 @@ A filter statement to be applied to this query.
 
 #### Defined in
 
-[index.ts:496](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L496)
+[index.ts:410](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L410)
 
 ## Methods
 
@@ -210,7 +210,7 @@ Execute the query and return the results as an Array of Objects
 
 #### Defined in
 
-[index.ts:519](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L519)
+[index.ts:433](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L433)
 
 ___
 
@@ -232,7 +232,7 @@ A filter statement to be applied to this query.
 
 #### Defined in
 
-[index.ts:491](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L491)
+[index.ts:405](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L405)
 
 ___
 
@@ -254,7 +254,7 @@ Sets the number of results that will be returned
 
 #### Defined in
 
-[index.ts:464](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L464)
+[index.ts:378](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L378)
 
 ___
 
@@ -280,7 +280,7 @@ MetricType for the different options
 
 #### Defined in
 
-[index.ts:511](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L511)
+[index.ts:425](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L425)
 
 ___
 
@@ -302,7 +302,7 @@ The number of probes used. A higher number makes search more accurate but also s
 
 #### Defined in
 
-[index.ts:482](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L482)
+[index.ts:396](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L396)
 
 ___
 
@@ -324,7 +324,7 @@ Refine the results by reading extra elements and re-ranking them in memory.
 
 #### Defined in
 
-[index.ts:473](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L473)
+[index.ts:387](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L387)
 
 ___
 
@@ -346,4 +346,4 @@ Return only the specified columns.
 
 #### Defined in
 
-[index.ts:502](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L502)
+[index.ts:416](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L416)

docs/src/javascript/enums/MetricType.md

@@ -22,7 +22,7 @@ Cosine distance
 
 #### Defined in
 
-[index.ts:567](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L567)
+[index.ts:481](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L481)
 
 ___
 
@@ -34,7 +34,7 @@ Dot product
 
 #### Defined in
 
-[index.ts:572](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L572)
+[index.ts:486](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L486)
 
 ___
 
@@ -46,4 +46,4 @@ Euclidean distance
 
 #### Defined in
 
-[index.ts:562](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L562)
+[index.ts:476](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L476)

docs/src/javascript/enums/WriteMode.md

@@ -22,7 +22,7 @@ Append new data to the table.
 
 #### Defined in
 
-[index.ts:552](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L552)
+[index.ts:466](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L466)
 
 ___
 
@@ -34,7 +34,7 @@ Create a new [Table](../interfaces/Table.md).
 
 #### Defined in
 
-[index.ts:548](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L548)
+[index.ts:462](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L462)
 
 ___
 
@@ -46,4 +46,4 @@ Overwrite the existing [Table](../interfaces/Table.md) if presented.
 
 #### Defined in
 
-[index.ts:550](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L550)
+[index.ts:464](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L464)

docs/src/javascript/interfaces/AwsCredentials.md

@@ -1,41 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / AwsCredentials
-
-# Interface: AwsCredentials
-
-## Table of contents
-
-### Properties
-
-- [accessKeyId](AwsCredentials.md#accesskeyid)
-- [secretKey](AwsCredentials.md#secretkey)
-- [sessionToken](AwsCredentials.md#sessiontoken)
-
-## Properties
-
-### accessKeyId
-
-• **accessKeyId**: `string`
-
-#### Defined in
-
-[index.ts:31](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L31)
-
-___
-
-### secretKey
-
-• **secretKey**: `string`
-
-#### Defined in
-
-[index.ts:33](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L33)
-
-___
-
-### sessionToken
-
-• `Optional` **sessionToken**: `string`
-
-#### Defined in
-
-[index.ts:35](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L35)

docs/src/javascript/interfaces/Connection.md

@@ -32,7 +32,7 @@ Connection could be local against filesystem or remote against a server.
 
 #### Defined in
 
-[index.ts:70](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L70)
+[index.ts:45](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L45)
 
 ## Methods
 
@@ -63,7 +63,7 @@ Creates a new Table and initialize it with new data.
 
 #### Defined in
 
-[index.ts:90](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L90)
+[index.ts:65](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L65)
 
 ___
 
@@ -84,7 +84,7 @@ ___
 
 #### Defined in
 
-[index.ts:92](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L92)
+[index.ts:67](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L67)
 
 ___
 
@@ -106,7 +106,7 @@ Drop an existing table.
 
 #### Defined in
 
-[index.ts:98](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L98)
+[index.ts:73](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L73)
 
 ___
 
@@ -135,7 +135,7 @@ Open a table in the database.
 
 #### Defined in
 
-[index.ts:80](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L80)
+[index.ts:55](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L55)
 
 ___
 
@@ -149,4 +149,4 @@ ___
 
 #### Defined in
 
-[index.ts:72](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L72)
+[index.ts:47](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L47)

docs/src/javascript/interfaces/ConnectionOptions.md

@@ -1,30 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / ConnectionOptions
-
-# Interface: ConnectionOptions
-
-## Table of contents
-
-### Properties
-
-- [awsCredentials](ConnectionOptions.md#awscredentials)
-- [uri](ConnectionOptions.md#uri)
-
-## Properties
-
-### awsCredentials
-
-• `Optional` **awsCredentials**: [`AwsCredentials`](AwsCredentials.md)
-
-#### Defined in
-
-[index.ts:40](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L40)
-
-___
-
-### uri
-
-• **uri**: `string`
-
-#### Defined in
-
-[index.ts:39](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L39)

docs/src/javascript/interfaces/EmbeddingFunction.md

@@ -45,7 +45,7 @@ Creates a vector representation for the given values.
 
 #### Defined in
 
-[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/embedding_function.ts#L27)
+[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/embedding_function.ts#L27)
 
 ___
 
@@ -57,4 +57,4 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/embedding_function.ts#L22)
+[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/embedding_function.ts#L22)

docs/src/javascript/interfaces/IvfPQIndexConfig.md

@@ -1,149 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / IvfPQIndexConfig
-
-# Interface: IvfPQIndexConfig
-
-## Table of contents
-
-### Properties
-
-- [column](IvfPQIndexConfig.md#column)
-- [index\_name](IvfPQIndexConfig.md#index_name)
-- [max\_iters](IvfPQIndexConfig.md#max_iters)
-- [max\_opq\_iters](IvfPQIndexConfig.md#max_opq_iters)
-- [metric\_type](IvfPQIndexConfig.md#metric_type)
-- [num\_bits](IvfPQIndexConfig.md#num_bits)
-- [num\_partitions](IvfPQIndexConfig.md#num_partitions)
-- [num\_sub\_vectors](IvfPQIndexConfig.md#num_sub_vectors)
-- [replace](IvfPQIndexConfig.md#replace)
-- [type](IvfPQIndexConfig.md#type)
-- [use\_opq](IvfPQIndexConfig.md#use_opq)
-
-## Properties
-
-### column
-
-• `Optional` **column**: `string`
-
-The column to be indexed
-
-#### Defined in
-
-[index.ts:382](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L382)
-
-___
-
-### index\_name
-
-• `Optional` **index\_name**: `string`
-
-A unique name for the index
-
-#### Defined in
-
-[index.ts:387](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L387)
-
-___
-
-### max\_iters
-
-• `Optional` **max\_iters**: `number`
-
-The max number of iterations for kmeans training.
-
-#### Defined in
-
-[index.ts:402](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L402)
-
-___
-
-### max\_opq\_iters
-
-• `Optional` **max\_opq\_iters**: `number`
-
-Max number of iterations to train OPQ, if `use_opq` is true.
-
-#### Defined in
-
-[index.ts:421](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L421)
-
-___
-
-### metric\_type
-
-• `Optional` **metric\_type**: [`MetricType`](../enums/MetricType.md)
-
-Metric type, L2 or Cosine
-
-#### Defined in
-
-[index.ts:392](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L392)
-
-___
-
-### num\_bits
-
-• `Optional` **num\_bits**: `number`
-
-The number of bits to present one PQ centroid.
-
-#### Defined in
-
-[index.ts:416](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L416)
-
-___
-
-### num\_partitions
-
-• `Optional` **num\_partitions**: `number`
-
-The number of partitions this index
-
-#### Defined in
-
-[index.ts:397](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L397)
-
-___
-
-### num\_sub\_vectors
-
-• `Optional` **num\_sub\_vectors**: `number`
-
-Number of subvectors to build PQ code
-
-#### Defined in
-
-[index.ts:412](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L412)
-
-___
-
-### replace
-
-• `Optional` **replace**: `boolean`
-
-Replace an existing index with the same name if it exists.
-
-#### Defined in
-
-[index.ts:426](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L426)
-
-___
-
-### type
-
-• **type**: ``"ivf_pq"``
-
-#### Defined in
-
-[index.ts:428](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L428)
-
-___
-
-### use\_opq
-
-• `Optional` **use\_opq**: `boolean`
-
-Train as optimized product quantization.
-
-#### Defined in
-
-[index.ts:407](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L407)

docs/src/javascript/interfaces/Table.md

@@ -52,7 +52,7 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:120](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L120)
+[index.ts:95](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L95)
 
 ___
 
@@ -72,13 +72,13 @@ Returns the number of rows in this table.
 
 #### Defined in
 
-[index.ts:140](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L140)
+[index.ts:115](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L115)
 
 ___
 
 ### createIndex
 
-• **createIndex**: (`indexParams`: [`IvfPQIndexConfig`](IvfPQIndexConfig.md)) => `Promise`<`any`\>
+• **createIndex**: (`indexParams`: `IvfPQIndexConfig`) => `Promise`<`any`\>
 
 #### Type declaration
 
@@ -94,7 +94,7 @@ VectorIndexParams.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `indexParams` | [`IvfPQIndexConfig`](IvfPQIndexConfig.md) | The parameters of this Index, |
+| `indexParams` | `IvfPQIndexConfig` | The parameters of this Index, |
 
 ##### Returns
 
@@ -102,7 +102,7 @@ VectorIndexParams.
 
 #### Defined in
 
-[index.ts:135](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L135)
+[index.ts:110](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L110)
 
 ___
 
@@ -116,37 +116,11 @@ ___
 
 Delete rows from this table.
 
-This can be used to delete a single row, many rows, all rows, or
-sometimes no rows (if your predicate matches nothing).
-
-**`Examples`**
-
-```ts
-const con = await lancedb.connect("./.lancedb")
-const data = [
-   {id: 1, vector: [1, 2]},
-   {id: 2, vector: [3, 4]},
-   {id: 3, vector: [5, 6]},
-];
-const tbl = await con.createTable("my_table", data)
-await tbl.delete("id = 2")
-await tbl.countRows() // Returns 2
-```
-
-If you have a list of values to delete, you can combine them into a
-stringified list and use the `IN` operator:
-
-```ts
-const to_remove = [1, 5];
-await tbl.delete(`id IN (${to_remove.join(",")})`)
-await tbl.countRows() // Returns 1
-```
-
 ##### Parameters
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `filter` | `string` | A filter in the same format used by a sql WHERE clause. The filter must not be empty. |
+| `filter` | `string` | A filter in the same format used by a sql WHERE clause. |
 
 ##### Returns
 
@@ -154,7 +128,7 @@ await tbl.countRows() // Returns 1
 
 #### Defined in
 
-[index.ts:174](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L174)
+[index.ts:122](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L122)
 
 ___
 
@@ -164,7 +138,7 @@ ___
 
 #### Defined in
 
-[index.ts:106](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L106)
+[index.ts:81](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L81)
 
 ___
 
@@ -192,7 +166,7 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:128](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L128)
+[index.ts:103](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L103)
 
 ___
 
@@ -218,4 +192,4 @@ Creates a search query to find the nearest neighbors of the given search term
 
 #### Defined in
 
-[index.ts:112](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L112)
+[index.ts:87](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L87)

docs/src/javascript/modules.md

@@ -18,11 +18,8 @@
 
 ### Interfaces
 
-- [AwsCredentials](interfaces/AwsCredentials.md)
 - [Connection](interfaces/Connection.md)
-- [ConnectionOptions](interfaces/ConnectionOptions.md)
 - [EmbeddingFunction](interfaces/EmbeddingFunction.md)
-- [IvfPQIndexConfig](interfaces/IvfPQIndexConfig.md)
 - [Table](interfaces/Table.md)
 
 ### Type Aliases
@@ -37,11 +34,11 @@
 
 ### VectorIndexParams
 
-Ƭ **VectorIndexParams**: [`IvfPQIndexConfig`](interfaces/IvfPQIndexConfig.md)
+Ƭ **VectorIndexParams**: `IvfPQIndexConfig`
 
 #### Defined in
 
-[index.ts:431](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L431)
+[index.ts:345](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L345)
 
 ## Functions
 
@@ -63,20 +60,4 @@ Connect to a LanceDB instance at the given URI
 
 #### Defined in
 
-[index.ts:47](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L47)
-
-▸ **connect**(`opts`): `Promise`<[`Connection`](interfaces/Connection.md)\>
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `opts` | `Partial`<[`ConnectionOptions`](interfaces/ConnectionOptions.md)\> |
-
-#### Returns
-
-`Promise`<[`Connection`](interfaces/Connection.md)\>
-
-#### Defined in
-
-[index.ts:48](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L48)
+[index.ts:34](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L34)

docs/src/notebooks/code_qa_bot.ipynb

@@ -10,11 +10,7 @@
     "\n",
     "This Q&A bot will allow you to query your own documentation easily using questions. We'll also demonstrate the use of LangChain and LanceDB using the OpenAI API. \n",
     "\n",
-    "In this example we'll use Pandas 2.0 documentation, but, this could be replaced for your own docs as well\n",
-    "\n",
-    "<a href=\"https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Code-Documentation-QA-Bot/main.ipynb\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"></a>\n",
-    "\n",
-    "Scripts -  [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](./examples/Code-Documentation-QA-Bot/main.py)  [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](./examples/Code-Documentation-QA-Bot/index.js)"
+    "In this example we'll use Pandas 2.0 documentation, but, this could be replaced for your own docs as well"
    ]
   },
   {
@@ -144,7 +140,7 @@
    "source": [
     "# Pre-processing and loading the documentation\n",
     "\n",
-    "Next, let's pre-process and load the documentation. To make sure we don't need to do this repeatedly if we were updating code, we're caching it using pickle so we can retrieve it again (this could take a few minutes to run the first time you do it). We'll also add some more metadata to the docs here such as the title and version of the code:"
+    "Next, let's pre-process and load the documentation. To make sure we don't need to do this repeatedly if we were updating code, we're caching it using pickle so we can retrieve it again (this could take a few minutes to run the first time yyou do it). We'll also add some more metadata to the docs here such as the title and version of the code:"
    ]
   },
   {
@@ -185,7 +181,7 @@
    "id": "c3852dd3",
    "metadata": {},
    "source": [
-    "# Generating embeddings from our docs\n",
+    "# Generating emebeddings from our docs\n",
     "\n",
     "Now that we have our raw documents loaded, we need to pre-process them to generate embeddings:"
    ]
@@ -255,7 +251,7 @@
    "id": "28d93b85",
    "metadata": {},
    "source": [
-    "And that's it! We're all set up. The next step is to run some queries, let's try a few:"
+    "And thats it! We're all setup. The next step is to run some queries, let's try a few:"
    ]
   },
   {

docs/src/notebooks/multimodal_search.ipynb

@@ -1,14 +1,5 @@
 {
  "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "![example](https://github.com/lancedb/vectordb-recipes/assets/15766192/799f94a1-a01d-4a5b-a627-2a733bbb4227)\n",
-    "\n",
-    " <a href=\"https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip/main.ipynb\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](./examples/multimodal_clip/main.py)  |"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": 2,
@@ -19,11 +10,11 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip available: \u001B[0m\u001B[31;49m22.3.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m23.1.2\u001B[0m\n",
-      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpip install --upgrade pip\u001B[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
       "\n",
-      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip available: \u001B[0m\u001B[31;49m22.3.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m23.1.2\u001B[0m\n",
-      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpip install --upgrade pip\u001B[0m\n"
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
      ]
     }
    ],
@@ -39,7 +30,6 @@
    "outputs": [],
    "source": [
     "import io\n",
-    "\n",
     "import PIL\n",
     "import duckdb\n",
     "import lancedb"
@@ -52,19 +42,6 @@
     "## First run setup: Download data and pre-process"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "### Get dataset\n",
-    "\n",
-    "!wget https://eto-public.s3.us-west-2.amazonaws.com/datasets/diffusiondb_lance.tar.gz\n",
-    "!tar -xvf diffusiondb_lance.tar.gz\n",
-    "!mv diffusiondb_test rawdata.lance\n"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": 30,
@@ -159,18 +136,18 @@
     "        \"db = lancedb.connect('~/datasets/demo')\\n\"\n",
     "        \"tbl = db.open_table('diffusiondb')\\n\\n\"\n",
     "        f\"embedding = embed_func('{query}')\\n\"\n",
-    "        \"tbl.search(embedding).limit(9).to_pandas()\"\n",
+    "        \"tbl.search(embedding).limit(9).to_df()\"\n",
     "    )\n",
-    "    return (_extract(tbl.search(emb).limit(9).to_pandas()), code)\n",
+    "    return (_extract(tbl.search(emb).limit(9).to_df()), code)\n",
     "\n",
     "def find_image_keywords(query):\n",
     "    code = (\n",
     "        \"import lancedb\\n\"\n",
     "        \"db = lancedb.connect('~/datasets/demo')\\n\"\n",
     "        \"tbl = db.open_table('diffusiondb')\\n\\n\"\n",
-    "        f\"tbl.search('{query}').limit(9).to_pandas()\"\n",
+    "        f\"tbl.search('{query}').limit(9).to_df()\"\n",
     "    )\n",
-    "    return (_extract(tbl.search(query).limit(9).to_pandas()), code)\n",
+    "    return (_extract(tbl.search(query).limit(9).to_df()), code)\n",
     "\n",
     "def find_image_sql(query):\n",
     "    code = (\n",
@@ -270,7 +247,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3.11.4 64-bit",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -284,12 +261,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.4"
-  },
-  "vscode": {
-   "interpreter": {
-    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
-   }
+   "version": "3.11.3"
   }
  },
  "nbformat": 4,

docs/src/notebooks/tables_guide.ipynb

@@ -1,831 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "d24eb4c6-e246-44ca-ba7c-6eae7923bd4c",
-   "metadata": {},
-   "source": [
-    "## LanceDB Tables\n",
-    "A Table is a collection of Records in a LanceDB Database.\n",
-    "\n",
-    "![illustration](../assets/ecosystem-illustration.png)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 50,
-   "id": "c1b4e34b-a49c-471d-a343-a5940bb5138a",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "!pip install lancedb -qq"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "4e5a8d07-d9a1-48c1-913a-8e0629289579",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import lancedb\n",
-    "db = lancedb.connect(\"./.lancedb\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "66fb93d5-3551-406b-99b2-488442d61d06",
-   "metadata": {},
-   "source": [
-    "LanceDB allows ingesting data from various sources - `dict`, `list[dict]`, `pd.DataFrame`, `pa.Table` or a `Iterator[pa.RecordBatch]`. Let's take a look at some of the these.\n",
-    "\n",
-    " ### From list of tuples or dictionaries"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "5df12f66-8d99-43ad-8d0b-22189ec0a6b9",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "pyarrow.Table\n",
-       "vector: fixed_size_list<item: float>[2]\n",
-       "  child 0, item: float\n",
-       "lat: double\n",
-       "long: double\n",
-       "----\n",
-       "vector: [[[1.1,1.2],[0.2,1.8]]]\n",
-       "lat: [[45.5,40.1]]\n",
-       "long: [[-122.7,-74.1]]"
-      ]
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import lancedb\n",
-    "\n",
-    "db = lancedb.connect(\"./.lancedb\")\n",
-    "\n",
-    "data = [{\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7},\n",
-    "        {\"vector\": [0.2, 1.8], \"lat\": 40.1, \"long\": -74.1}]\n",
-    "\n",
-    "db.create_table(\"my_table\", data)\n",
-    "\n",
-    "db[\"my_table\"].head()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "10ce802f-1a10-49ee-8ee3-a9bfb302d86c",
-   "metadata": {},
-   "source": [
-    "## From pandas DataFrame\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "f4d87ae9-0ccb-48eb-b31d-bb8f2370e47e",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "pyarrow.Table\n",
-       "vector: fixed_size_list<item: float>[2]\n",
-       "  child 0, item: float\n",
-       "lat: double\n",
-       "long: double\n",
-       "----\n",
-       "vector: [[[1.1,1.2],[0.2,1.8]]]\n",
-       "lat: [[45.5,40.1]]\n",
-       "long: [[-122.7,-74.1]]"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import pandas as pd\n",
-    "\n",
-    "data = pd.DataFrame({\n",
-    "    \"vector\": [[1.1, 1.2], [0.2, 1.8]],\n",
-    "    \"lat\": [45.5, 40.1],\n",
-    "    \"long\": [-122.7, -74.1]\n",
-    "})\n",
-    "\n",
-    "db.create_table(\"table2\", data)\n",
-    "\n",
-    "db[\"table2\"].head() "
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "4be81469-5b57-4f78-9c72-3938c0378d9d",
-   "metadata": {},
-   "source": [
-    "Data is converted to Arrow before being written to disk. For maximum control over how data is saved, either provide the PyArrow schema to convert to or else provide a PyArrow Table directly.\n",
-    "  "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "id": "25f34bcf-fca0-4431-8601-eac95d1bd347",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "vector: fixed_size_list<item: float>[2]\n",
-       "  child 0, item: float\n",
-       "lat: float\n",
-       "long: float"
-      ]
-     },
-     "execution_count": 8,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import pyarrow as pa\n",
-    "\n",
-    "custom_schema = pa.schema([\n",
-    "pa.field(\"vector\", pa.list_(pa.float32(), 2)),\n",
-    "pa.field(\"lat\", pa.float32()),\n",
-    "pa.field(\"long\", pa.float32())\n",
-    "])\n",
-    "\n",
-    "table = db.create_table(\"table3\", data, schema=custom_schema, mode=\"overwrite\")\n",
-    "table.schema"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "4df51925-7ca2-4005-9c72-38b3d26240c6",
-   "metadata": {},
-   "source": [
-    "### From PyArrow Tables\n",
-    "\n",
-    "You can also create LanceDB tables directly from pyarrow tables"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "id": "90a880f6-be43-4c9d-ba65-0b05197c0f6f",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "vector: fixed_size_list<item: float>[2]\n",
-       "  child 0, item: float\n",
-       "item: string\n",
-       "price: double"
-      ]
-     },
-     "execution_count": 12,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "table = pa.Table.from_arrays(\n",
-    "        [\n",
-    "            pa.array([[3.1, 4.1], [5.9, 26.5]],\n",
-    "                    pa.list_(pa.float32(), 2)),\n",
-    "            pa.array([\"foo\", \"bar\"]),\n",
-    "            pa.array([10.0, 20.0]),\n",
-    "        ],\n",
-    "        [\"vector\", \"item\", \"price\"],\n",
-    "    )\n",
-    "\n",
-    "db = lancedb.connect(\"db\")\n",
-    "\n",
-    "tbl = db.create_table(\"test1\", table, mode=\"overwrite\")\n",
-    "tbl.schema"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "0f36c51c-d902-449d-8292-700e53990c32",
-   "metadata": {},
-   "source": [
-    "### From Pydantic Models\n",
-    "\n",
-    "LanceDB supports to create Apache Arrow Schema from a Pydantic BaseModel."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "id": "d81121d7-e4b7-447c-a48c-974b6ebb464a",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "movie_id: int64 not null\n",
-       "vector: fixed_size_list<item: float>[128] not null\n",
-       "  child 0, item: float\n",
-       "genres: string not null\n",
-       "title: string not null\n",
-       "imdb_id: int64 not null"
-      ]
-     },
-     "execution_count": 13,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "from lancedb.pydantic import Vector, LanceModel\n",
-    "\n",
-    "class Content(LanceModel):\n",
-    "    movie_id: int\n",
-    "    vector: Vector(128)\n",
-    "    genres: str\n",
-    "    title: str\n",
-    "    imdb_id: int\n",
-    "        \n",
-    "    @property\n",
-    "    def imdb_url(self) -> str:\n",
-    "        return f\"https://www.imdb.com/title/tt{self.imdb_id}\"\n",
-    "\n",
-    "import pyarrow as pa\n",
-    "db = lancedb.connect(\"~/.lancedb\")\n",
-    "table_name = \"movielens_small\"\n",
-    "table = db.create_table(table_name, schema=Content)\n",
-    "table.schema"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "860e1f77-e860-46a9-98b7-b2979092ccd6",
-   "metadata": {},
-   "source": [
-    "### Using Iterators / Writing Large Datasets\n",
-    "\n",
-    "It is recommended to use itertators to add large datasets in batches when creating your table in one go. This does not create multiple versions of your dataset unlike manually adding batches using `table.add()`\n",
-    "\n",
-    "LanceDB additionally supports pyarrow's `RecordBatch` Iterators or other generators producing supported data types.\n",
-    "\n",
-    "## Here's an example using using `RecordBatch` iterator for creating tables."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 14,
-   "id": "bc247142-4e3c-41a2-b94c-8e00d2c2a508",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "LanceTable(table4)"
-      ]
-     },
-     "execution_count": 14,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import pyarrow as pa\n",
-    "\n",
-    "def make_batches():\n",
-    "    for i in range(5):\n",
-    "        yield pa.RecordBatch.from_arrays(\n",
-    "            [\n",
-    "                pa.array([[3.1, 4.1], [5.9, 26.5]],\n",
-    "                        pa.list_(pa.float32(), 2)),\n",
-    "                pa.array([\"foo\", \"bar\"]),\n",
-    "                pa.array([10.0, 20.0]),\n",
-    "            ],\n",
-    "            [\"vector\", \"item\", \"price\"],\n",
-    "        )\n",
-    "\n",
-    "schema = pa.schema([\n",
-    "    pa.field(\"vector\", pa.list_(pa.float32(), 2)),\n",
-    "    pa.field(\"item\", pa.utf8()),\n",
-    "    pa.field(\"price\", pa.float32()),\n",
-    "])\n",
-    "\n",
-    "db.create_table(\"table4\", make_batches(), schema=schema)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "94f7dd2b-bae4-4bdf-8534-201437c31027",
-   "metadata": {},
-   "source": [
-    "### Using pandas `DataFrame` Iterator and Pydantic Schema\n",
-    "\n",
-    "You can set the schema via pyarrow schema object or using Pydantic object"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 16,
-   "id": "25ad3523-e0c9-4c28-b3df-38189c4e0e5f",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "vector: fixed_size_list<item: float>[2] not null\n",
-       "  child 0, item: float\n",
-       "item: string not null\n",
-       "price: double not null"
-      ]
-     },
-     "execution_count": 16,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import pyarrow as pa\n",
-    "import pandas as pd\n",
-    "\n",
-    "class PydanticSchema(LanceModel):\n",
-    "    vector: Vector(2)\n",
-    "    item: str\n",
-    "    price: float\n",
-    "\n",
-    "def make_batches():\n",
-    "    for i in range(5):\n",
-    "        yield pd.DataFrame(\n",
-    "                    {\n",
-    "                        \"vector\": [[3.1, 4.1], [1, 1]],\n",
-    "                        \"item\": [\"foo\", \"bar\"],\n",
-    "                        \"price\": [10.0, 20.0],\n",
-    "                    })\n",
-    "\n",
-    "tbl = db.create_table(\"table5\", make_batches(), schema=PydanticSchema)\n",
-    "tbl.schema"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "4aa955e9-fcd0-4c99-b644-f218f3bb3f1a",
-   "metadata": {},
-   "source": [
-    "## Creating Empty Table\n",
-    "\n",
-    "You can create an empty table by just passing the schema and later add to it using `table.add()`"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 17,
-   "id": "2814173a-eacc-4dd8-a64d-6312b44582cc",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import lancedb\n",
-    "from lancedb.pydantic import LanceModel, Vector\n",
-    "\n",
-    "class Model(LanceModel):\n",
-    "      vector: Vector(2)\n",
-    "\n",
-    "tbl = db.create_table(\"table6\", schema=Model.to_arrow_schema())"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "1d1b0f5c-a1d9-459f-8614-8376b6f577e1",
-   "metadata": {},
-   "source": [
-    "## Open Existing Tables\n",
-    "\n",
-    "If you forget the name of your table, you can always get a listing of all table names:\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 18,
-   "id": "df9e13c0-41f6-437f-9dfa-2fd71d3d9c45",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "['table6', 'table4', 'table5', 'movielens_small']"
-      ]
-     },
-     "execution_count": 18,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "db.table_names()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 20,
-   "id": "9343f5ad-6024-42ee-ac2f-6c1471df8679",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>vector</th>\n",
-       "      <th>item</th>\n",
-       "      <th>price</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>[3.1, 4.1]</td>\n",
-       "      <td>foo</td>\n",
-       "      <td>10.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>[5.9, 26.5]</td>\n",
-       "      <td>bar</td>\n",
-       "      <td>20.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>[3.1, 4.1]</td>\n",
-       "      <td>foo</td>\n",
-       "      <td>10.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>[5.9, 26.5]</td>\n",
-       "      <td>bar</td>\n",
-       "      <td>20.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>[3.1, 4.1]</td>\n",
-       "      <td>foo</td>\n",
-       "      <td>10.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>5</th>\n",
-       "      <td>[5.9, 26.5]</td>\n",
-       "      <td>bar</td>\n",
-       "      <td>20.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>6</th>\n",
-       "      <td>[3.1, 4.1]</td>\n",
-       "      <td>foo</td>\n",
-       "      <td>10.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>7</th>\n",
-       "      <td>[5.9, 26.5]</td>\n",
-       "      <td>bar</td>\n",
-       "      <td>20.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>8</th>\n",
-       "      <td>[3.1, 4.1]</td>\n",
-       "      <td>foo</td>\n",
-       "      <td>10.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9</th>\n",
-       "      <td>[5.9, 26.5]</td>\n",
-       "      <td>bar</td>\n",
-       "      <td>20.0</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "        vector item  price\n",
-       "0   [3.1, 4.1]  foo   10.0\n",
-       "1  [5.9, 26.5]  bar   20.0\n",
-       "2   [3.1, 4.1]  foo   10.0\n",
-       "3  [5.9, 26.5]  bar   20.0\n",
-       "4   [3.1, 4.1]  foo   10.0\n",
-       "5  [5.9, 26.5]  bar   20.0\n",
-       "6   [3.1, 4.1]  foo   10.0\n",
-       "7  [5.9, 26.5]  bar   20.0\n",
-       "8   [3.1, 4.1]  foo   10.0\n",
-       "9  [5.9, 26.5]  bar   20.0"
-      ]
-     },
-     "execution_count": 20,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "tbl = db.open_table(\"table4\")\n",
-    "tbl.to_pandas()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "5019246f-12e3-4f78-88a8-9f4939802c76",
-   "metadata": {},
-   "source": [
-    "## Adding to table\n",
-    "After a table has been created, you can always add more data to it using\n",
-    "\n",
-    "You can add any of the valid data structures accepted by LanceDB table, i.e, `dict`, `list[dict]`, `pd.DataFrame`, or a `Iterator[pa.RecordBatch]`. Here are some examples."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 21,
-   "id": "8a56250f-73a1-4c26-a6ad-5c7a0ce3a9ab",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df = pd.DataFrame([{\"vector\": [1.3, 1.4], \"item\": \"fizz\", \"price\": 100.0},\n",
-    "              {\"vector\": [9.5, 56.2], \"item\": \"buzz\", \"price\": 200.0}])\n",
-    "tbl.add(df)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "9985f6ee-67e1-45a9-b233-94e3d121ecbf",
-   "metadata": {},
-   "source": [
-    "You can also add a large dataset batch in one go using Iterator of supported data types\n",
-    "\n",
-    "### Adding via Iterator\n",
-    "\n",
-    "here, we'll use pandas DataFrame Iterator"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 22,
-   "id": "030c7057-b98e-4e2f-be14-b8c1f927f83c",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "\n",
-    "import pandas as pd\n",
-    "\n",
-    "def make_batches():\n",
-    "    for i in range(5):\n",
-    "        yield pd.DataFrame(\n",
-    "                {\n",
-    "                    \"vector\": [[3.1, 4.1], [1, 1]],\n",
-    "                    \"item\": [\"foo\", \"bar\"],\n",
-    "                    \"price\": [10.0, 20.0],\n",
-    "                })\n",
-    "tbl.add(make_batches())"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "b8316d5d-0a23-4675-b0ee-178711db873a",
-   "metadata": {},
-   "source": [
-    "## Deleting from a Table\n",
-    "\n",
-    "Use the `delete()` method on tables to delete rows from a table. To choose which rows to delete, provide a filter that matches on the metadata columns. This can delete any number of rows that match the filter, like:\n",
-    "\n",
-    "\n",
-    "```python\n",
-    "tbl.delete('item = \"fizz\"')\n",
-    "```\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 24,
-   "id": "e7a17de2-08d2-41b7-bd05-f63d1045ab1f",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "32\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "17"
-      ]
-     },
-     "execution_count": 24,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "print(len(tbl))\n",
-    "      \n",
-    "tbl.delete(\"price = 20.0\")\n",
-    "      \n",
-    "len(tbl)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "74ac180b-5432-4c14-b1a8-22c35ac83af8",
-   "metadata": {},
-   "source": [
-    "### Delete from a list of values"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 30,
-   "id": "fe3310bd-08f4-4a22-a63b-b3127d22f9f7",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "         vector  item  price\n",
-      "0    [3.1, 4.1]   foo   10.0\n",
-      "1    [3.1, 4.1]   foo   10.0\n",
-      "2    [3.1, 4.1]   foo   10.0\n",
-      "3    [3.1, 4.1]   foo   10.0\n",
-      "4    [3.1, 4.1]   foo   10.0\n",
-      "5    [1.3, 1.4]  fizz  100.0\n",
-      "6   [9.5, 56.2]  buzz  200.0\n",
-      "7    [3.1, 4.1]   foo   10.0\n",
-      "8    [3.1, 4.1]   foo   10.0\n",
-      "9    [3.1, 4.1]   foo   10.0\n",
-      "10   [3.1, 4.1]   foo   10.0\n",
-      "11   [3.1, 4.1]   foo   10.0\n",
-      "12   [3.1, 4.1]   foo   10.0\n",
-      "13   [3.1, 4.1]   foo   10.0\n",
-      "14   [3.1, 4.1]   foo   10.0\n",
-      "15   [3.1, 4.1]   foo   10.0\n",
-      "16   [3.1, 4.1]   foo   10.0\n"
-     ]
-    },
-    {
-     "ename": "OSError",
-     "evalue": "LanceError(IO): Error during planning: column foo does not exist",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn[30], line 4\u001b[0m\n\u001b[1;32m      2\u001b[0m to_remove \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mstr\u001b[39m(v) \u001b[38;5;28;01mfor\u001b[39;00m v \u001b[38;5;129;01min\u001b[39;00m to_remove)\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28mprint\u001b[39m(tbl\u001b[38;5;241m.\u001b[39mto_pandas())\n\u001b[0;32m----> 4\u001b[0m \u001b[43mtbl\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mitem IN (\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mto_remove\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m)\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      5\u001b[0m tbl\u001b[38;5;241m.\u001b[39mto_pandas()\n",
-      "File \u001b[0;32m~/Documents/lancedb/lancedb/python/lancedb/table.py:610\u001b[0m, in \u001b[0;36mLanceTable.delete\u001b[0;34m(self, where)\u001b[0m\n\u001b[1;32m    609\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdelete\u001b[39m(\u001b[38;5;28mself\u001b[39m, where: \u001b[38;5;28mstr\u001b[39m):\n\u001b[0;32m--> 610\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dataset\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete\u001b[49m\u001b[43m(\u001b[49m\u001b[43mwhere\u001b[49m\u001b[43m)\u001b[49m\n",
-      "File \u001b[0;32m~/Documents/lancedb/lancedb/env/lib/python3.11/site-packages/lance/dataset.py:489\u001b[0m, in \u001b[0;36mLanceDataset.delete\u001b[0;34m(self, predicate)\u001b[0m\n\u001b[1;32m    487\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(predicate, pa\u001b[38;5;241m.\u001b[39mcompute\u001b[38;5;241m.\u001b[39mExpression):\n\u001b[1;32m    488\u001b[0m     predicate \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(predicate)\n\u001b[0;32m--> 489\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_ds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpredicate\u001b[49m\u001b[43m)\u001b[49m\n",
-      "\u001b[0;31mOSError\u001b[0m: LanceError(IO): Error during planning: column foo does not exist"
-     ]
-    }
-   ],
-   "source": [
-    "to_remove = [\"foo\", \"buzz\"]\n",
-    "to_remove = \", \".join(str(v) for v in to_remove)\n",
-    "print(tbl.to_pandas())\n",
-    "tbl.delete(f\"item IN ({to_remove})\")\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 43,
-   "id": "87d5bc21-847f-4c81-b56e-f6dbe5d05aac",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df = pd.DataFrame(\n",
-    "                    {\n",
-    "                        \"vector\": [[3.1, 4.1], [1, 1]],\n",
-    "                        \"item\": [\"foo\", \"bar\"],\n",
-    "                        \"price\": [10.0, 20.0],\n",
-    "                    })\n",
-    "\n",
-    "tbl = db.create_table(\"table7\", data=df, mode=\"overwrite\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 44,
-   "id": "9cba4519-eb3a-4941-ab7e-873d762e750f",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "to_remove = [10.0, 20.0]\n",
-    "to_remove = \", \".join(str(v) for v in to_remove)\n",
-    "\n",
-    "tbl.delete(f\"price IN ({to_remove})\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 46,
-   "id": "5bdc9801-d5ed-4871-92d0-88b27108e788",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>vector</th>\n",
-       "      <th>item</th>\n",
-       "      <th>price</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "Empty DataFrame\n",
-       "Columns: [vector, item, price]\n",
-       "Index: []"
-      ]
-     },
-     "execution_count": 46,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "tbl.to_pandas()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "752d33d4-ce1c-48e5-90d2-c85f0982182d",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.4"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

docs/src/notebooks/youtube_transcript_search.ipynb

@@ -8,12 +8,7 @@
    "source": [
     "# Youtube Transcript Search QA Bot\n",
     "\n",
-    "This Q&A bot will allow you to search through youtube transcripts using natural language! By going through this notebook, we'll introduce how you can use LanceDB to store and manage your data easily.\n",
-    "\n",
-    "\n",
-    "<a href=\"https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/youtube_bot/main.ipynb\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\">\n",
-    "\n",
-    "Scripts - [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](./examples/youtube_bot/main.py)  [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](./examples/youtube_bot/index.js)\n"
+    "This Q&A bot will allow you to search through youtube transcripts using natural language! By going through this notebook, we'll introduce how you can use LanceDB to store and manage your data easily."
    ]
   },
   {
@@ -27,11 +22,11 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.0\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m23.1.1\u001B[0m\n",
-      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpip install --upgrade pip\u001B[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.1\u001b[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
       "\n",
-      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.0\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m23.1.1\u001B[0m\n",
-      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpip install --upgrade pip\u001B[0m\n"
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.1\u001b[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
      ]
     }
    ],
@@ -184,7 +179,7 @@
     "df = (contextualize(data.to_pandas())\n",
     "      .groupby(\"title\").text_col(\"text\")\n",
     "      .window(20).stride(4)\n",
-    "      .to_pandas())\n",
+    "      .to_df())\n",
     "df.head(1)"
    ]
   },
@@ -603,7 +598,7 @@
    "outputs": [],
    "source": [
     "# Use LanceDB to get top 3 most relevant context\n",
-    "context = tbl.search(emb).limit(3).to_pandas()"
+    "context = tbl.search(emb).limit(3).to_df()"
    ]
   },
   {

docs/src/python/arrow.md

@@ -74,12 +74,12 @@ table = db.open_table("pd_table")
 
 query_vector = [100, 100]
 # Pandas DataFrame
-df = table.search(query_vector).limit(1).to_pandas()
+df = table.search(query_vector).limit(1).to_df()
 print(df)
 ```
 
 ```
-    vector     item  price    _distance
+    vector     item  price        score
 0  [5.9, 26.5]  bar   20.0  14257.05957
 ```
 
@@ -89,12 +89,12 @@ If you have more complex criteria, you can always apply the filter to the result
 ```python
 
 # Apply the filter via LanceDB
-results = table.search([100, 100]).where("price < 15").to_pandas()
+results = table.search([100, 100]).where("price < 15").to_df()
 assert len(results) == 1
 assert results["item"].iloc[0] == "foo"
 
 # Apply the filter via Pandas
-df = results = table.search([100, 100]).to_pandas()
+df = results = table.search([100, 100]).to_df()
 results = df[df.price < 15]
 assert len(results) == 1
 assert results["item"].iloc[0] == "foo"

docs/src/python/pydantic.md

@@ -1,7 +1,6 @@
 # Pydantic
 
 [Pydantic](https://docs.pydantic.dev/latest/) is a data validation library in Python.
-LanceDB integrates with Pydantic for schema inference, data ingestion, and query result casting.
 
 ## Schema
 
@@ -13,10 +12,10 @@ via [pydantic_to_schema()](python.md##lancedb.pydantic.pydantic_to_schema) metho
 
 ## Vector Field
 
-LanceDB provides a [`Vector(dim)`](python.md#lancedb.pydantic.Vector) method to define a
+LanceDB provides a [`vector(dim)`](python.md#lancedb.pydantic.vector) method to define a
 vector Field in a Pydantic Model.
 
-::: lancedb.pydantic.Vector
+::: lancedb.pydantic.vector
 
 ## Type Conversion
 
@@ -33,4 +32,4 @@ Current supported type conversions:
 | `str`               | `pyarrow.utf8()`    |
 | `list`              | `pyarrow.List`    |
 | `BaseModel`         | `pyarrow.Struct`    |
-| `Vector(n)`         | `pyarrow.FixedSizeList(float32, n)` |
+| `vector(n)`         | `pyarrow.FixedSizeList(float32, n)` |

docs/src/python/python.md

@@ -26,20 +26,10 @@ pip install lancedb
 
 ## Embeddings
 
-::: lancedb.embeddings.functions.EmbeddingFunctionRegistry
-
-::: lancedb.embeddings.functions.EmbeddingFunction
-
-::: lancedb.embeddings.functions.TextEmbeddingFunction
-
-::: lancedb.embeddings.functions.SentenceTransformerEmbeddings
-
-::: lancedb.embeddings.functions.OpenAIEmbeddings
-
-::: lancedb.embeddings.functions.OpenClipEmbeddings
-
 ::: lancedb.embeddings.with_embeddings
 
+::: lancedb.embeddings.EmbeddingFunction
+
 ## Context
 
 ::: lancedb.context.contextualize
@@ -66,4 +56,4 @@ pip install lancedb
 
 ::: lancedb.pydantic.vector
 
-::: lancedb.pydantic.LanceModel
+

docs/src/search.md

@@ -25,8 +25,8 @@ Currently, we support the following metrics:
 
 ### Flat Search
 
-If you do not create a vector index, LanceDB would need to exhaustively scan the entire vector column (via `Flat Search`)
-and compute the distance for *every* vector in order to find the closest matches. This is effectively a KNN search.
+If LanceDB does not create a vector index, LanceDB would need to scan (`Flat Search`) the entire vector column
+and compute the distance for each vector in order to find the closest matches.
 
 
 <!-- Setup Code
@@ -67,7 +67,7 @@ await db_setup.createTable('my_vectors', data)
 
     df = tbl.search(np.random.random((1536))) \
         .limit(10) \
-        .to_list()
+        .to_df()
     ```
 
 === "JavaScript"
@@ -92,7 +92,7 @@ as well.
     df = tbl.search(np.random.random((1536))) \
         .metric("cosine") \
         .limit(10) \
-        .to_list()
+        .to_df()
     ```
 
 
@@ -110,7 +110,7 @@ as well.
 
 To accelerate vector retrievals, it is common to build vector indices.
 A vector index is a data structure specifically designed to efficiently organize and
-search vector data based on their similarity via the chosen distance metric.
+search vector data based on their similarity or distance metrics.
 By constructing a vector index, you can reduce the search space and avoid the need
 for brute-force scanning of the entire vector column.
 

docs/src/styles/global.css

@@ -4,12 +4,3 @@
     --md-text-font: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
     --md-code-font: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
 }
-
-.md-nav__item, .md-tabs__item {
-    font-size: large;
-}
-
-/* Maximum space for text block */
-.md-grid {
-    max-width: 90%;
-  }

docs/test/md_testing.js

@@ -2,17 +2,18 @@ const glob = require("glob");
 const fs = require("fs");
 const path = require("path");
 
-const globString = "../src/**/*.md";
-const excludedGlobs = [
+const excludedFiles = [
   "../src/fts.md",
   "../src/embedding.md",
-  "../src/examples/*.md",
-  "../src/guides/tables.md",
+  "../src/examples/serverless_lancedb_with_s3_and_lambda.md",
+  "../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
+  "../src/examples/transformerjs_embedding_search_nodejs.md",
+  "../src/examples/youtube_transcript_bot_with_nodejs.md",
 ];
-
 const nodePrefix = "javascript";
 const nodeFile = ".js";
 const nodeFolder = "node";
+const globString = "../src/**/*.md";
 const asyncPrefix = "(async () => {\n";
 const asyncSuffix = "})();";
 
@@ -31,7 +32,6 @@ function* yieldLines(lines, prefix, suffix) {
 }
 
 const files = glob.sync(globString, { recursive: true });
-const excludedFiles = glob.sync(excludedGlobs, { recursive: true });
 
 for (const file of files.filter((file) => !excludedFiles.includes(file))) {
   const lines = [];

docs/test/md_testing.py

@@ -2,22 +2,18 @@ import glob
 from typing import Iterator
 from pathlib import Path
 
-glob_string = "../src/**/*.md"
-excluded_globs = [
+excluded_files = [
     "../src/fts.md",
     "../src/embedding.md",
-    "../src/examples/*.md",
-    "../src/integrations/voxel51.md",
-    "../src/guides/tables.md",
-    "../src/python/duckdb.md",
+    "../src/examples/serverless_lancedb_with_s3_and_lambda.md",
+    "../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
+    "../src/examples/youtube_transcript_bot_with_nodejs.md"
 ]
 
 python_prefix = "py"
 python_file = ".py"
 python_folder = "python"
-
-files = glob.glob(glob_string, recursive=True)
-excluded_files = [f for excluded_glob in excluded_globs for f in glob.glob(excluded_glob, recursive=True)]
+glob_string = "../src/**/*.md"
 
 def yield_lines(lines: Iterator[str], prefix: str, suffix: str):
     in_code_block = False
@@ -33,7 +29,7 @@ def yield_lines(lines: Iterator[str], prefix: str, suffix: str):
         elif in_code_block:
             yield line[strip_length:]
 
-for file in filter(lambda file: file not in excluded_files, files):
+for file in filter(lambda file: file not in excluded_files, glob.glob(glob_string, recursive=True)):
     with open(file, "r") as f:
         lines = list(yield_lines(iter(f), "```", "```"))
 

docs/test/requirements.txt

@@ -1,8 +1,5 @@
--e ../../python
+lancedb @ git+https://github.com/lancedb/lancedb.git#egg=subdir&subdirectory=python
 numpy
 pandas
 pylance
 duckdb
---extra-index-url https://download.pytorch.org/whl/cpu
-torch
-

node/examples/js-transformers/index.js

@@ -50,7 +50,7 @@ async function example() {
         { id: 5, text: 'Banana', type: 'fruit' }
     ]
 
-    const table = await db.createTable('food_table', data, embed_fun)
+    const table = await db.createTable('food_table', data, "create", embed_fun)
 
 
     // Query the table

node/examples/js-transformers/package.json

@@ -10,7 +10,7 @@
   "license": "Apache-2.0",
   "dependencies": {
     "@xenova/transformers": "^2.4.1",
-    "vectordb": "file:../.."
+    "vectordb": "^0.1.12"
   }
 
 }

node/native.js

@@ -12,25 +12,26 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-const { currentTarget } = require('@neon-rs/load')
+const { currentTarget } = require('@neon-rs/load');
 
-let nativeLib
+let nativeLib;
 
 try {
-  // When developing locally, give preference to the local built library
-  nativeLib = require('./index.node')
-} catch {
+    nativeLib = require(`vectordb-${currentTarget()}`);
+} catch (e) {
     try {
-    nativeLib = require(`@lancedb/vectordb-${currentTarget()}`)
-  } catch (e) {
+        // Might be developing locally, so try that. But don't expose that error
+        // to the user.
+        nativeLib = require("./index.node");
+    } catch {
         throw new Error(`vectordb: failed to load native library.
-  You may need to run \`npm install @lancedb/vectordb-${currentTarget()}\`.
+  You may need to run \`npm install vectordb-${currentTarget()}\`.
       
   If that does not work, please file a bug report at https://github.com/lancedb/lancedb/issues
       
-  Source error: ${e}`)
+  Source error: ${e}`);
     }   
 }
 
 // Dynamic require for runtime.
-module.exports = nativeLib
+module.exports = nativeLib;

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.3.0",
+  "version": "0.1.13",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.3.0",
+      "version": "0.1.13",
       "cpu": [
         "x64",
         "arm64"
@@ -20,18 +20,16 @@
       "dependencies": {
         "@apache-arrow/ts": "^12.0.0",
         "@neon-rs/load": "^0.0.74",
-        "apache-arrow": "^12.0.0",
-        "axios": "^1.4.0"
+        "apache-arrow": "^12.0.0"
       },
       "devDependencies": {
-        "@neon-rs/cli": "^0.0.160",
+        "@neon-rs/cli": "^0.0.74",
         "@types/chai": "^4.3.4",
         "@types/chai-as-promised": "^7.1.5",
         "@types/mocha": "^10.0.1",
         "@types/node": "^18.16.2",
         "@types/sinon": "^10.0.15",
         "@types/temp": "^0.9.1",
-        "@types/uuid": "^9.0.3",
         "@typescript-eslint/eslint-plugin": "^5.59.1",
         "cargo-cp-artifact": "^0.1",
         "chai": "^4.3.7",
@@ -49,15 +47,14 @@
         "ts-node-dev": "^2.0.0",
         "typedoc": "^0.24.7",
         "typedoc-plugin-markdown": "^3.15.3",
-        "typescript": "*",
-        "uuid": "^9.0.0"
+        "typescript": "*"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.3.0",
-        "@lancedb/vectordb-darwin-x64": "0.3.0",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.3.0",
-        "@lancedb/vectordb-linux-x64-gnu": "0.3.0",
-        "@lancedb/vectordb-win32-x64-msvc": "0.3.0"
+        "vectordb-darwin-arm64": "0.1.13",
+        "vectordb-darwin-x64": "0.1.13",
+        "vectordb-linux-arm64-gnu": "0.1.13",
+        "vectordb-linux-x64-gnu": "0.1.13",
+        "vectordb-win32-x64-msvc": "0.1.13"
       }
     },
     "node_modules/@apache-arrow/ts": {
@@ -87,97 +84,6 @@
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
       "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg=="
     },
-    "node_modules/@cargo-messages/android-arm-eabi": {
-      "version": "0.0.160",
-      "resolved": "https://registry.npmjs.org/@cargo-messages/android-arm-eabi/-/android-arm-eabi-0.0.160.tgz",
-      "integrity": "sha512-PTgCEmBHEPKJbxwlHVXB3aGES+NqpeBvn6hJNYWIkET3ZQCSJnScMlIDQXEkWndK7J+hW3Or3H32a93B/MbbfQ==",
-      "cpu": [
-        "arm"
-      ],
-      "dev": true,
-      "optional": true,
-      "os": [
-        "android"
-      ]
-    },
-    "node_modules/@cargo-messages/darwin-arm64": {
-      "version": "0.0.160",
-      "resolved": "https://registry.npmjs.org/@cargo-messages/darwin-arm64/-/darwin-arm64-0.0.160.tgz",
-      "integrity": "sha512-YSVUuc8TUTi/XmZVg9KrH0bDywKLqC1zeTyZYAYDDmqVDZW9KeTnbBUECKRs56iyHeO+kuEkVW7MKf7j2zb/FA==",
-      "cpu": [
-        "arm64"
-      ],
-      "dev": true,
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
-    "node_modules/@cargo-messages/darwin-x64": {
-      "version": "0.0.160",
-      "resolved": "https://registry.npmjs.org/@cargo-messages/darwin-x64/-/darwin-x64-0.0.160.tgz",
-      "integrity": "sha512-U+YlAR+9tKpBljnNPWMop5YhvtwfIPQSAaUYN2llteC7ZNU5/cv8CGT1vm7uFNxr2LeGuAtRbzIh2gUmTV8mng==",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
-    "node_modules/@cargo-messages/linux-arm-gnueabihf": {
-      "version": "0.0.160",
-      "resolved": "https://registry.npmjs.org/@cargo-messages/linux-arm-gnueabihf/-/linux-arm-gnueabihf-0.0.160.tgz",
-      "integrity": "sha512-wqAelTzVv1E7Ls4aviqUbem5xjzCaJQxQtVnLhv6pf1k0UyEHCS2WdufFFmWcojGe7QglI4uve3KTe01MKYj0A==",
-      "cpu": [
-        "arm"
-      ],
-      "dev": true,
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@cargo-messages/linux-x64-gnu": {
-      "version": "0.0.160",
-      "resolved": "https://registry.npmjs.org/@cargo-messages/linux-x64-gnu/-/linux-x64-gnu-0.0.160.tgz",
-      "integrity": "sha512-LQ6e7O7YYkWfDNIi/53q2QG/+lZok72LOG+NKDVCrrY4TYUcrTqWAybOV6IlkVntKPnpx8YB95umSQGeVuvhpQ==",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@cargo-messages/win32-arm64-msvc": {
-      "version": "0.0.160",
-      "resolved": "https://registry.npmjs.org/@cargo-messages/win32-arm64-msvc/-/win32-arm64-msvc-0.0.160.tgz",
-      "integrity": "sha512-VDMBhyun02gIDwmEhkYP1W9Z0tYqn4drgY5Iua1qV2tYOU58RVkWhzUYxM9rzYbnwKZlltgM46J/j5QZ3VaFrA==",
-      "cpu": [
-        "arm64"
-      ],
-      "dev": true,
-      "optional": true,
-      "os": [
-        "win32"
-      ]
-    },
-    "node_modules/@cargo-messages/win32-x64-msvc": {
-      "version": "0.0.160",
-      "resolved": "https://registry.npmjs.org/@cargo-messages/win32-x64-msvc/-/win32-x64-msvc-0.0.160.tgz",
-      "integrity": "sha512-vnoglDxF6zj0W/Co9D0H/bgnrhUuO5EumIf9v3ujLtBH94rAX11JsXh/FgC/8wQnQSsLyWSq70YxNS2wdETxjA==",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "optional": true,
-      "os": [
-        "win32"
-      ]
-    },
     "node_modules/@cspotcode/source-map-support": {
       "version": "0.8.1",
       "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
@@ -316,82 +222,13 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
-    "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.3.0.tgz",
-      "integrity": "sha512-Fg+k/cSnqmNQlSWyDp0PpaAJ67kAISfZAD+zZ3mcE8/3ml2I/wM/GVjPy2zeiQX9aR93lG1mZXFSNTDUc74tWQ==",
-      "cpu": [
-        "arm64"
-      ],
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.3.0.tgz",
-      "integrity": "sha512-CXp4b/brMbnBPZuGzKIOskd9uD90R73rWubaJ0du/Kt6fcyQX1dM1wEhWTLxI6eKf8IDL/R9QLL2cIahm1J86w==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.3.0.tgz",
-      "integrity": "sha512-1bjaRzYcDsWIRUbO2K/f+ohNmNvCgKcrrOhmiXSHVlYY8kH1LUMFZj+BhqBC0Ea0Stt7/1rsRLMRXRtaeVOEHw==",
-      "cpu": [
-        "arm64"
-      ],
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.3.0.tgz",
-      "integrity": "sha512-BEDIJ6ReGAi+tLTS/RzxIw621yo1UUUiVNTzPGV2didyiJCr1chIGbES+39d/wiFQM43Xs3CBZLNzp+jKkv0/w==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.3.0.tgz",
-      "integrity": "sha512-7K2kbWbShuifQF/6L/tWSz2DhKfIreHKlBdVOuBTYYOReQMHn5cJxgwuFgQHqMubZ9zcagtHpmo+Wtqd034OKQ==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "win32"
-      ]
-    },
     "node_modules/@neon-rs/cli": {
-      "version": "0.0.160",
-      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.160.tgz",
-      "integrity": "sha512-GQjzHPJVTOARbX3nP/fAWqBq7JlQ8XgfYlCa+iwzIXf0LC1EyfJTX+vqGD/36b9lKoyY01Z/aDUB9o/qF6ztHA==",
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
+      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
       "dev": true,
       "bin": {
         "neon": "index.js"
-      },
-      "optionalDependencies": {
-        "@cargo-messages/android-arm-eabi": "0.0.160",
-        "@cargo-messages/darwin-arm64": "0.0.160",
-        "@cargo-messages/darwin-x64": "0.0.160",
-        "@cargo-messages/linux-arm-gnueabihf": "0.0.160",
-        "@cargo-messages/linux-x64-gnu": "0.0.160",
-        "@cargo-messages/win32-arm64-msvc": "0.0.160",
-        "@cargo-messages/win32-x64-msvc": "0.0.160"
       }
     },
     "node_modules/@neon-rs/load": {
@@ -598,12 +435,6 @@
         "@types/node": "*"
       }
     },
-    "node_modules/@types/uuid": {
-      "version": "9.0.3",
-      "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.3.tgz",
-      "integrity": "sha512-taHQQH/3ZyI3zP8M/puluDEIEvtQHVYcC6y3N8ijFtAd28+Ey/G4sg1u2gB01S8MwybLOKAp9/yCMu/uR5l3Ug==",
-      "dev": true
-    },
     "node_modules/@typescript-eslint/eslint-plugin": {
       "version": "5.59.1",
       "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.59.1.tgz",
@@ -1011,7 +842,8 @@
     "node_modules/asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
+      "dev": true
     },
     "node_modules/available-typed-arrays": {
       "version": "1.0.5",
@@ -1026,13 +858,12 @@
       }
     },
     "node_modules/axios": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz",
-      "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==",
+      "version": "0.26.1",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
+      "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
+      "dev": true,
       "dependencies": {
-        "follow-redirects": "^1.15.0",
-        "form-data": "^4.0.0",
-        "proxy-from-env": "^1.1.0"
+        "follow-redirects": "^1.14.8"
       }
     },
     "node_modules/balanced-match": {
@@ -1263,6 +1094,7 @@
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
       "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+      "dev": true,
       "dependencies": {
         "delayed-stream": "~1.0.0"
       },
@@ -1485,6 +1317,7 @@
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
       "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+      "dev": true,
       "engines": {
         "node": ">=0.4.0"
       }
@@ -2251,6 +2084,7 @@
       "version": "1.15.2",
       "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
       "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
+      "dev": true,
       "funding": [
         {
           "type": "individual",
@@ -2279,6 +2113,7 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
       "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "dev": true,
       "dependencies": {
         "asynckit": "^0.4.0",
         "combined-stream": "^1.0.8",
@@ -3152,6 +2987,7 @@
       "version": "1.52.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
       "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "dev": true,
       "engines": {
         "node": ">= 0.6"
       }
@@ -3160,6 +2996,7 @@
       "version": "2.1.35",
       "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
       "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "dev": true,
       "dependencies": {
         "mime-db": "1.52.0"
       },
@@ -3453,15 +3290,6 @@
         "form-data": "^4.0.0"
       }
     },
-    "node_modules/openai/node_modules/axios": {
-      "version": "0.26.1",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
-      "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
-      "dev": true,
-      "dependencies": {
-        "follow-redirects": "^1.14.8"
-      }
-    },
     "node_modules/optionator": {
       "version": "0.9.1",
       "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz",
@@ -3613,11 +3441,6 @@
         "node": ">= 0.8.0"
       }
     },
-    "node_modules/proxy-from-env": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
-      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
-    },
     "node_modules/punycode": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
@@ -4459,21 +4282,48 @@
         "punycode": "^2.1.0"
       }
     },
-    "node_modules/uuid": {
-      "version": "9.0.0",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz",
-      "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==",
-      "dev": true,
-      "bin": {
-        "uuid": "dist/bin/uuid"
-      }
-    },
     "node_modules/v8-compile-cache-lib": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
       "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==",
       "dev": true
     },
+    "node_modules/vectordb-darwin-arm64": {
+      "version": "0.1.13",
+      "resolved": "https://registry.npmjs.org/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.1.13.tgz",
+      "integrity": "sha512-9lLuX5P8m75EfP85pfC4LxO9J7Tzu4LngX55BVAdFe6qPRHu+iHmLw0QYYSVDqNm3GtDr2qFJlL2ILlsApyYyg==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/vectordb-darwin-x64": {
+      "version": "0.1.13",
+      "resolved": "https://registry.npmjs.org/vectordb-darwin-x64/-/vectordb-darwin-x64-0.1.13.tgz",
+      "integrity": "sha512-5mkhBJlcfAqcty7Ww2csgYogq+b0NhtllAbag9IIznvqfcrvITU0H0vm5LGWbRuE/BUUxC25MJhm93YWBzqEVA==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/vectordb-linux-x64-gnu": {
+      "version": "0.1.13",
+      "resolved": "https://registry.npmjs.org/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.1.13.tgz",
+      "integrity": "sha512-fU+sIHUkXyMdrWjggT93p0blKD+pbgr+x01tn9d2/pbA1ePo2AwuE86rYPA+BjyCUE1QifPgKadzGVVpqWYmnQ==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
     "node_modules/vscode-oniguruma": {
       "version": "1.7.0",
       "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz",
@@ -4719,55 +4569,6 @@
         }
       }
     },
-    "@cargo-messages/android-arm-eabi": {
-      "version": "0.0.160",
-      "resolved": "https://registry.npmjs.org/@cargo-messages/android-arm-eabi/-/android-arm-eabi-0.0.160.tgz",
-      "integrity": "sha512-PTgCEmBHEPKJbxwlHVXB3aGES+NqpeBvn6hJNYWIkET3ZQCSJnScMlIDQXEkWndK7J+hW3Or3H32a93B/MbbfQ==",
-      "dev": true,
-      "optional": true
-    },
-    "@cargo-messages/darwin-arm64": {
-      "version": "0.0.160",
-      "resolved": "https://registry.npmjs.org/@cargo-messages/darwin-arm64/-/darwin-arm64-0.0.160.tgz",
-      "integrity": "sha512-YSVUuc8TUTi/XmZVg9KrH0bDywKLqC1zeTyZYAYDDmqVDZW9KeTnbBUECKRs56iyHeO+kuEkVW7MKf7j2zb/FA==",
-      "dev": true,
-      "optional": true
-    },
-    "@cargo-messages/darwin-x64": {
-      "version": "0.0.160",
-      "resolved": "https://registry.npmjs.org/@cargo-messages/darwin-x64/-/darwin-x64-0.0.160.tgz",
-      "integrity": "sha512-U+YlAR+9tKpBljnNPWMop5YhvtwfIPQSAaUYN2llteC7ZNU5/cv8CGT1vm7uFNxr2LeGuAtRbzIh2gUmTV8mng==",
-      "dev": true,
-      "optional": true
-    },
-    "@cargo-messages/linux-arm-gnueabihf": {
-      "version": "0.0.160",
-      "resolved": "https://registry.npmjs.org/@cargo-messages/linux-arm-gnueabihf/-/linux-arm-gnueabihf-0.0.160.tgz",
-      "integrity": "sha512-wqAelTzVv1E7Ls4aviqUbem5xjzCaJQxQtVnLhv6pf1k0UyEHCS2WdufFFmWcojGe7QglI4uve3KTe01MKYj0A==",
-      "dev": true,
-      "optional": true
-    },
-    "@cargo-messages/linux-x64-gnu": {
-      "version": "0.0.160",
-      "resolved": "https://registry.npmjs.org/@cargo-messages/linux-x64-gnu/-/linux-x64-gnu-0.0.160.tgz",
-      "integrity": "sha512-LQ6e7O7YYkWfDNIi/53q2QG/+lZok72LOG+NKDVCrrY4TYUcrTqWAybOV6IlkVntKPnpx8YB95umSQGeVuvhpQ==",
-      "dev": true,
-      "optional": true
-    },
-    "@cargo-messages/win32-arm64-msvc": {
-      "version": "0.0.160",
-      "resolved": "https://registry.npmjs.org/@cargo-messages/win32-arm64-msvc/-/win32-arm64-msvc-0.0.160.tgz",
-      "integrity": "sha512-VDMBhyun02gIDwmEhkYP1W9Z0tYqn4drgY5Iua1qV2tYOU58RVkWhzUYxM9rzYbnwKZlltgM46J/j5QZ3VaFrA==",
-      "dev": true,
-      "optional": true
-    },
-    "@cargo-messages/win32-x64-msvc": {
-      "version": "0.0.160",
-      "resolved": "https://registry.npmjs.org/@cargo-messages/win32-x64-msvc/-/win32-x64-msvc-0.0.160.tgz",
-      "integrity": "sha512-vnoglDxF6zj0W/Co9D0H/bgnrhUuO5EumIf9v3ujLtBH94rAX11JsXh/FgC/8wQnQSsLyWSq70YxNS2wdETxjA==",
-      "dev": true,
-      "optional": true
-    },
     "@cspotcode/source-map-support": {
       "version": "0.8.1",
       "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
@@ -4868,50 +4669,11 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
-    "@lancedb/vectordb-darwin-arm64": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.3.0.tgz",
-      "integrity": "sha512-Fg+k/cSnqmNQlSWyDp0PpaAJ67kAISfZAD+zZ3mcE8/3ml2I/wM/GVjPy2zeiQX9aR93lG1mZXFSNTDUc74tWQ==",
-      "optional": true
-    },
-    "@lancedb/vectordb-darwin-x64": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.3.0.tgz",
-      "integrity": "sha512-CXp4b/brMbnBPZuGzKIOskd9uD90R73rWubaJ0du/Kt6fcyQX1dM1wEhWTLxI6eKf8IDL/R9QLL2cIahm1J86w==",
-      "optional": true
-    },
-    "@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.3.0.tgz",
-      "integrity": "sha512-1bjaRzYcDsWIRUbO2K/f+ohNmNvCgKcrrOhmiXSHVlYY8kH1LUMFZj+BhqBC0Ea0Stt7/1rsRLMRXRtaeVOEHw==",
-      "optional": true
-    },
-    "@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.3.0.tgz",
-      "integrity": "sha512-BEDIJ6ReGAi+tLTS/RzxIw621yo1UUUiVNTzPGV2didyiJCr1chIGbES+39d/wiFQM43Xs3CBZLNzp+jKkv0/w==",
-      "optional": true
-    },
-    "@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.3.0.tgz",
-      "integrity": "sha512-7K2kbWbShuifQF/6L/tWSz2DhKfIreHKlBdVOuBTYYOReQMHn5cJxgwuFgQHqMubZ9zcagtHpmo+Wtqd034OKQ==",
-      "optional": true
-    },
     "@neon-rs/cli": {
-      "version": "0.0.160",
-      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.160.tgz",
-      "integrity": "sha512-GQjzHPJVTOARbX3nP/fAWqBq7JlQ8XgfYlCa+iwzIXf0LC1EyfJTX+vqGD/36b9lKoyY01Z/aDUB9o/qF6ztHA==",
-      "dev": true,
-      "requires": {
-        "@cargo-messages/android-arm-eabi": "0.0.160",
-        "@cargo-messages/darwin-arm64": "0.0.160",
-        "@cargo-messages/darwin-x64": "0.0.160",
-        "@cargo-messages/linux-arm-gnueabihf": "0.0.160",
-        "@cargo-messages/linux-x64-gnu": "0.0.160",
-        "@cargo-messages/win32-arm64-msvc": "0.0.160",
-        "@cargo-messages/win32-x64-msvc": "0.0.160"
-      }
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
+      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
+      "dev": true
     },
     "@neon-rs/load": {
       "version": "0.0.74",
@@ -5110,12 +4872,6 @@
         "@types/node": "*"
       }
     },
-    "@types/uuid": {
-      "version": "9.0.3",
-      "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.3.tgz",
-      "integrity": "sha512-taHQQH/3ZyI3zP8M/puluDEIEvtQHVYcC6y3N8ijFtAd28+Ey/G4sg1u2gB01S8MwybLOKAp9/yCMu/uR5l3Ug==",
-      "dev": true
-    },
     "@typescript-eslint/eslint-plugin": {
       "version": "5.59.1",
       "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.59.1.tgz",
@@ -5379,7 +5135,8 @@
     "asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
+      "dev": true
     },
     "available-typed-arrays": {
       "version": "1.0.5",
@@ -5388,13 +5145,12 @@
       "dev": true
     },
     "axios": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz",
-      "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==",
+      "version": "0.26.1",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
+      "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
+      "dev": true,
       "requires": {
-        "follow-redirects": "^1.15.0",
-        "form-data": "^4.0.0",
-        "proxy-from-env": "^1.1.0"
+        "follow-redirects": "^1.14.8"
       }
     },
     "balanced-match": {
@@ -5574,6 +5330,7 @@
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
       "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+      "dev": true,
       "requires": {
         "delayed-stream": "~1.0.0"
       }
@@ -5740,7 +5497,8 @@
     "delayed-stream": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
-      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="
+      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+      "dev": true
     },
     "diff": {
       "version": "4.0.2",
@@ -6310,7 +6068,8 @@
     "follow-redirects": {
       "version": "1.15.2",
       "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
-      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA=="
+      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
+      "dev": true
     },
     "for-each": {
       "version": "0.3.3",
@@ -6325,6 +6084,7 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
       "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "dev": true,
       "requires": {
         "asynckit": "^0.4.0",
         "combined-stream": "^1.0.8",
@@ -6938,12 +6698,14 @@
     "mime-db": {
       "version": "1.52.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
-      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "dev": true
     },
     "mime-types": {
       "version": "2.1.35",
       "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
       "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "dev": true,
       "requires": {
         "mime-db": "1.52.0"
       }
@@ -7169,17 +6931,6 @@
       "requires": {
         "axios": "^0.26.0",
         "form-data": "^4.0.0"
-      },
-      "dependencies": {
-        "axios": {
-          "version": "0.26.1",
-          "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
-          "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
-          "dev": true,
-          "requires": {
-            "follow-redirects": "^1.14.8"
-          }
-        }
       }
     },
     "optionator": {
@@ -7288,11 +7039,6 @@
       "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==",
       "dev": true
     },
-    "proxy-from-env": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
-      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
-    },
     "punycode": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
@@ -7867,18 +7613,30 @@
         "punycode": "^2.1.0"
       }
     },
-    "uuid": {
-      "version": "9.0.0",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz",
-      "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==",
-      "dev": true
-    },
     "v8-compile-cache-lib": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
       "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==",
       "dev": true
     },
+    "vectordb-darwin-arm64": {
+      "version": "0.1.13",
+      "resolved": "https://registry.npmjs.org/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.1.13.tgz",
+      "integrity": "sha512-9lLuX5P8m75EfP85pfC4LxO9J7Tzu4LngX55BVAdFe6qPRHu+iHmLw0QYYSVDqNm3GtDr2qFJlL2ILlsApyYyg==",
+      "optional": true
+    },
+    "vectordb-darwin-x64": {
+      "version": "0.1.13",
+      "resolved": "https://registry.npmjs.org/vectordb-darwin-x64/-/vectordb-darwin-x64-0.1.13.tgz",
+      "integrity": "sha512-5mkhBJlcfAqcty7Ww2csgYogq+b0NhtllAbag9IIznvqfcrvITU0H0vm5LGWbRuE/BUUxC25MJhm93YWBzqEVA==",
+      "optional": true
+    },
+    "vectordb-linux-x64-gnu": {
+      "version": "0.1.13",
+      "resolved": "https://registry.npmjs.org/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.1.13.tgz",
+      "integrity": "sha512-fU+sIHUkXyMdrWjggT93p0blKD+pbgr+x01tn9d2/pbA1ePo2AwuE86rYPA+BjyCUE1QifPgKadzGVVpqWYmnQ==",
+      "optional": true
+    },
     "vscode-oniguruma": {
       "version": "1.7.0",
       "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz",

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.3.0",
+  "version": "0.1.13",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -9,8 +9,7 @@
     "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json",
     "build-release": "npm run build -- --release",
     "test": "npm run tsc && mocha -recursive dist/test",
-    "integration-test": "npm run tsc && mocha -recursive dist/integration_test",
-    "lint": "eslint native.js src --ext .js,.ts",
+    "lint": "eslint src --ext .js,.ts",
     "clean": "rm -rf node_modules *.node dist/",
     "pack-build": "neon pack-build",
     "check-npm": "printenv && which node && which npm && npm --version"
@@ -28,14 +27,13 @@
   "author": "Lance Devs",
   "license": "Apache-2.0",
   "devDependencies": {
-    "@neon-rs/cli": "^0.0.160",
+    "@neon-rs/cli": "^0.0.74",
     "@types/chai": "^4.3.4",
     "@types/chai-as-promised": "^7.1.5",
     "@types/mocha": "^10.0.1",
     "@types/node": "^18.16.2",
     "@types/sinon": "^10.0.15",
     "@types/temp": "^0.9.1",
-    "@types/uuid": "^9.0.3",
     "@typescript-eslint/eslint-plugin": "^5.59.1",
     "cargo-cp-artifact": "^0.1",
     "chai": "^4.3.7",
@@ -53,14 +51,12 @@
     "ts-node-dev": "^2.0.0",
     "typedoc": "^0.24.7",
     "typedoc-plugin-markdown": "^3.15.3",
-    "typescript": "*",
-    "uuid": "^9.0.0"
+    "typescript": "*"
   },
   "dependencies": {
     "@apache-arrow/ts": "^12.0.0",
     "@neon-rs/load": "^0.0.74",
-    "apache-arrow": "^12.0.0",
-    "axios": "^1.4.0"
+    "apache-arrow": "^12.0.0"
   },
   "os": [
     "darwin",
@@ -73,18 +69,18 @@
   ],
   "neon": {
     "targets": {
-      "x86_64-apple-darwin": "@lancedb/vectordb-darwin-x64",
-      "aarch64-apple-darwin": "@lancedb/vectordb-darwin-arm64",
-      "x86_64-unknown-linux-gnu": "@lancedb/vectordb-linux-x64-gnu",
-      "aarch64-unknown-linux-gnu": "@lancedb/vectordb-linux-arm64-gnu",
-      "x86_64-pc-windows-msvc": "@lancedb/vectordb-win32-x64-msvc"
+      "x86_64-apple-darwin": "vectordb-darwin-x64",
+      "aarch64-apple-darwin": "vectordb-darwin-arm64",
+      "x86_64-unknown-linux-gnu": "vectordb-linux-x64-gnu",
+      "aarch64-unknown-linux-gnu": "vectordb-linux-arm64-gnu",
+      "x86_64-pc-windows-msvc": "vectordb-win32-x64-msvc"
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.3.0",
-    "@lancedb/vectordb-darwin-x64": "0.3.0",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.3.0",
-    "@lancedb/vectordb-linux-x64-gnu": "0.3.0",
-    "@lancedb/vectordb-win32-x64-msvc": "0.3.0"
+    "vectordb-darwin-arm64": "0.1.13",
+    "vectordb-darwin-x64": "0.1.13",
+    "vectordb-linux-arm64-gnu": "0.1.13",
+    "vectordb-linux-x64-gnu": "0.1.13",
+    "vectordb-win32-x64-msvc": "0.1.13"
   }
 }

node/src/arrow.ts

@@ -13,19 +13,18 @@
 // limitations under the License.
 
 import {
-  Field, type FixedSizeListBuilder,
+  Field,
   Float32,
+  List, type ListBuilder,
   makeBuilder,
   RecordBatchFileWriter,
-  Utf8,
+  Table, Utf8,
   type Vector,
-  FixedSizeList,
-  vectorFromArray, type Schema, Table as ArrowTable
+  vectorFromArray
 } from 'apache-arrow'
 import { type EmbeddingFunction } from './index'
 
-// Converts an Array of records into an Arrow Table, optionally applying an embeddings function to it.
-export async function convertToTable<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<ArrowTable> {
+export async function convertToTable<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Table> {
   if (data.length === 0) {
     throw new Error('At least one record needs to be provided')
   }
@@ -35,8 +34,8 @@ export async function convertToTable<T> (data: Array<Record<string, unknown>>, e
 
   for (const columnsKey of columns) {
     if (columnsKey === 'vector') {
+      const listBuilder = newVectorListBuilder()
       const vectorSize = (data[0].vector as any[]).length
-      const listBuilder = newVectorBuilder(vectorSize)
       for (const datum of data) {
         if ((datum[columnsKey] as any[]).length !== vectorSize) {
           throw new Error(`Invalid vector size, expected ${vectorSize}`)
@@ -53,7 +52,9 @@ export async function convertToTable<T> (data: Array<Record<string, unknown>>, e
 
       if (columnsKey === embeddings?.sourceColumn) {
         const vectors = await embeddings.embed(values as T[])
-        records.vector = vectorFromArray(vectors, newVectorType(vectors[0].length))
+        const listBuilder = newVectorListBuilder()
+        vectors.map(v => listBuilder.append(v))
+        records.vector = listBuilder.finish().toVector()
       }
 
       if (typeof values[0] === 'string') {
@@ -65,47 +66,20 @@ export async function convertToTable<T> (data: Array<Record<string, unknown>>, e
     }
   }
 
-  return new ArrowTable(records)
+  return new Table(records)
 }
 
 // Creates a new Arrow ListBuilder that stores a Vector column
-function newVectorBuilder (dim: number): FixedSizeListBuilder<Float32> {
+function newVectorListBuilder (): ListBuilder<Float32, any> {
+  const children = new Field<Float32>('item', new Float32())
+  const list = new List(children)
   return makeBuilder({
-    type: newVectorType(dim)
+    type: list
   })
 }
 
-// Creates the Arrow Type for a Vector column with dimension `dim`
-function newVectorType (dim: number): FixedSizeList<Float32> {
-  const children = new Field<Float32>('item', new Float32())
-  return new FixedSizeList(dim, children)
-}
-
-// Converts an Array of records into Arrow IPC format
 export async function fromRecordsToBuffer<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Buffer> {
   const table = await convertToTable(data, embeddings)
   const writer = RecordBatchFileWriter.writeAll(table)
   return Buffer.from(await writer.toUint8Array())
 }
-
-// Converts an Arrow Table into Arrow IPC format
-export async function fromTableToBuffer<T> (table: ArrowTable, embeddings?: EmbeddingFunction<T>): Promise<Buffer> {
-  if (embeddings !== undefined) {
-    const source = table.getChild(embeddings.sourceColumn)
-
-    if (source === null) {
-      throw new Error(`The embedding source column ${embeddings.sourceColumn} was not found in the Arrow Table`)
-    }
-
-    const vectors = await embeddings.embed(source.toArray() as T[])
-    const column = vectorFromArray(vectors, newVectorType(vectors[0].length))
-    table = table.assign(new ArrowTable({ vector: column }))
-  }
-  const writer = RecordBatchFileWriter.writeAll(table)
-  return Buffer.from(await writer.toUint8Array())
-}
-
-// Creates an empty Arrow Table
-export function createEmptyTable (schema: Schema): ArrowTable {
-  return new ArrowTable(schema)
-}

node/src/embedding/embedding_function.ts

@@ -26,8 +26,3 @@ export interface EmbeddingFunction<T> {
      */
   embed: (data: T[]) => Promise<number[][]>
 }
-
-export function isEmbeddingFunction<T> (value: any): value is EmbeddingFunction<T> {
-  return typeof value.sourceColumn === 'string' &&
-      typeof value.embed === 'function'
-}

node/src/index.ts

@@ -13,19 +13,17 @@
 // limitations under the License.
 
 import {
-  type Schema,
-  Table as ArrowTable
+  RecordBatchFileWriter,
+  type Table as ArrowTable,
+  tableFromIPC,
+  Vector
 } from 'apache-arrow'
-import { createEmptyTable, fromRecordsToBuffer, fromTableToBuffer } from './arrow'
+import { fromRecordsToBuffer } from './arrow'
 import type { EmbeddingFunction } from './embedding/embedding_function'
-import { RemoteConnection } from './remote'
-import { Query } from './query'
-import { isEmbeddingFunction } from './embedding/embedding_function'
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete, tableCleanupOldVersions, tableCompactFiles } = require('../native.js')
+const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableSearch, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete } = require('../native.js')
 
-export { Query }
 export type { EmbeddingFunction }
 export { OpenAIEmbeddingFunction } from './embedding/openai'
 
@@ -39,52 +37,7 @@ export interface AwsCredentials {
 
 export interface ConnectionOptions {
   uri: string
-
   awsCredentials?: AwsCredentials
-
-  awsRegion?: string
-
-  // API key for the remote connections
-  apiKey?: string
-  // Region to connect
-  region?: string
-
-  // override the host for the remote connections
-  hostOverride?: string
-}
-
-function getAwsArgs (opts: ConnectionOptions): any[] {
-  const callArgs = []
-  const awsCredentials = opts.awsCredentials
-  if (awsCredentials !== undefined) {
-    callArgs.push(awsCredentials.accessKeyId)
-    callArgs.push(awsCredentials.secretKey)
-    callArgs.push(awsCredentials.sessionToken)
-  } else {
-    callArgs.push(undefined)
-    callArgs.push(undefined)
-    callArgs.push(undefined)
-  }
-
-  callArgs.push(opts.awsRegion)
-  return callArgs
-}
-
-export interface CreateTableOptions<T> {
-  // Name of Table
-  name: string
-
-  // Data to insert into the Table
-  data?: Array<Record<string, unknown>> | ArrowTable | undefined
-
-  // Optional Arrow Schema for this table
-  schema?: Schema | undefined
-
-  // Optional embedding function used to create embeddings
-  embeddingFunction?: EmbeddingFunction<T> | undefined
-
-  // WriteOptions for this operation
-  writeOptions?: WriteOptions | undefined
 }
 
 /**
@@ -101,16 +54,9 @@ export async function connect (arg: string | Partial<ConnectionOptions>): Promis
     // opts = { uri: arg.uri, awsCredentials = arg.awsCredentials }
     opts = Object.assign({
       uri: '',
-      awsCredentials: undefined,
-      apiKey: undefined,
-      region: 'us-west-2'
+      awsCredentials: undefined
     }, arg)
   }
-
-  if (opts.uri.startsWith('db://')) {
-    // Remote connection
-    return new RemoteConnection(opts)
-  }
   const db = await databaseNew(opts.uri)
   return new LocalConnection(db, opts)
 }
@@ -133,51 +79,17 @@ export interface Connection {
    */
   openTable<T>(name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
 
-  /**
-   * Creates a new Table, optionally initializing it with new data.
-   *
-   * @param {string} name - The name of the table.
-   * @param data - Array of Records to be inserted into the table
-   * @param schema - An Arrow Schema that describe this table columns
-   * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
-   * @param {WriteOptions} writeOptions - The write options to use when creating the table.
-   */
-  createTable<T> ({ name, data, schema, embeddingFunction, writeOptions }: CreateTableOptions<T>): Promise<Table<T>>
-
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param {string} name - The name of the table.
-   * @param data - Non-empty Array of Records to be inserted into the table
-   */
-  createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table>
-
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param {string} name - The name of the table.
-   * @param data - Non-empty Array of Records to be inserted into the table
-   * @param {WriteOptions} options - The write options to use when creating the table.
-   */
-  createTable (name: string, data: Array<Record<string, unknown>>, options: WriteOptions): Promise<Table>
-
   /**
    * Creates a new Table and initialize it with new data.
    *
    * @param {string} name - The name of the table.
    * @param data - Non-empty Array of Records to be inserted into the table
+   * @param {WriteMode} mode - The write mode to use when creating the table.
    * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
    */
-  createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param {string} name - The name of the table.
-   * @param data - Non-empty Array of Records to be inserted into the table
-   * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
-   * @param {WriteOptions} options - The write options to use when creating the table.
-   */
-  createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>, options: WriteOptions): Promise<Table<T>>
+  createTable<T>(name: string, data: Array<Record<string, unknown>>, mode?: WriteMode, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
+
+  createTableArrow(name: string, table: ArrowTable): Promise<Table>
 
   /**
    * Drop an existing table.
@@ -230,34 +142,7 @@ export interface Table<T = number[]> {
   /**
    * Delete rows from this table.
    *
-   * This can be used to delete a single row, many rows, all rows, or
-   * sometimes no rows (if your predicate matches nothing).
-   *
-   * @param filter  A filter in the same format used by a sql WHERE clause. The
-   *                filter must not be empty.
-   *
-   * @examples
-   *
-   * ```ts
-   * const con = await lancedb.connect("./.lancedb")
-   * const data = [
-   *    {id: 1, vector: [1, 2]},
-   *    {id: 2, vector: [3, 4]},
-   *    {id: 3, vector: [5, 6]},
-   * ];
-   * const tbl = await con.createTable("my_table", data)
-   * await tbl.delete("id = 2")
-   * await tbl.countRows() // Returns 2
-   * ```
-   *
-   * If you have a list of values to delete, you can combine them into a
-   * stringified list and use the `IN` operator:
-   *
-   * ```ts
-   * const to_remove = [1, 5];
-   * await tbl.delete(`id IN (${to_remove.join(",")})`)
-   * await tbl.countRows() // Returns 1
-   * ```
+   * @param filter  A filter in the same format used by a sql WHERE clause.
    */
   delete: (filter: string) => Promise<void>
 }
@@ -266,16 +151,16 @@ export interface Table<T = number[]> {
  * A connection to a LanceDB database.
  */
 export class LocalConnection implements Connection {
-  private readonly _options: () => ConnectionOptions
+  private readonly _options: ConnectionOptions
   private readonly _db: any
 
   constructor (db: any, options: ConnectionOptions) {
-    this._options = () => options
+    this._options = options
     this._db = db
   }
 
   get uri (): string {
-    return this._options().uri
+    return this._options.uri
   }
 
   /**
@@ -291,7 +176,6 @@ export class LocalConnection implements Connection {
    * @param name The name of the table.
    */
   async openTable (name: string): Promise<Table>
-
   /**
    * Open a table in the database.
    *
@@ -301,66 +185,61 @@ export class LocalConnection implements Connection {
   async openTable<T> (name: string, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
   async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
   async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
-    const tbl = await databaseOpenTable.call(this._db, name, ...getAwsArgs(this._options()))
+    const tbl = await databaseOpenTable.call(this._db, name)
     if (embeddings !== undefined) {
-      return new LocalTable(tbl, name, this._options(), embeddings)
+      return new LocalTable(tbl, name, this._options, embeddings)
     } else {
-      return new LocalTable(tbl, name, this._options())
+      return new LocalTable(tbl, name, this._options)
     }
   }
 
-  async createTable<T> (name: string | CreateTableOptions<T>, data?: Array<Record<string, unknown>>, optsOrEmbedding?: WriteOptions | EmbeddingFunction<T>, opt?: WriteOptions): Promise<Table<T>> {
-    if (typeof name === 'string') {
-      let writeOptions: WriteOptions = new DefaultWriteOptions()
-      if (opt !== undefined && isWriteOptions(opt)) {
-        writeOptions = opt
-      } else if (optsOrEmbedding !== undefined && isWriteOptions(optsOrEmbedding)) {
-        writeOptions = optsOrEmbedding
+  /**
+   * Creates a new Table and initialize it with new data.
+   *
+   * @param name The name of the table.
+   * @param data Non-empty Array of Records to be inserted into the Table
+   * @param mode The write mode to use when creating the table.
+   */
+  async createTable (name: string, data: Array<Record<string, unknown>>, mode?: WriteMode): Promise<Table>
+  async createTable (name: string, data: Array<Record<string, unknown>>, mode: WriteMode): Promise<Table>
+
+  /**
+   * Creates a new Table and initialize it with new data.
+   *
+   * @param name The name of the table.
+   * @param data Non-empty Array of Records to be inserted into the Table
+   * @param mode The write mode to use when creating the table.
+   * @param embeddings An embedding function to use on this Table
+   */
+  async createTable<T> (name: string, data: Array<Record<string, unknown>>, mode: WriteMode, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
+  async createTable<T> (name: string, data: Array<Record<string, unknown>>, mode: WriteMode, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
+  async createTable<T> (name: string, data: Array<Record<string, unknown>>, mode: WriteMode, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
+    if (mode === undefined) {
+      mode = WriteMode.Create
     }
 
-      let embeddings: undefined | EmbeddingFunction<T>
-      if (optsOrEmbedding !== undefined && isEmbeddingFunction(optsOrEmbedding)) {
-        embeddings = optsOrEmbedding
+    const createArgs = [this._db, name, await fromRecordsToBuffer(data, embeddings), mode.toLowerCase()]
+    if (this._options.awsCredentials !== undefined) {
+      createArgs.push(this._options.awsCredentials.accessKeyId)
+      createArgs.push(this._options.awsCredentials.secretKey)
+      if (this._options.awsCredentials.sessionToken !== undefined) {
+        createArgs.push(this._options.awsCredentials.sessionToken)
       }
-      return await this.createTableImpl({ name, data, embeddingFunction: embeddings, writeOptions })
-    }
-    return await this.createTableImpl(name)
     }
 
-  private async createTableImpl<T> ({ name, data, schema, embeddingFunction, writeOptions = new DefaultWriteOptions() }: {
-    name: string
-    data?: Array<Record<string, unknown>> | ArrowTable | undefined
-    schema?: Schema | undefined
-    embeddingFunction?: EmbeddingFunction<T> | undefined
-    writeOptions?: WriteOptions | undefined
-  }): Promise<Table<T>> {
-    let buffer: Buffer
+    const tbl = await tableCreate.call(...createArgs)
 
-    function isEmpty (data: Array<Record<string, unknown>> | ArrowTable<any>): boolean {
-      if (data instanceof ArrowTable) {
-        return data.data.length === 0
-      }
-      return data.length === 0
-    }
-
-    if ((data === undefined) || isEmpty(data)) {
-      if (schema === undefined) {
-        throw new Error('Either data or schema needs to defined')
-      }
-      buffer = await fromTableToBuffer(createEmptyTable(schema))
-    } else if (data instanceof ArrowTable) {
-      buffer = await fromTableToBuffer(data, embeddingFunction)
+    if (embeddings !== undefined) {
+      return new LocalTable(tbl, name, this._options, embeddings)
     } else {
-      // data is Array<Record<...>>
-      buffer = await fromRecordsToBuffer(data, embeddingFunction)
+      return new LocalTable(tbl, name, this._options)
+    }
   }
 
-    const tbl = await tableCreate.call(this._db, name, buffer, writeOptions?.writeMode?.toString(), ...getAwsArgs(this._options()))
-    if (embeddingFunction !== undefined) {
-      return new LocalTable(tbl, name, this._options(), embeddingFunction)
-    } else {
-      return new LocalTable(tbl, name, this._options())
-    }
+  async createTableArrow (name: string, table: ArrowTable): Promise<Table> {
+    const writer = RecordBatchFileWriter.writeAll(table)
+    await tableCreate.call(this._db, name, Buffer.from(await writer.toUint8Array()))
+    return await this.openTable(name)
   }
 
   /**
@@ -373,10 +252,10 @@ export class LocalConnection implements Connection {
 }
 
 export class LocalTable<T = number[]> implements Table<T> {
-  private _tbl: any
+  private readonly _tbl: any
   private readonly _name: string
   private readonly _embeddings?: EmbeddingFunction<T>
-  private readonly _options: () => ConnectionOptions
+  private readonly _options: ConnectionOptions
 
   constructor (tbl: any, name: string, options: ConnectionOptions)
   /**
@@ -390,7 +269,7 @@ export class LocalTable<T = number[]> implements Table<T> {
     this._tbl = tbl
     this._name = name
     this._embeddings = embeddings
-    this._options = () => options
+    this._options = options
   }
 
   get name (): string {
@@ -402,7 +281,7 @@ export class LocalTable<T = number[]> implements Table<T> {
    * @param query The query search term
    */
   search (query: T): Query<T> {
-    return new Query(query, this._tbl, this._embeddings)
+    return new Query(this._tbl, query, this._embeddings)
   }
 
   /**
@@ -412,12 +291,15 @@ export class LocalTable<T = number[]> implements Table<T> {
    * @return The number of rows added to the table
    */
   async add (data: Array<Record<string, unknown>>): Promise<number> {
-    return tableAdd.call(
-      this._tbl,
-      await fromRecordsToBuffer(data, this._embeddings),
-      WriteMode.Append.toString(),
-      ...getAwsArgs(this._options())
-    ).then((newTable: any) => { this._tbl = newTable })
+    const callArgs = [this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Append.toString()]
+    if (this._options.awsCredentials !== undefined) {
+      callArgs.push(this._options.awsCredentials.accessKeyId)
+      callArgs.push(this._options.awsCredentials.secretKey)
+      if (this._options.awsCredentials.sessionToken !== undefined) {
+        callArgs.push(this._options.awsCredentials.sessionToken)
+      }
+    }
+    return tableAdd.call(...callArgs)
   }
 
   /**
@@ -427,12 +309,15 @@ export class LocalTable<T = number[]> implements Table<T> {
    * @return The number of rows added to the table
    */
   async overwrite (data: Array<Record<string, unknown>>): Promise<number> {
-    return tableAdd.call(
-      this._tbl,
-      await fromRecordsToBuffer(data, this._embeddings),
-      WriteMode.Overwrite.toString(),
-      ...getAwsArgs(this._options())
-    ).then((newTable: any) => { this._tbl = newTable })
+    const callArgs = [this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Overwrite.toString()]
+    if (this._options.awsCredentials !== undefined) {
+      callArgs.push(this._options.awsCredentials.accessKeyId)
+      callArgs.push(this._options.awsCredentials.secretKey)
+      if (this._options.awsCredentials.sessionToken !== undefined) {
+        callArgs.push(this._options.awsCredentials.sessionToken)
+      }
+    }
+    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Overwrite.toString())
   }
 
   /**
@@ -441,7 +326,7 @@ export class LocalTable<T = number[]> implements Table<T> {
    * @param indexParams The parameters of this Index, @see VectorIndexParams.
    */
   async createIndex (indexParams: VectorIndexParams): Promise<any> {
-    return tableCreateVectorIndex.call(this._tbl, indexParams).then((newTable: any) => { this._tbl = newTable })
+    return tableCreateVectorIndex.call(this._tbl, indexParams)
   }
 
   /**
@@ -457,113 +342,8 @@ export class LocalTable<T = number[]> implements Table<T> {
    * @param filter A filter in the same format used by a sql WHERE clause.
    */
   async delete (filter: string): Promise<void> {
-    return tableDelete.call(this._tbl, filter).then((newTable: any) => { this._tbl = newTable })
+    return tableDelete.call(this._tbl, filter)
   }
-
-  /**
-   * Clean up old versions of the table, freeing disk space.
-   *
-   * @param olderThan The minimum age in minutes of the versions to delete. If not
-   *                  provided, defaults to two weeks.
-   * @param deleteUnverified Because they may be part of an in-progress
-   *                  transaction, uncommitted files newer than 7 days old are
-   *                  not deleted by default. This means that failed transactions
-   *                  can leave around data that takes up disk space for up to
-   *                  7 days. You can override this safety mechanism by setting
-   *                 this option to `true`, only if you promise there are no
-   *                 in progress writes while you run this operation. Failure to
-   *                 uphold this promise can lead to corrupted tables.
-   * @returns
-   */
-  async cleanupOldVersions (olderThan?: number, deleteUnverified?: boolean): Promise<CleanupStats> {
-    return tableCleanupOldVersions.call(this._tbl, olderThan, deleteUnverified)
-      .then((res: { newTable: any, metrics: CleanupStats }) => {
-        this._tbl = res.newTable
-        return res.metrics
-      })
-  }
-
-  /**
-   * Run the compaction process on the table.
-   *
-   * This can be run after making several small appends to optimize the table
-   * for faster reads.
-   *
-   * @param options Advanced options configuring compaction. In most cases, you
-   *               can omit this arguments, as the default options are sensible
-   *               for most tables.
-   * @returns Metrics about the compaction operation.
-   */
-  async compactFiles (options?: CompactionOptions): Promise<CompactionMetrics> {
-    const optionsArg = options ?? {}
-    return tableCompactFiles.call(this._tbl, optionsArg)
-      .then((res: { newTable: any, metrics: CompactionMetrics }) => {
-        this._tbl = res.newTable
-        return res.metrics
-      })
-  }
-}
-
-export interface CleanupStats {
-  /**
-   * The number of bytes removed from disk.
-   */
-  bytesRemoved: number
-  /**
-   * The number of old table versions removed.
-   */
-  oldVersions: number
-}
-
-export interface CompactionOptions {
-  /**
-   * The number of rows per fragment to target. Fragments that have fewer rows
-   * will be compacted into adjacent fragments to produce larger fragments.
-   * Defaults to 1024 * 1024.
-   */
-  targetRowsPerFragment?: number
-  /**
-   * The maximum number of rows per group. Defaults to 1024.
-   */
-  maxRowsPerGroup?: number
-  /**
-   * If true, fragments that have rows that are deleted may be compacted to
-   * remove the deleted rows. This can improve the performance of queries.
-   * Default is true.
-   */
-  materializeDeletions?: boolean
-  /**
-   * A number between 0 and 1, representing the proportion of rows that must be
-   * marked deleted before a fragment is a candidate for compaction to remove
-   * the deleted rows. Default is 10%.
-   */
-  materializeDeletionsThreshold?: number
-  /**
-   * The number of threads to use for compaction. If not provided, defaults to
-   * the number of cores on the machine.
-   */
-  numThreads?: number
-}
-
-export interface CompactionMetrics {
-  /**
-   * The number of fragments that were removed.
-   */
-  fragmentsRemoved: number
-  /**
-   * The number of new fragments that were created.
-   */
-  fragmentsAdded: number
-  /**
-   * The number of files that were removed. Each fragment may have more than one
-   * file.
-   */
-  filesRemoved: number
-  /**
-   * The number of files added. This is typically equal to the number of
-   * fragments added.
-   */
-  filesAdded: number
 }
 
 /// Config to build IVF_PQ index.
@@ -623,6 +403,116 @@ export interface IvfPQIndexConfig {
 
 export type VectorIndexParams = IvfPQIndexConfig
 
+/**
+ * A builder for nearest neighbor queries for LanceDB.
+ */
+export class Query<T = number[]> {
+  private readonly _tbl: any
+  private readonly _query: T
+  private _queryVector?: number[]
+  private _limit: number
+  private _refineFactor?: number
+  private _nprobes: number
+  private _select?: string[]
+  private _filter?: string
+  private _metricType?: MetricType
+  private readonly _embeddings?: EmbeddingFunction<T>
+
+  constructor (tbl: any, query: T, embeddings?: EmbeddingFunction<T>) {
+    this._tbl = tbl
+    this._query = query
+    this._limit = 10
+    this._nprobes = 20
+    this._refineFactor = undefined
+    this._select = undefined
+    this._filter = undefined
+    this._metricType = undefined
+    this._embeddings = embeddings
+  }
+
+  /***
+   * Sets the number of results that will be returned
+   * @param value number of results
+   */
+  limit (value: number): Query<T> {
+    this._limit = value
+    return this
+  }
+
+  /**
+   * Refine the results by reading extra elements and re-ranking them in memory.
+   * @param value refine factor to use in this query.
+   */
+  refineFactor (value: number): Query<T> {
+    this._refineFactor = value
+    return this
+  }
+
+  /**
+   * The number of probes used. A higher number makes search more accurate but also slower.
+   * @param value The number of probes used.
+   */
+  nprobes (value: number): Query<T> {
+    this._nprobes = value
+    return this
+  }
+
+  /**
+   * A filter statement to be applied to this query.
+   * @param value A filter in the same format used by a sql WHERE clause.
+   */
+  filter (value: string): Query<T> {
+    this._filter = value
+    return this
+  }
+
+  where = this.filter
+
+  /** Return only the specified columns.
+   *
+   * @param value Only select the specified columns. If not specified, all columns will be returned.
+   */
+  select (value: string[]): Query<T> {
+    this._select = value
+    return this
+  }
+
+  /**
+   * The MetricType used for this Query.
+   * @param value The metric to the. @see MetricType for the different options
+   */
+  metricType (value: MetricType): Query<T> {
+    this._metricType = value
+    return this
+  }
+
+  /**
+   * Execute the query and return the results as an Array of Objects
+   */
+  async execute<T = Record<string, unknown>> (): Promise<T[]> {
+    if (this._embeddings !== undefined) {
+      this._queryVector = (await this._embeddings.embed([this._query]))[0]
+    } else {
+      this._queryVector = this._query as number[]
+    }
+
+    const buffer = await tableSearch.call(this._tbl, this)
+    const data = tableFromIPC(buffer)
+
+    return data.toArray().map((entry: Record<string, unknown>) => {
+      const newObject: Record<string, unknown> = {}
+      Object.keys(entry).forEach((key: string) => {
+        if (entry[key] instanceof Vector) {
+          newObject[key] = (entry[key] as Vector).toArray()
+        } else {
+          newObject[key] = entry[key]
+        }
+      })
+      return newObject as unknown as T
+    })
+  }
+}
+
 /**
  * Write mode for writing a table.
  */
@@ -635,23 +525,6 @@ export enum WriteMode {
   Append = 'append'
 }
 
-/**
- * Write options when creating a Table.
- */
-export interface WriteOptions {
-  /** A {@link WriteMode} to use on this operation */
-  writeMode?: WriteMode
-}
-
-export class DefaultWriteOptions implements WriteOptions {
-  writeMode = WriteMode.Create
-}
-
-export function isWriteOptions (value: any): value is WriteOptions {
-  return Object.keys(value).length === 1 &&
-      (value.writeMode === undefined || typeof value.writeMode === 'string')
-}
-
 /**
  * Distance metrics type.
  */

node/src/integration_test/test.ts

@@ -1,173 +0,0 @@
-// Copyright 2023 LanceDB Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { describe } from 'mocha'
-import * as chai from 'chai'
-import * as chaiAsPromised from 'chai-as-promised'
-import { v4 as uuidv4 } from 'uuid'
-
-import * as lancedb from '../index'
-import { tmpdir } from 'os'
-import * as fs from 'fs'
-import * as path from 'path'
-
-const assert = chai.assert
-chai.use(chaiAsPromised)
-
-describe('LanceDB AWS Integration test', function () {
-  it('s3+ddb schema is processed correctly', async function () {
-    this.timeout(15000)
-
-    // WARNING: specifying engine is NOT a publicly supported feature in lancedb yet
-    // THE API WILL CHANGE
-    const conn = await lancedb.connect('s3://lancedb-integtest?engine=ddb&ddbTableName=lancedb-integtest')
-    const data = [{ vector: Array(128).fill(1.0) }]
-
-    const tableName = uuidv4()
-    let table = await conn.createTable(tableName, data, { writeMode: lancedb.WriteMode.Overwrite })
-
-    const futs = [table.add(data), table.add(data), table.add(data), table.add(data), table.add(data)]
-    await Promise.allSettled(futs)
-
-    table = await conn.openTable(tableName)
-    assert.equal(await table.countRows(), 6)
-  })
-})
-
-describe('LanceDB Mirrored Store Integration test', function () {
-  it('s3://...?mirroredStore=... param is processed correctly', async function () {
-    this.timeout(600000)
-
-    const dir = tmpdir()
-    console.log(dir)
-    const conn = await lancedb.connect(`s3://lancedb-integtest?mirroredStore=${dir}`)
-    const data = Array(200).fill({ vector: Array(128).fill(1.0), id: 0 })
-    data.push(...Array(200).fill({ vector: Array(128).fill(1.0), id: 1 }))
-    data.push(...Array(200).fill({ vector: Array(128).fill(1.0), id: 2 }))
-    data.push(...Array(200).fill({ vector: Array(128).fill(1.0), id: 3 }))
-
-    const tableName = uuidv4()
-
-    // try create table and check if it's mirrored
-    const t = await conn.createTable(tableName, data, { writeMode: lancedb.WriteMode.Overwrite })
-
-    const mirroredPath = path.join(dir, `${tableName}.lance`)
-    fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
-      if (err != null) throw err
-      // there should be two dirs
-      assert.equal(files.length, 2)
-      assert.isTrue(files[0].isDirectory())
-      assert.isTrue(files[1].isDirectory())
-
-      fs.readdir(path.join(mirroredPath, '_transactions'), { withFileTypes: true }, (err, files) => {
-        if (err != null) throw err
-        assert.equal(files.length, 1)
-        assert.isTrue(files[0].name.endsWith('.txn'))
-      })
-
-      fs.readdir(path.join(mirroredPath, 'data'), { withFileTypes: true }, (err, files) => {
-        if (err != null) throw err
-        assert.equal(files.length, 1)
-        assert.isTrue(files[0].name.endsWith('.lance'))
-      })
-    })
-
-    // try create index and check if it's mirrored
-    await t.createIndex({ column: 'vector', type: 'ivf_pq' })
-
-    fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
-      if (err != null) throw err
-      // there should be two dirs
-      assert.equal(files.length, 3)
-      assert.isTrue(files[0].isDirectory())
-      assert.isTrue(files[1].isDirectory())
-      assert.isTrue(files[2].isDirectory())
-
-      // Two TXs now
-      fs.readdir(path.join(mirroredPath, '_transactions'), { withFileTypes: true }, (err, files) => {
-        if (err != null) throw err
-        assert.equal(files.length, 2)
-        assert.isTrue(files[0].name.endsWith('.txn'))
-        assert.isTrue(files[1].name.endsWith('.txn'))
-      })
-
-      fs.readdir(path.join(mirroredPath, 'data'), { withFileTypes: true }, (err, files) => {
-        if (err != null) throw err
-        assert.equal(files.length, 1)
-        assert.isTrue(files[0].name.endsWith('.lance'))
-      })
-
-      fs.readdir(path.join(mirroredPath, '_indices'), { withFileTypes: true }, (err, files) => {
-        if (err != null) throw err
-        assert.equal(files.length, 1)
-        assert.isTrue(files[0].isDirectory())
-
-        fs.readdir(path.join(mirroredPath, '_indices', files[0].name), { withFileTypes: true }, (err, files) => {
-          if (err != null) throw err
-
-          assert.equal(files.length, 1)
-          assert.isTrue(files[0].isFile())
-          assert.isTrue(files[0].name.endsWith('.idx'))
-        })
-      })
-    })
-
-    // try delete and check if it's mirrored
-    await t.delete('id = 0')
-
-    fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
-      if (err != null) throw err
-      // there should be two dirs
-      assert.equal(files.length, 4)
-      assert.isTrue(files[0].isDirectory())
-      assert.isTrue(files[1].isDirectory())
-      assert.isTrue(files[2].isDirectory())
-      assert.isTrue(files[3].isDirectory())
-
-      // Three TXs now
-      fs.readdir(path.join(mirroredPath, '_transactions'), { withFileTypes: true }, (err, files) => {
-        if (err != null) throw err
-        assert.equal(files.length, 3)
-        assert.isTrue(files[0].name.endsWith('.txn'))
-        assert.isTrue(files[1].name.endsWith('.txn'))
-      })
-
-      fs.readdir(path.join(mirroredPath, 'data'), { withFileTypes: true }, (err, files) => {
-        if (err != null) throw err
-        assert.equal(files.length, 1)
-        assert.isTrue(files[0].name.endsWith('.lance'))
-      })
-
-      fs.readdir(path.join(mirroredPath, '_indices'), { withFileTypes: true }, (err, files) => {
-        if (err != null) throw err
-        assert.equal(files.length, 1)
-        assert.isTrue(files[0].isDirectory())
-
-        fs.readdir(path.join(mirroredPath, '_indices', files[0].name), { withFileTypes: true }, (err, files) => {
-          if (err != null) throw err
-
-          assert.equal(files.length, 1)
-          assert.isTrue(files[0].isFile())
-          assert.isTrue(files[0].name.endsWith('.idx'))
-        })
-      })
-
-      fs.readdir(path.join(mirroredPath, '_deletions'), { withFileTypes: true }, (err, files) => {
-        if (err != null) throw err
-        assert.equal(files.length, 1)
-        assert.isTrue(files[0].name.endsWith('.arrow'))
-      })
-    })
-  })
-})

node/src/query.ts

@@ -1,141 +0,0 @@
-// Copyright 2023 LanceDB Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { Vector, tableFromIPC } from 'apache-arrow'
-import { type EmbeddingFunction } from './embedding/embedding_function'
-import { type MetricType } from '.'
-
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const { tableSearch } = require('../native.js')
-
-/**
- * A builder for nearest neighbor queries for LanceDB.
- */
-export class Query<T = number[]> {
-  private readonly _query: T
-  private readonly _tbl?: any
-  private _queryVector?: number[]
-  private _limit: number
-  private _refineFactor?: number
-  private _nprobes: number
-  private _select?: string[]
-  private _filter?: string
-  private _metricType?: MetricType
-  protected readonly _embeddings?: EmbeddingFunction<T>
-
-  constructor (query: T, tbl?: any, embeddings?: EmbeddingFunction<T>) {
-    this._tbl = tbl
-    this._query = query
-    this._limit = 10
-    this._nprobes = 20
-    this._refineFactor = undefined
-    this._select = undefined
-    this._filter = undefined
-    this._metricType = undefined
-    this._embeddings = embeddings
-  }
-
-  /***
-     * Sets the number of results that will be returned
-     * @param value number of results
-     */
-  limit (value: number): Query<T> {
-    this._limit = value
-    return this
-  }
-
-  /**
-     * Refine the results by reading extra elements and re-ranking them in memory.
-     * @param value refine factor to use in this query.
-     */
-  refineFactor (value: number): Query<T> {
-    this._refineFactor = value
-    return this
-  }
-
-  /**
-     * The number of probes used. A higher number makes search more accurate but also slower.
-     * @param value The number of probes used.
-     */
-  nprobes (value: number): Query<T> {
-    this._nprobes = value
-    return this
-  }
-
-  /**
-     * A filter statement to be applied to this query.
-     * @param value A filter in the same format used by a sql WHERE clause.
-     */
-  filter (value: string): Query<T> {
-    this._filter = value
-    return this
-  }
-
-  where = this.filter
-
-  /** Return only the specified columns.
-     *
-     * @param value Only select the specified columns. If not specified, all columns will be returned.
-     */
-  select (value: string[]): Query<T> {
-    this._select = value
-    return this
-  }
-
-  /**
-     * The MetricType used for this Query.
-     * @param value The metric to the. @see MetricType for the different options
-     */
-  metricType (value: MetricType): Query<T> {
-    this._metricType = value
-    return this
-  }
-
-  /**
-     * Execute the query and return the results as an Array of Objects
-     */
-  async execute<T = Record<string, unknown>> (): Promise<T[]> {
-    if (this._embeddings !== undefined) {
-      this._queryVector = (await this._embeddings.embed([this._query]))[0]
-    } else {
-      this._queryVector = this._query as number[]
-    }
-
-    const isElectron = this.isElectron()
-    const buffer = await tableSearch.call(this._tbl, this, isElectron)
-    const data = tableFromIPC(buffer)
-
-    return data.toArray().map((entry: Record<string, unknown>) => {
-      const newObject: Record<string, unknown> = {}
-      Object.keys(entry).forEach((key: string) => {
-        if (entry[key] instanceof Vector) {
-          newObject[key] = (entry[key] as Vector).toArray()
-        } else {
-          newObject[key] = entry[key]
-        }
-      })
-      return newObject as unknown as T
-    })
-  }
-
-  // See https://github.com/electron/electron/issues/2288
-  private isElectron (): boolean {
-    try {
-      // eslint-disable-next-line no-prototype-builtins
-      return (process?.versions?.hasOwnProperty('electron') || navigator?.userAgent?.toLowerCase()?.includes(' electron'))
-    } catch (e) {
-      return false
-    }
-  }
-}

node/src/remote/client.ts

@@ -1,137 +0,0 @@
-// Copyright 2023 LanceDB Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import axios, { type AxiosResponse } from 'axios'
-
-import { tableFromIPC, type Table as ArrowTable } from 'apache-arrow'
-
-export class HttpLancedbClient {
-  private readonly _url: string
-  private readonly _apiKey: () => string
-
-  public constructor (
-    url: string,
-    apiKey: string,
-    private readonly _dbName?: string
-  ) {
-    this._url = url
-    this._apiKey = () => apiKey
-  }
-
-  get uri (): string {
-    return this._url
-  }
-
-  public async search (
-    tableName: string,
-    vector: number[],
-    k: number,
-    nprobes: number,
-    refineFactor?: number,
-    columns?: string[],
-    filter?: string
-  ): Promise<ArrowTable<any>> {
-    const response = await axios.post(
-              `${this._url}/v1/table/${tableName}/query/`,
-              {
-                vector,
-                k,
-                nprobes,
-                refineFactor,
-                columns,
-                filter
-              },
-              {
-                headers: {
-                  'Content-Type': 'application/json',
-                  'x-api-key': this._apiKey(),
-                  ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
-                },
-                responseType: 'arraybuffer',
-                timeout: 10000
-              }
-    ).catch((err) => {
-      console.error('error: ', err)
-      return err.response
-    })
-    if (response.status !== 200) {
-      const errorData = new TextDecoder().decode(response.data)
-      throw new Error(
-        `Server Error, status: ${response.status as number}, ` +
-        `message: ${response.statusText as string}: ${errorData}`
-      )
-    }
-
-    const table = tableFromIPC(response.data)
-    return table
-  }
-
-  /**
-   * Sent GET request.
-   */
-  public async get (path: string, params?: Record<string, string | number>): Promise<AxiosResponse> {
-    const response = await axios.get(
-      `${this._url}${path}`,
-      {
-        headers: {
-          'Content-Type': 'application/json',
-          'x-api-key': this._apiKey()
-        },
-        params,
-        timeout: 10000
-      }
-    ).catch((err) => {
-      console.error('error: ', err)
-      return err.response
-    })
-    if (response.status !== 200) {
-      const errorData = new TextDecoder().decode(response.data)
-      throw new Error(
-        `Server Error, status: ${response.status as number}, ` +
-        `message: ${response.statusText as string}: ${errorData}`
-      )
-    }
-    return response
-  }
-
-  /**
-   * Sent POST request.
-   */
-  public async post (path: string, data?: any, params?: Record<string, string | number>): Promise<AxiosResponse> {
-    const response = await axios.post(
-        `${this._url}${path}`,
-        data,
-        {
-          headers: {
-            'Content-Type': 'application/json',
-            'x-api-key': this._apiKey(),
-            ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
-          },
-          params,
-          timeout: 30000
-        }
-    ).catch((err) => {
-      console.error('error: ', err)
-      return err.response
-    })
-    if (response.status !== 200) {
-      const errorData = new TextDecoder().decode(response.data)
-      throw new Error(
-          `Server Error, status: ${response.status as number}, ` +
-          `message: ${response.statusText as string}: ${errorData}`
-      )
-    }
-    return response
-  }
-}

node/src/remote/index.ts

@@ -1,162 +0,0 @@
-// Copyright 2023 LanceDB Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import {
-  type EmbeddingFunction, type Table, type VectorIndexParams, type Connection,
-  type ConnectionOptions, type CreateTableOptions, type WriteOptions
-} from '../index'
-import { Query } from '../query'
-
-import { Vector } from 'apache-arrow'
-import { HttpLancedbClient } from './client'
-
-/**
- * Remote connection.
- */
-export class RemoteConnection implements Connection {
-  private readonly _client: HttpLancedbClient
-  private readonly _dbName: string
-
-  constructor (opts: ConnectionOptions) {
-    if (!opts.uri.startsWith('db://')) {
-      throw new Error(`Invalid remote DB URI: ${opts.uri}`)
-    }
-    if (opts.apiKey === undefined || opts.region === undefined) {
-      throw new Error('API key and region are not supported for remote connections')
-    }
-
-    this._dbName = opts.uri.slice('db://'.length)
-    let server: string
-    if (opts.hostOverride === undefined) {
-      server = `https://${this._dbName}.${opts.region}.api.lancedb.com`
-    } else {
-      server = opts.hostOverride
-    }
-    this._client = new HttpLancedbClient(server, opts.apiKey, opts.hostOverride === undefined ? undefined : this._dbName)
-  }
-
-  get uri (): string {
-    // add the lancedb+ prefix back
-    return 'db://' + this._client.uri
-  }
-
-  async tableNames (): Promise<string[]> {
-    const response = await this._client.get('/v1/table/')
-    return response.data.tables
-  }
-
-  async openTable (name: string): Promise<Table>
-  async openTable<T> (name: string, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
-  async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
-    if (embeddings !== undefined) {
-      return new RemoteTable(this._client, name, embeddings)
-    } else {
-      return new RemoteTable(this._client, name)
-    }
-  }
-
-  async createTable<T> (name: string | CreateTableOptions<T>, data?: Array<Record<string, unknown>>, optsOrEmbedding?: WriteOptions | EmbeddingFunction<T>, opt?: WriteOptions): Promise<Table<T>> {
-    throw new Error('Not implemented')
-  }
-
-  async dropTable (name: string): Promise<void> {
-    await this._client.post(`/v1/table/${name}/drop/`)
-  }
-}
-
-export class RemoteQuery<T = number[]> extends Query<T> {
-  constructor (query: T, private readonly _client: HttpLancedbClient,
-    private readonly _name: string, embeddings?: EmbeddingFunction<T>) {
-    super(query, undefined, embeddings)
-  }
-
-  // TODO: refactor this to a base class + queryImpl pattern
-  async execute<T = Record<string, unknown>>(): Promise<T[]> {
-    const embeddings = this._embeddings
-    const query = (this as any)._query
-    let queryVector: number[]
-
-    if (embeddings !== undefined) {
-      queryVector = (await embeddings.embed([query]))[0]
-    } else {
-      queryVector = query as number[]
-    }
-
-    const data = await this._client.search(
-      this._name,
-      queryVector,
-      (this as any)._limit,
-      (this as any)._nprobes,
-      (this as any)._refineFactor,
-      (this as any)._select,
-      (this as any)._filter
-    )
-
-    return data.toArray().map((entry: Record<string, unknown>) => {
-      const newObject: Record<string, unknown> = {}
-      Object.keys(entry).forEach((key: string) => {
-        if (entry[key] instanceof Vector) {
-          newObject[key] = (entry[key] as Vector).toArray()
-        } else {
-          newObject[key] = entry[key]
-        }
-      })
-      return newObject as unknown as T
-    })
-  }
-}
-
-// we are using extend until we have next next version release
-// Table and Connection has both been refactored to interfaces
-export class RemoteTable<T = number[]> implements Table<T> {
-  private readonly _client: HttpLancedbClient
-  private readonly _embeddings?: EmbeddingFunction<T>
-  private readonly _name: string
-
-  constructor (client: HttpLancedbClient, name: string)
-  constructor (client: HttpLancedbClient, name: string, embeddings: EmbeddingFunction<T>)
-  constructor (client: HttpLancedbClient, name: string, embeddings?: EmbeddingFunction<T>) {
-    this._client = client
-    this._name = name
-    this._embeddings = embeddings
-  }
-
-  get name (): string {
-    return this._name
-  }
-
-  search (query: T): Query<T> {
-    return new RemoteQuery(query, this._client, this._name)//, this._embeddings_new)
-  }
-
-  async add (data: Array<Record<string, unknown>>): Promise<number> {
-    throw new Error('Not implemented')
-  }
-
-  async overwrite (data: Array<Record<string, unknown>>): Promise<number> {
-    throw new Error('Not implemented')
-  }
-
-  async createIndex (indexParams: VectorIndexParams): Promise<any> {
-    throw new Error('Not implemented')
-  }
-
-  async countRows (): Promise<number> {
-    throw new Error('Not implemented')
-  }
-
-  async delete (filter: string): Promise<void> {
-    throw new Error('Not implemented')
-  }
-}

node/src/test/embedding/openai.ts

@@ -16,7 +16,6 @@ import { describe } from 'mocha'
 import { assert } from 'chai'
 
 import { OpenAIEmbeddingFunction } from '../../embedding/openai'
-import { isEmbeddingFunction } from '../../embedding/embedding_function'
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const { OpenAIApi } = require('openai')
@@ -48,10 +47,4 @@ describe('OpenAPIEmbeddings', function () {
       assert.deepEqual(vectors[1], stubValue.data.data[1].embedding)
     })
   })
-
-  describe('isEmbeddingFunction', function () {
-    it('should match the isEmbeddingFunction guard', function () {
-      assert.isTrue(isEmbeddingFunction(new OpenAIEmbeddingFunction('text', 'sk-key')))
-    })
-  })
 })

node/src/test/io.ts

@@ -47,9 +47,7 @@ describe('LanceDB S3 client', function () {
         }
       }
       const table = await createTestDB(opts, 2, 20)
-      console.log(table)
       const con = await lancedb.connect(opts)
-      console.log(con)
       assert.equal(con.uri, opts.uri)
 
       const results = await table.search([0.1, 0.3]).limit(5).execute()
@@ -72,5 +70,5 @@ async function createTestDB (opts: ConnectionOptions, numDimensions: number = 2,
     data.push({ id: i + 1, name: `name_${i}`, price: i + 10, is_active: (i % 2 === 0), vector })
   }
 
-  return await con.createTable('vectors_2', data)
+  return await con.createTable('vectors', data)
 }

node/src/test/test.ts

@@ -18,8 +18,7 @@ import * as chai from 'chai'
 import * as chaiAsPromised from 'chai-as-promised'
 
 import * as lancedb from '../index'
-import { type AwsCredentials, type EmbeddingFunction, MetricType, Query, WriteMode, DefaultWriteOptions, isWriteOptions, type LocalTable } from '../index'
-import { FixedSizeList, Field, Int32, makeVector, Schema, Utf8, Table as ArrowTable, vectorFromArray, Float32 } from 'apache-arrow'
+import { type AwsCredentials, type EmbeddingFunction, MetricType, Query, WriteMode } from '../index'
 
 const expect = chai.expect
 const assert = chai.assert
@@ -108,9 +107,9 @@ describe('LanceDB client', function () {
       const table = await con.openTable('vectors')
       const results = await table.search([0.1, 0.1]).select(['is_active']).execute()
       assert.equal(results.length, 2)
-      // vector and _distance are always returned
+      // vector and score are always returned
       assert.isDefined(results[0].vector)
-      assert.isDefined(results[0]._distance)
+      assert.isDefined(results[0].score)
       assert.isDefined(results[0].is_active)
 
       assert.isUndefined(results[0].id)
@@ -120,45 +119,6 @@ describe('LanceDB client', function () {
   })
 
   describe('when creating a new dataset', function () {
-    it('create an empty table', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-
-      const schema = new Schema(
-        [new Field('id', new Int32()), new Field('name', new Utf8())]
-      )
-      const table = await con.createTable({ name: 'vectors', schema })
-      assert.equal(table.name, 'vectors')
-      assert.deepEqual(await con.tableNames(), ['vectors'])
-    })
-
-    it('create a table with a empty data array', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-
-      const schema = new Schema(
-        [new Field('id', new Int32()), new Field('name', new Utf8())]
-      )
-      const table = await con.createTable({ name: 'vectors', schema, data: [] })
-      assert.equal(table.name, 'vectors')
-      assert.deepEqual(await con.tableNames(), ['vectors'])
-    })
-
-    it('create a table from an Arrow Table', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-
-      const i32s = new Int32Array(new Array<number>(10))
-      const i32 = makeVector(i32s)
-
-      const data = new ArrowTable({ vector: i32 })
-
-      const table = await con.createTable({ name: 'vectors', data })
-      assert.equal(table.name, 'vectors')
-      assert.equal(await table.countRows(), 10)
-      assert.deepEqual(await con.tableNames(), ['vectors'])
-    })
-
     it('creates a new table from javascript objects', async function () {
       const dir = await track().mkdir('lancejs')
       const con = await lancedb.connect(dir)
@@ -174,18 +134,6 @@ describe('LanceDB client', function () {
       assert.equal(await table.countRows(), 2)
     })
 
-    it('fails to create a new table when the vector column is missing', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-
-      const data = [
-        { id: 1, price: 10 }
-      ]
-
-      const create = con.createTable('missing_vector', data)
-      await expect(create).to.be.rejectedWith(Error, 'column \'vector\' is missing')
-    })
-
     it('use overwrite flag to overwrite existing table', async function () {
       const dir = await track().mkdir('lancejs')
       const con = await lancedb.connect(dir)
@@ -196,7 +144,7 @@ describe('LanceDB client', function () {
       ]
 
       const tableName = 'overwrite'
-      await con.createTable(tableName, data, { writeMode: WriteMode.Create })
+      await con.createTable(tableName, data, WriteMode.Create)
 
       const newData = [
         { id: 1, vector: [0.1, 0.2], price: 10 },
@@ -206,7 +154,7 @@ describe('LanceDB client', function () {
 
       await expect(con.createTable(tableName, newData)).to.be.rejectedWith(Error, 'already exists')
 
-      const table = await con.createTable(tableName, newData, { writeMode: WriteMode.Overwrite })
+      const table = await con.createTable(tableName, newData, WriteMode.Overwrite)
       assert.equal(table.name, tableName)
       assert.equal(await table.countRows(), 3)
     })
@@ -258,36 +206,6 @@ describe('LanceDB client', function () {
     })
   })
 
-  describe('when searching an empty dataset', function () {
-    it('should not fail', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-
-      const schema = new Schema(
-        [new Field('vector', new FixedSizeList(128, new Field('float32', new Float32())))]
-      )
-      const table = await con.createTable({ name: 'vectors', schema })
-      const result = await table.search(Array(128).fill(0.1)).execute()
-      assert.isEmpty(result)
-    })
-  })
-
-  describe('when searching an empty-after-delete dataset', function () {
-    it('should not fail', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-
-      const schema = new Schema(
-        [new Field('vector', new FixedSizeList(128, new Field('float32', new Float32())))]
-      )
-      const table = await con.createTable({ name: 'vectors', schema })
-      await table.add([{ vector: Array(128).fill(0.1) }])
-      await table.delete('vector IS NOT NULL')
-      const result = await table.search(Array(128).fill(0.1)).execute()
-      assert.isEmpty(result)
-    })
-  })
-
   describe('when creating a vector index', function () {
     it('overwrite all records in a table', async function () {
       const uri = await createTestDB(32, 300)
@@ -312,22 +230,6 @@ describe('LanceDB client', function () {
       // Default replace = true
       await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
     }).timeout(50_000)
-
-    it('it should fail when the column is not a vector', async function () {
-      const uri = await createTestDB(32, 300)
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-      const createIndex = table.createIndex({ type: 'ivf_pq', column: 'name', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
-      await expect(createIndex).to.be.rejectedWith(/VectorIndex requires the column data type to be fixed size list of float32s/)
-    })
-
-    it('it should fail when the column is not a vector', async function () {
-      const uri = await createTestDB(32, 300)
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-      const createIndex = table.createIndex({ type: 'ivf_pq', column: 'name', num_partitions: -1, max_iters: 2, num_sub_vectors: 2 })
-      await expect(createIndex).to.be.rejectedWith('num_partitions: must be > 0')
-    })
   })
 
   describe('when using a custom embedding function', function () {
@@ -357,21 +259,7 @@ describe('LanceDB client', function () {
         { price: 10, name: 'foo' },
         { price: 50, name: 'bar' }
       ]
-      const table = await con.createTable('vectors', data, embeddings, { writeMode: WriteMode.Create })
-      const results = await table.search('foo').execute()
-      assert.equal(results.length, 2)
-    })
-
-    it('should create embeddings for Arrow Table', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-      const embeddingFunction = new TextEmbedding('name')
-
-      const names = vectorFromArray(['foo', 'bar'], new Utf8())
-      const data = new ArrowTable({ name: names })
-
-      const table = await con.createTable({ name: 'vectors', data, embeddingFunction })
-      assert.equal(table.name, 'vectors')
+      const table = await con.createTable('vectors', data, WriteMode.Create, embeddings)
       const results = await table.search('foo').execute()
       assert.equal(results.length, 2)
     })
@@ -380,7 +268,7 @@ describe('LanceDB client', function () {
 
 describe('Query object', function () {
   it('sets custom parameters', async function () {
-    const query = new Query([0.1, 0.3])
+    const query = new Query(undefined, [0.1, 0.3])
       .limit(1)
       .metricType(MetricType.Cosine)
       .refineFactor(100)
@@ -429,62 +317,3 @@ describe('Drop table', function () {
     assert.deepEqual(await con.tableNames(), ['t2'])
   })
 })
-
-describe('WriteOptions', function () {
-  context('#isWriteOptions', function () {
-    it('should not match empty object', function () {
-      assert.equal(isWriteOptions({}), false)
-    })
-    it('should match write options', function () {
-      assert.equal(isWriteOptions({ writeMode: WriteMode.Create }), true)
-    })
-    it('should match undefined write mode', function () {
-      assert.equal(isWriteOptions({ writeMode: undefined }), true)
-    })
-    it('should match default write options', function () {
-      assert.equal(isWriteOptions(new DefaultWriteOptions()), true)
-    })
-  })
-})
-
-describe('Compact and cleanup', function () {
-  it('can cleanup after compaction', async function () {
-    const dir = await track().mkdir('lancejs')
-    const con = await lancedb.connect(dir)
-
-    const data = [
-      { price: 10, name: 'foo', vector: [1, 2, 3] },
-      { price: 50, name: 'bar', vector: [4, 5, 6] }
-    ]
-    const table = await con.createTable('t1', data) as LocalTable
-
-    const newData = [
-      { price: 30, name: 'baz', vector: [7, 8, 9] }
-    ]
-    await table.add(newData)
-
-    const compactionMetrics = await table.compactFiles({
-      numThreads: 2
-    })
-    assert.equal(compactionMetrics.fragmentsRemoved, 2)
-    assert.equal(compactionMetrics.fragmentsAdded, 1)
-    assert.equal(await table.countRows(), 3)
-
-    await table.cleanupOldVersions()
-    assert.equal(await table.countRows(), 3)
-
-    // should have no effect, but this validates the arguments are parsed.
-    await table.compactFiles({
-      targetRowsPerFragment: 1024 * 10,
-      maxRowsPerGroup: 1024,
-      materializeDeletions: true,
-      materializeDeletionsThreshold: 0.5,
-      numThreads: 2
-    })
-
-    const cleanupMetrics = await table.cleanupOldVersions(0, true)
-    assert.isAtLeast(cleanupMetrics.bytesRemoved, 1)
-    assert.isAtLeast(cleanupMetrics.oldVersions, 1)
-    assert.equal(await table.countRows(), 3)
-  })
-})

python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.1
+current_version = 0.1.8
 commit = True
 message = [python] Bump version: {current_version} → {new_version}
 tag = True

python/README.md

@@ -16,7 +16,7 @@ pip install lancedb
 import lancedb
 db = lancedb.connect('<PATH_TO_LANCEDB_DATASET>')
 table = db.open_table('my_table')
-results = table.search([0.1, 0.3]).limit(20).to_list()
+results = table.search([0.1, 0.3]).limit(20).to_df()
 print(results)
 ```
 

python/lancedb/__init__.py

@@ -11,23 +11,15 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-import importlib.metadata
 from typing import Optional
 
-__version__ = importlib.metadata.version("lancedb")
-
 from .db import URI, DBConnection, LanceDBConnection
 from .remote.db import RemoteDBConnection
 from .schema import vector
-from .utils import sentry_log
 
 
 def connect(
-    uri: URI,
-    *,
-    api_key: Optional[str] = None,
-    region: str = "us-west-2",
-    host_override: Optional[str] = None,
+    uri: URI, *, api_key: Optional[str] = None, region: str = "us-west-2"
 ) -> DBConnection:
     """Connect to a LanceDB database.
 
@@ -35,13 +27,9 @@ def connect(
     ----------
     uri: str or Path
         The uri of the database.
-    api_key: str, optional
+    api_token: str, optional
         If presented, connect to LanceDB cloud.
         Otherwise, connect to a database on file system or cloud storage.
-    region: str, default "us-west-2"
-        The region to use for LanceDB Cloud.
-    host_override: str, optional
-        The override url for LanceDB Cloud.
 
     Examples
     --------
@@ -67,5 +55,5 @@ def connect(
     if isinstance(uri, str) and uri.startswith("db://"):
         if api_key is None:
             raise ValueError(f"api_key is required to connected LanceDB cloud: {uri}")
-        return RemoteDBConnection(uri, api_key, region, host_override)
+        return RemoteDBConnection(uri, api_key, region)
     return LanceDBConnection(uri)

python/lancedb/cli/__init__.py

@@ -1,12 +0,0 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.

python/lancedb/cli/cli.py

@@ -1,46 +0,0 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-import click
-
-from lancedb.utils import CONFIG
-
-
-@click.group()
-@click.version_option(help="LanceDB command line interface entry point")
-def cli():
-    "LanceDB command line interface"
-
-
-diagnostics_help = """
-Enable or disable LanceDB diagnostics. When enabled, LanceDB will send anonymous events to help us improve LanceDB.
-These diagnostics are used only for error reporting and no data is collected. You can find more about diagnosis on
-our docs: https://lancedb.github.io/lancedb/cli_config/
-"""
-
-
-@cli.command(help=diagnostics_help)
-@click.option("--enabled/--disabled", default=True)
-def diagnostics(enabled):
-    CONFIG.update({"diagnostics": True if enabled else False})
-    click.echo("LanceDB diagnostics is %s" % ("enabled" if enabled else "disabled"))
-
-
-@cli.command(help="Show current LanceDB configuration")
-def config():
-    # TODO: pretty print as table with colors and formatting
-    click.echo("Current LanceDB configuration:")
-    cfg = CONFIG.copy()
-    cfg.pop("uuid")  # Don't show uuid as it is not configurable
-    for item, amount in cfg.items():
-        click.echo("{} ({})".format(item, amount))

python/lancedb/common.py

@@ -11,18 +11,17 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 from pathlib import Path
-from typing import Iterable, List, Union
+from typing import List, Union
 
 import numpy as np
+import pandas as pd
 import pyarrow as pa
 
-from .util import safe_import_pandas
-
-pd = safe_import_pandas()
-
-DATA = Union[List[dict], dict, "pd.DataFrame", pa.Table, Iterable[pa.RecordBatch]]
 VEC = Union[list, np.ndarray, pa.Array, pa.ChunkedArray]
 URI = Union[str, Path]
+
+# TODO support generator
+DATA = Union[List[dict], dict, pd.DataFrame]
 VECTOR_COLUMN_NAME = "vector"
 
 

python/lancedb/conftest.py

@@ -1,10 +1,7 @@
 import os
 
-import numpy as np
 import pytest
 
-from .embeddings import EmbeddingFunctionRegistry, TextEmbeddingFunction
-
 # import lancedb so we don't have to in every example
 
 
@@ -17,24 +14,3 @@ def doctest_setup(monkeypatch, tmpdir):
     monkeypatch.setitem(os.environ, "COLUMNS", "80")
     # Work in a temporary directory
     monkeypatch.chdir(tmpdir)
-
-
-registry = EmbeddingFunctionRegistry.get_instance()
-
-
-@registry.register("test")
-class MockTextEmbeddingFunction(TextEmbeddingFunction):
-    """
-    Return the hash of the first 10 characters
-    """
-
-    def generate_embeddings(self, texts):
-        return [self._compute_one_embedding(row) for row in texts]
-
-    def _compute_one_embedding(self, row):
-        emb = np.array([float(hash(c)) for c in row[:10]])
-        emb /= np.linalg.norm(emb)
-        return emb
-
-    def ndims(self):
-        return 10

python/lancedb/context.py

@@ -12,16 +12,12 @@
 #  limitations under the License.
 from __future__ import annotations
 
-import deprecation
+import pandas as pd
 
-from . import __version__
 from .exceptions import MissingColumnError, MissingValueError
-from .util import safe_import_pandas
-
-pd = safe_import_pandas()
 
 
-def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
+def contextualize(raw_df: pd.DataFrame) -> Contextualizer:
     """Create a Contextualizer object for the given DataFrame.
 
     Used to create context windows. Context windows are rolling subsets of text
@@ -46,7 +42,7 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
     this how many tokens, but depending on the input data, it could be sentences,
     paragraphs, messages, etc.
 
-    >>> contextualize(data).window(3).stride(1).text_col('token').to_pandas()
+    >>> contextualize(data).window(3).stride(1).text_col('token').to_df()
                     token  document_id
     0     The quick brown            1
     1     quick brown fox            1
@@ -59,7 +55,7 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
     8          dog I love            1
     9   I love sandwiches            2
     10    love sandwiches            2
-    >>> contextualize(data).window(7).stride(1).min_window_size(7).text_col('token').to_pandas()
+    >>> contextualize(data).window(7).stride(1).min_window_size(7).text_col('token').to_df()
                                       token  document_id
     0   The quick brown fox jumped over the            1
     1  quick brown fox jumped over the lazy            1
@@ -71,7 +67,7 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
     ``stride`` determines how many rows to skip between each window start. This can
     be used to reduce the total number of windows generated.
 
-    >>> contextualize(data).window(4).stride(2).text_col('token').to_pandas()
+    >>> contextualize(data).window(4).stride(2).text_col('token').to_df()
                         token  document_id
     0     The quick brown fox            1
     2   brown fox jumped over            1
@@ -84,7 +80,7 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
     context windows that don't cross document boundaries. In this case, we can
     pass ``document_id`` as the group by.
 
-    >>> contextualize(data).window(4).stride(2).text_col('token').groupby('document_id').to_pandas()
+    >>> contextualize(data).window(4).stride(2).text_col('token').groupby('document_id').to_df()
                        token  document_id
     0    The quick brown fox            1
     2  brown fox jumped over            1
@@ -96,14 +92,14 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
     This can be used to trim the last few context windows which have size less than
     ``min_window_size``. By default context windows of size 1 are skipped.
 
-    >>> contextualize(data).window(6).stride(3).text_col('token').groupby('document_id').to_pandas()
+    >>> contextualize(data).window(6).stride(3).text_col('token').groupby('document_id').to_df()
                                  token  document_id
     0  The quick brown fox jumped over            1
     3     fox jumped over the lazy dog            1
     6                     the lazy dog            1
     9                I love sandwiches            2
 
-    >>> contextualize(data).window(6).stride(3).min_window_size(4).text_col('token').groupby('document_id').to_pandas()
+    >>> contextualize(data).window(6).stride(3).min_window_size(4).text_col('token').groupby('document_id').to_df()
                                  token  document_id
     0  The quick brown fox jumped over            1
     3     fox jumped over the lazy dog            1
@@ -179,21 +175,8 @@ class Contextualizer:
         self._min_window_size = min_window_size
         return self
 
-    @deprecation.deprecated(
-        deprecated_in="0.3.1",
-        removed_in="0.4.0",
-        current_version=__version__,
-        details="Use the bar function instead",
-    )
-    def to_df(self) -> "pd.DataFrame":
-        return self.to_pandas()
-
-    def to_pandas(self) -> "pd.DataFrame":
+    def to_df(self) -> pd.DataFrame:
         """Create the context windows and return a DataFrame."""
-        if pd is None:
-            raise ImportError(
-                "pandas is required to create context windows using lancedb"
-            )
 
         if self._text_col not in self._raw_df.columns.tolist():
             raise MissingColumnError(self._text_col)

python/lancedb/db.py

@@ -16,16 +16,15 @@ from __future__ import annotations
 import os
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import List, Optional, Union
+from typing import Dict, Iterable, List, Optional, Tuple, Union
 
+import pandas as pd
 import pyarrow as pa
 from pyarrow import fs
 
 from .common import DATA, URI
-from .embeddings import EmbeddingFunctionConfig
-from .pydantic import LanceModel
 from .table import LanceTable, Table
-from .util import fs_from_uri, get_uri_location, get_uri_scheme
+from .util import get_uri_location, get_uri_scheme
 
 
 class DBConnection(ABC):
@@ -40,8 +39,10 @@ class DBConnection(ABC):
     def create_table(
         self,
         name: str,
-        data: Optional[DATA] = None,
-        schema: Optional[Union[pa.Schema, LanceModel]] = None,
+        data: Optional[
+            Union[List[dict], dict, pd.DataFrame, pa.Table, Iterable[pa.RecordBatch]],
+        ] = None,
+        schema: Optional[pa.Schema] = None,
         mode: str = "create",
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
@@ -54,7 +55,7 @@ class DBConnection(ABC):
             The name of the table.
         data: list, tuple, dict, pd.DataFrame; optional
             The data to initialize the table. User must provide at least one of `data` or `schema`.
-        schema: pyarrow.Schema or LanceModel; optional
+        schema: pyarrow.Schema; optional
             The schema of the table.
         mode: str; default "create"
             The mode to use when creating the table. Can be either "create" or "overwrite".
@@ -150,14 +151,14 @@ class DBConnection(ABC):
         ...     for i in range(5):
         ...         yield pa.RecordBatch.from_arrays(
         ...             [
-        ...                 pa.array([[3.1, 4.1], [5.9, 26.5]], pa.list_(pa.float32(), 2)),
+        ...                 pa.array([[3.1, 4.1], [5.9, 26.5]]),
         ...                 pa.array(["foo", "bar"]),
         ...                 pa.array([10.0, 20.0]),
         ...             ],
         ...             ["vector", "item", "price"],
         ...         )
         >>> schema=pa.schema([
-        ...     pa.field("vector", pa.list_(pa.float32(), 2)),
+        ...     pa.field("vector", pa.list_(pa.float32())),
         ...     pa.field("item", pa.utf8()),
         ...     pa.field("price", pa.float32()),
         ... ])
@@ -194,13 +195,6 @@ class DBConnection(ABC):
         """
         raise NotImplementedError
 
-    def drop_database(self):
-        """
-        Drop database
-        This is the same thing as dropping all the tables
-        """
-        raise NotImplementedError
-
 
 class LanceDBConnection(DBConnection):
     """
@@ -258,7 +252,7 @@ class LanceDBConnection(DBConnection):
             A list of table names.
         """
         try:
-            filesystem, path = fs_from_uri(self.uri)
+            filesystem, path = fs.FileSystem.from_uri(self.uri)
         except pa.ArrowInvalid:
             raise NotImplementedError("Unsupported scheme: " + self.uri)
 
@@ -285,12 +279,11 @@ class LanceDBConnection(DBConnection):
     def create_table(
         self,
         name: str,
-        data: Optional[DATA] = None,
-        schema: Optional[Union[pa.Schema, LanceModel]] = None,
+        data: Optional[Union[List[dict], dict, pd.DataFrame]] = None,
+        schema: pa.Schema = None,
         mode: str = "create",
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
-        embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
     ) -> LanceTable:
         """Create a table in the database.
 
@@ -309,7 +302,6 @@ class LanceDBConnection(DBConnection):
             mode=mode,
             on_bad_vectors=on_bad_vectors,
             fill_value=fill_value,
-            embedding_functions=embedding_functions,
         )
         return tbl
 
@@ -327,24 +319,14 @@ class LanceDBConnection(DBConnection):
         """
         return LanceTable.open(self, name)
 
-    def drop_table(self, name: str, ignore_missing: bool = False):
+    def drop_table(self, name: str):
         """Drop a table from the database.
 
         Parameters
         ----------
         name: str
             The name of the table.
-        ignore_missing: bool, default False
-            If True, ignore if the table does not exist.
         """
-        try:
-            filesystem, path = fs_from_uri(self.uri)
+        filesystem, path = pa.fs.FileSystem.from_uri(self.uri)
         table_path = os.path.join(path, name + ".lance")
         filesystem.delete_dir(table_path)
-        except FileNotFoundError:
-            if not ignore_missing:
-                raise
-
-    def drop_database(self):
-        filesystem, path = fs_from_uri(self.uri)
-        filesystem.delete_dir(path)

python/lancedb/embeddings.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2023. LanceDB Developers
+#  Copyright 2023 LanceDB Developers
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -16,20 +16,15 @@ import sys
 from typing import Callable, Union
 
 import numpy as np
+import pandas as pd
 import pyarrow as pa
 from lance.vector import vec_to_table
 from retry import retry
 
-from ..util import safe_import_pandas
-from ..utils.general import LOGGER
-
-pd = safe_import_pandas()
-DATA = Union[pa.Table, "pd.DataFrame"]
-
 
 def with_embeddings(
     func: Callable,
-    data: DATA,
+    data: Union[pa.Table, pd.DataFrame],
     column: str = "text",
     wrap_api: bool = True,
     show_progress: bool = False,
@@ -59,24 +54,20 @@ def with_embeddings(
     pa.Table
         The input table with a new column called "vector" containing the embeddings.
     """
-    func = FunctionWrapper(func)
+    func = EmbeddingFunction(func)
     if wrap_api:
         func = func.retry().rate_limit()
     func = func.batch_size(batch_size)
     if show_progress:
         func = func.show_progress()
-    if pd is not None and isinstance(data, pd.DataFrame):
+    if isinstance(data, pd.DataFrame):
         data = pa.Table.from_pandas(data, preserve_index=False)
     embeddings = func(data[column].to_numpy())
     table = vec_to_table(np.array(embeddings))
     return data.append_column("vector", table["vector"])
 
 
-class FunctionWrapper:
-    """
-    A wrapper for embedding functions that adds rate limiting, retries, and batching.
-    """
-
+class EmbeddingFunction:
     def __init__(self, func: Callable):
         self.func = func
         self.rate_limiter_kwargs = {}
@@ -153,8 +144,3 @@ class FunctionWrapper:
             yield from tqdm(_chunker(arr), total=math.ceil(length / self._batch_size))
         else:
             yield from _chunker(arr)
-
-
-def api_key_not_found_help(provider):
-    LOGGER.error(f"Could not find API key for {provider}.")
-    raise ValueError(f"Please set the {provider.upper()}_API_KEY environment variable.")

python/lancedb/embeddings/__init__.py

@@ -1,25 +0,0 @@
-#  Copyright (c) 2023. LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-
-from .cohere import CohereEmbeddingFunction
-from .functions import (
-    EmbeddingFunction,
-    EmbeddingFunctionConfig,
-    EmbeddingFunctionRegistry,
-    OpenAIEmbeddings,
-    OpenClipEmbeddings,
-    SentenceTransformerEmbeddings,
-    TextEmbeddingFunction,
-)
-from .utils import with_embeddings

python/lancedb/embeddings/cohere.py

@@ -1,86 +0,0 @@
-#  Copyright (c) 2023. LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-import os
-from typing import ClassVar, List, Union
-
-import numpy as np
-
-from .functions import TextEmbeddingFunction, register
-from .utils import api_key_not_found_help
-
-
-@register("cohere")
-class CohereEmbeddingFunction(TextEmbeddingFunction):
-    """
-    An embedding function that uses the Cohere API
-
-    https://docs.cohere.com/docs/multilingual-language-models
-
-    Parameters
-    ----------
-    name: str, default "embed-multilingual-v2.0"
-        The name of the model to use. See the Cohere documentation for a list of available models.
-
-    Examples
-    --------
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import EmbeddingFunctionRegistry
-
-    cohere = EmbeddingFunctionRegistry.get_instance().get("cohere").create(name="embed-multilingual-v2.0")
-
-    class TextModel(LanceModel):
-        text: str = cohere.SourceField()
-        vector: Vector(cohere.ndims()) =  cohere.VectorField()
-
-    data = [ { "text": "hello world" },
-            { "text": "goodbye world" }]
-
-    db = lancedb.connect("~/.lancedb")
-    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
-
-    tbl.add(data)
-
-    """
-
-    name: str = "embed-multilingual-v2.0"
-    client: ClassVar = None
-
-    def ndims(self):
-        # TODO: fix hardcoding
-        return 768
-
-    def generate_embeddings(
-        self, texts: Union[List[str], np.ndarray]
-    ) -> List[np.array]:
-        """
-        Get the embeddings for the given texts
-
-        Parameters
-        ----------
-        texts: list[str] or np.ndarray (of str)
-            The texts to embed
-        """
-        # TODO retry, rate limit, token limit
-        self._init_client()
-        rs = CohereEmbeddingFunction.client.embed(texts=texts, model=self.name)
-
-        return [emb for emb in rs.embeddings]
-
-    def _init_client(self):
-        cohere = self.safe_import("cohere")
-        if CohereEmbeddingFunction.client is None:
-            if os.environ.get("COHERE_API_KEY") is None:
-                api_key_not_found_help("cohere")
-            CohereEmbeddingFunction.client = cohere.Client(os.environ["COHERE_API_KEY"])

python/lancedb/embeddings/functions.py

@@ -1,578 +0,0 @@
-#  Copyright (c) 2023. LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-import concurrent.futures
-import importlib
-import io
-import json
-import os
-import socket
-import urllib.error
-import urllib.parse as urlparse
-import urllib.request
-from abc import ABC, abstractmethod
-from typing import Dict, List, Optional, Union
-
-import numpy as np
-import pyarrow as pa
-from cachetools import cached
-from pydantic import BaseModel, Field, PrivateAttr
-from tqdm import tqdm
-
-
-class EmbeddingFunctionRegistry:
-    """
-    This is a singleton class used to register embedding functions
-    and fetch them by name. It also handles serializing and deserializing.
-    You can implement your own embedding function by subclassing EmbeddingFunction
-    or TextEmbeddingFunction and registering it with the registry.
-
-    Examples
-    --------
-    >>> registry = EmbeddingFunctionRegistry.get_instance()
-    >>> @registry.register("my-embedding-function")
-    ... class MyEmbeddingFunction(EmbeddingFunction):
-    ...     def ndims(self) -> int:
-    ...         return 128
-    ...
-    ...     def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]:
-    ...         return self.compute_source_embeddings(query, *args, **kwargs)
-    ...
-    ...     def compute_source_embeddings(self, texts: TEXT, *args, **kwargs) -> List[np.array]:
-    ...         return [np.random.rand(self.ndims()) for _ in range(len(texts))]
-    ...
-    >>> registry.get("my-embedding-function")
-    <class 'lancedb.embeddings.functions.MyEmbeddingFunction'>
-    """
-
-    @classmethod
-    def get_instance(cls):
-        return __REGISTRY__
-
-    def __init__(self):
-        self._functions = {}
-
-    def register(self, alias: str = None):
-        """
-        This creates a decorator that can be used to register
-        an EmbeddingFunction.
-
-        Parameters
-        ----------
-        alias : Optional[str]
-            a human friendly name for the embedding function. If not
-            provided, the class name will be used.
-        """
-
-        # This is a decorator for a class that inherits from BaseModel
-        # It adds the class to the registry
-        def decorator(cls):
-            if not issubclass(cls, EmbeddingFunction):
-                raise TypeError("Must be a subclass of EmbeddingFunction")
-            if cls.__name__ in self._functions:
-                raise KeyError(f"{cls.__name__} was already registered")
-            key = alias or cls.__name__
-            self._functions[key] = cls
-            cls.__embedding_function_registry_alias__ = alias
-            return cls
-
-        return decorator
-
-    def reset(self):
-        """
-        Reset the registry to its initial state
-        """
-        self._functions = {}
-
-    def get(self, name: str):
-        """
-        Fetch an embedding function class by name
-
-        Parameters
-        ----------
-        name : str
-            The name of the embedding function to fetch
-            Either the alias or the class name if no alias was provided
-            during registration
-        """
-        return self._functions[name]
-
-    def parse_functions(
-        self, metadata: Optional[Dict[bytes, bytes]]
-    ) -> Dict[str, "EmbeddingFunctionConfig"]:
-        """
-        Parse the metadata from an arrow table and
-        return a mapping of the vector column to the
-        embedding function and source column
-
-        Parameters
-        ----------
-        metadata : Optional[Dict[bytes, bytes]]
-            The metadata from an arrow table. Note that
-            the keys and values are bytes (pyarrow api)
-
-        Returns
-        -------
-        functions : dict
-            A mapping of vector column name to embedding function.
-            An empty dict is returned if input is None or does not
-            contain b"embedding_functions".
-        """
-        if metadata is None or b"embedding_functions" not in metadata:
-            return {}
-        serialized = metadata[b"embedding_functions"]
-        raw_list = json.loads(serialized.decode("utf-8"))
-        return {
-            obj["vector_column"]: EmbeddingFunctionConfig(
-                vector_column=obj["vector_column"],
-                source_column=obj["source_column"],
-                function=self.get(obj["name"])(**obj["model"]),
-            )
-            for obj in raw_list
-        }
-
-    def function_to_metadata(self, conf: "EmbeddingFunctionConfig"):
-        """
-        Convert the given embedding function and source / vector column configs
-        into a config dictionary that can be serialized into arrow metadata
-        """
-        func = conf.function
-        name = getattr(
-            func, "__embedding_function_registry_alias__", func.__class__.__name__
-        )
-        json_data = func.safe_model_dump()
-        return {
-            "name": name,
-            "model": json_data,
-            "source_column": conf.source_column,
-            "vector_column": conf.vector_column,
-        }
-
-    def get_table_metadata(self, func_list):
-        """
-        Convert a list of embedding functions and source / vector configs
-        into a config dictionary that can be serialized into arrow metadata
-        """
-        if func_list is None or len(func_list) == 0:
-            return None
-        json_data = [self.function_to_metadata(func) for func in func_list]
-        # Note that metadata dictionary values must be bytes
-        # so we need to json dump then utf8 encode
-        metadata = json.dumps(json_data, indent=2).encode("utf-8")
-        return {"embedding_functions": metadata}
-
-
-# Global instance
-__REGISTRY__ = EmbeddingFunctionRegistry()
-
-
-TEXT = Union[str, List[str], pa.Array, pa.ChunkedArray, np.ndarray]
-IMAGES = Union[
-    str, bytes, List[str], List[bytes], pa.Array, pa.ChunkedArray, np.ndarray
-]
-
-
-class EmbeddingFunction(BaseModel, ABC):
-    """
-    An ABC for embedding functions.
-
-    All concrete embedding functions must implement the following:
-    1. compute_query_embeddings() which takes a query and returns a list of embeddings
-    2. get_source_embeddings() which returns a list of embeddings for the source column
-    For text data, the two will be the same. For multi-modal data, the source column
-    might be images and the vector column might be text.
-    3. ndims method which returns the number of dimensions of the vector column
-    """
-
-    _ndims: int = PrivateAttr()
-
-    @classmethod
-    def create(cls, **kwargs):
-        """
-        Create an instance of the embedding function
-        """
-        return cls(**kwargs)
-
-    @abstractmethod
-    def compute_query_embeddings(self, *args, **kwargs) -> List[np.array]:
-        """
-        Compute the embeddings for a given user query
-        """
-        pass
-
-    @abstractmethod
-    def compute_source_embeddings(self, *args, **kwargs) -> List[np.array]:
-        """
-        Compute the embeddings for the source column in the database
-        """
-        pass
-
-    def sanitize_input(self, texts: TEXT) -> Union[List[str], np.ndarray]:
-        """
-        Sanitize the input to the embedding function.
-        """
-        if isinstance(texts, str):
-            texts = [texts]
-        elif isinstance(texts, pa.Array):
-            texts = texts.to_pylist()
-        elif isinstance(texts, pa.ChunkedArray):
-            texts = texts.combine_chunks().to_pylist()
-        return texts
-
-    @classmethod
-    def safe_import(cls, module: str, mitigation=None):
-        """
-        Import the specified module. If the module is not installed,
-        raise an ImportError with a helpful message.
-
-        Parameters
-        ----------
-        module : str
-            The name of the module to import
-        mitigation : Optional[str]
-            The package(s) to install to mitigate the error.
-            If not provided then the module name will be used.
-        """
-        try:
-            return importlib.import_module(module)
-        except ImportError:
-            raise ImportError(f"Please install {mitigation or module}")
-
-    def safe_model_dump(self):
-        from ..pydantic import PYDANTIC_VERSION
-
-        if PYDANTIC_VERSION.major < 2:
-            return dict(self)
-        return self.model_dump()
-
-    @abstractmethod
-    def ndims(self):
-        """
-        Return the dimensions of the vector column
-        """
-        pass
-
-    def SourceField(self, **kwargs):
-        """
-        Creates a pydantic Field that can automatically annotate
-        the source column for this embedding function
-        """
-        return Field(json_schema_extra={"source_column_for": self}, **kwargs)
-
-    def VectorField(self, **kwargs):
-        """
-        Creates a pydantic Field that can automatically annotate
-        the target vector column for this embedding function
-        """
-        return Field(json_schema_extra={"vector_column_for": self}, **kwargs)
-
-
-class EmbeddingFunctionConfig(BaseModel):
-    """
-    This model encapsulates the configuration for a embedding function
-    in a lancedb table. It holds the embedding function, the source column,
-    and the vector column
-    """
-
-    vector_column: str
-    source_column: str
-    function: EmbeddingFunction
-
-
-class TextEmbeddingFunction(EmbeddingFunction):
-    """
-    A callable ABC for embedding functions that take text as input
-    """
-
-    def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]:
-        return self.compute_source_embeddings(query, *args, **kwargs)
-
-    def compute_source_embeddings(self, texts: TEXT, *args, **kwargs) -> List[np.array]:
-        texts = self.sanitize_input(texts)
-        return self.generate_embeddings(texts)
-
-    @abstractmethod
-    def generate_embeddings(
-        self, texts: Union[List[str], np.ndarray]
-    ) -> List[np.array]:
-        """
-        Generate the embeddings for the given texts
-        """
-        pass
-
-
-# @EmbeddingFunctionRegistry.get_instance().register(name) doesn't work in 3.8
-register = lambda name: EmbeddingFunctionRegistry.get_instance().register(name)
-
-
-@register("sentence-transformers")
-class SentenceTransformerEmbeddings(TextEmbeddingFunction):
-    """
-    An embedding function that uses the sentence-transformers library
-
-    https://huggingface.co/sentence-transformers
-    """
-
-    name: str = "all-MiniLM-L6-v2"
-    device: str = "cpu"
-    normalize: bool = True
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self._ndims = None
-
-    @property
-    def embedding_model(self):
-        """
-        Get the sentence-transformers embedding model specified by the
-        name and device. This is cached so that the model is only loaded
-        once per process.
-        """
-        return self.__class__.get_embedding_model(self.name, self.device)
-
-    def ndims(self):
-        if self._ndims is None:
-            self._ndims = len(self.generate_embeddings("foo")[0])
-        return self._ndims
-
-    def generate_embeddings(
-        self, texts: Union[List[str], np.ndarray]
-    ) -> List[np.array]:
-        """
-        Get the embeddings for the given texts
-
-        Parameters
-        ----------
-        texts: list[str] or np.ndarray (of str)
-            The texts to embed
-        """
-        return self.embedding_model.encode(
-            list(texts),
-            convert_to_numpy=True,
-            normalize_embeddings=self.normalize,
-        ).tolist()
-
-    @classmethod
-    @cached(cache={})
-    def get_embedding_model(cls, name, device):
-        """
-        Get the sentence-transformers embedding model specified by the
-        name and device. This is cached so that the model is only loaded
-        once per process.
-
-        Parameters
-        ----------
-        name : str
-            The name of the model to load
-        device : str
-            The device to load the model on
-
-        TODO: use lru_cache instead with a reasonable/configurable maxsize
-        """
-        sentence_transformers = cls.safe_import(
-            "sentence_transformers", "sentence-transformers"
-        )
-        return sentence_transformers.SentenceTransformer(name, device=device)
-
-
-@register("openai")
-class OpenAIEmbeddings(TextEmbeddingFunction):
-    """
-    An embedding function that uses the OpenAI API
-
-    https://platform.openai.com/docs/guides/embeddings
-    """
-
-    name: str = "text-embedding-ada-002"
-
-    def ndims(self):
-        # TODO don't hardcode this
-        return 1536
-
-    def generate_embeddings(
-        self, texts: Union[List[str], np.ndarray]
-    ) -> List[np.array]:
-        """
-        Get the embeddings for the given texts
-
-        Parameters
-        ----------
-        texts: list[str] or np.ndarray (of str)
-            The texts to embed
-        """
-        # TODO retry, rate limit, token limit
-        openai = self.safe_import("openai")
-        rs = openai.Embedding.create(input=texts, model=self.name)["data"]
-        return [v["embedding"] for v in rs]
-
-
-@register("open-clip")
-class OpenClipEmbeddings(EmbeddingFunction):
-    """
-    An embedding function that uses the OpenClip API
-    For multi-modal text-to-image search
-
-    https://github.com/mlfoundations/open_clip
-    """
-
-    name: str = "ViT-B-32"
-    pretrained: str = "laion2b_s34b_b79k"
-    device: str = "cpu"
-    batch_size: int = 64
-    normalize: bool = True
-    _model = PrivateAttr()
-    _preprocess = PrivateAttr()
-    _tokenizer = PrivateAttr()
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        open_clip = self.safe_import("open_clip", "open-clip")
-        model, _, preprocess = open_clip.create_model_and_transforms(
-            self.name, pretrained=self.pretrained
-        )
-        model.to(self.device)
-        self._model, self._preprocess = model, preprocess
-        self._tokenizer = open_clip.get_tokenizer(self.name)
-        self._ndims = None
-
-    def ndims(self):
-        if self._ndims is None:
-            self._ndims = self.generate_text_embeddings("foo").shape[0]
-        return self._ndims
-
-    def compute_query_embeddings(
-        self, query: Union[str, "PIL.Image.Image"], *args, **kwargs
-    ) -> List[np.ndarray]:
-        """
-        Compute the embeddings for a given user query
-
-        Parameters
-        ----------
-        query : Union[str, PIL.Image.Image]
-            The query to embed. A query can be either text or an image.
-        """
-        if isinstance(query, str):
-            return [self.generate_text_embeddings(query)]
-        else:
-            PIL = self.safe_import("PIL", "pillow")
-            if isinstance(query, PIL.Image.Image):
-                return [self.generate_image_embedding(query)]
-            else:
-                raise TypeError("OpenClip supports str or PIL Image as query")
-
-    def generate_text_embeddings(self, text: str) -> np.ndarray:
-        torch = self.safe_import("torch")
-        text = self.sanitize_input(text)
-        text = self._tokenizer(text)
-        text.to(self.device)
-        with torch.no_grad():
-            text_features = self._model.encode_text(text.to(self.device))
-            if self.normalize:
-                text_features /= text_features.norm(dim=-1, keepdim=True)
-            return text_features.cpu().numpy().squeeze()
-
-    def sanitize_input(self, images: IMAGES) -> Union[List[bytes], np.ndarray]:
-        """
-        Sanitize the input to the embedding function.
-        """
-        if isinstance(images, (str, bytes)):
-            images = [images]
-        elif isinstance(images, pa.Array):
-            images = images.to_pylist()
-        elif isinstance(images, pa.ChunkedArray):
-            images = images.combine_chunks().to_pylist()
-        return images
-
-    def compute_source_embeddings(
-        self, images: IMAGES, *args, **kwargs
-    ) -> List[np.array]:
-        """
-        Get the embeddings for the given images
-        """
-        images = self.sanitize_input(images)
-        embeddings = []
-        for i in range(0, len(images), self.batch_size):
-            j = min(i + self.batch_size, len(images))
-            batch = images[i:j]
-            embeddings.extend(self._parallel_get(batch))
-        return embeddings
-
-    def _parallel_get(self, images: Union[List[str], List[bytes]]) -> List[np.ndarray]:
-        """
-        Issue concurrent requests to retrieve the image data
-        """
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [
-                executor.submit(self.generate_image_embedding, image)
-                for image in images
-            ]
-            return [future.result() for future in tqdm(futures)]
-
-    def generate_image_embedding(
-        self, image: Union[str, bytes, "PIL.Image.Image"]
-    ) -> np.ndarray:
-        """
-        Generate the embedding for a single image
-
-        Parameters
-        ----------
-        image : Union[str, bytes, PIL.Image.Image]
-            The image to embed. If the image is a str, it is treated as a uri.
-            If the image is bytes, it is treated as the raw image bytes.
-        """
-        torch = self.safe_import("torch")
-        # TODO handle retry and errors for https
-        image = self._to_pil(image)
-        image = self._preprocess(image).unsqueeze(0)
-        with torch.no_grad():
-            return self._encode_and_normalize_image(image)
-
-    def _to_pil(self, image: Union[str, bytes]):
-        PIL = self.safe_import("PIL", "pillow")
-        if isinstance(image, bytes):
-            return PIL.Image.open(io.BytesIO(image))
-        if isinstance(image, PIL.Image.Image):
-            return image
-        elif isinstance(image, str):
-            parsed = urlparse.urlparse(image)
-            # TODO handle drive letter on windows.
-            if parsed.scheme == "file":
-                return PIL.Image.open(parsed.path)
-            elif parsed.scheme == "":
-                return PIL.Image.open(image if os.name == "nt" else parsed.path)
-            elif parsed.scheme.startswith("http"):
-                return PIL.Image.open(io.BytesIO(url_retrieve(image)))
-            else:
-                raise NotImplementedError("Only local and http(s) urls are supported")
-
-    def _encode_and_normalize_image(self, image_tensor: "torch.Tensor"):
-        """
-        encode a single image tensor and optionally normalize the output
-        """
-        image_features = self._model.encode_image(image_tensor.to(self.device))
-        if self.normalize:
-            image_features /= image_features.norm(dim=-1, keepdim=True)
-        return image_features.cpu().numpy().squeeze()
-
-
-def url_retrieve(url: str):
-    """
-    Parameters
-    ----------
-    url: str
-        URL to download from
-    """
-    try:
-        with urllib.request.urlopen(url) as conn:
-            return conn.read()
-    except (socket.gaierror, urllib.error.URLError) as err:
-        raise ConnectionError("could not download {} due to {}".format(url, err))

python/lancedb/pydantic.py

@@ -11,7 +11,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-"""Pydantic (v1 / v2) adapter for LanceDB"""
+"""Pydantic adapter for LanceDB"""
 
 from __future__ import annotations
 
@@ -19,21 +19,11 @@ import inspect
 import sys
 import types
 from abc import ABC, abstractmethod
-from typing import Any, Callable, Dict, Generator, List, Type, Union, _GenericAlias
+from typing import Any, List, Type, Union, _GenericAlias
 
-import numpy as np
 import pyarrow as pa
 import pydantic
-import semver
-
-from .embeddings import EmbeddingFunctionRegistry
-
-PYDANTIC_VERSION = semver.Version.parse(pydantic.__version__)
-try:
-    from pydantic_core import CoreSchema, core_schema
-except ImportError:
-    if PYDANTIC_VERSION >= (2,):
-        raise
+from pydantic_core import CoreSchema, core_schema
 
 
 class FixedSizeListMixin(ABC):
@@ -48,19 +38,7 @@ class FixedSizeListMixin(ABC):
         raise NotImplementedError
 
 
-def vector(dim: int, value_type: pa.DataType = pa.float32()):
-    # TODO: remove in future release
-    from warnings import warn
-
-    warn(
-        "lancedb.pydantic.vector() is deprecated, use lancedb.pydantic.Vector instead."
-        "This function will be removed in future release",
-        DeprecationWarning,
-    )
-    return Vector(dim, value_type)
-
-
-def Vector(
+def vector(
     dim: int, value_type: pa.DataType = pa.float32()
 ) -> Type[FixedSizeListMixin]:
     """Pydantic Vector Type.
@@ -79,12 +57,12 @@ def Vector(
     --------
 
     >>> import pydantic
-    >>> from lancedb.pydantic import Vector
+    >>> from lancedb.pydantic import vector
     ...
     >>> class MyModel(pydantic.BaseModel):
     ...     id: int
     ...     url: str
-    ...     embeddings: Vector(768)
+    ...     embeddings: vector(768)
     >>> schema = pydantic_to_schema(MyModel)
     >>> assert schema == pa.schema([
     ...     pa.field("id", pa.int64(), False),
@@ -95,9 +73,6 @@ def Vector(
 
     # TODO: make a public parameterized type.
     class FixedSizeList(list, FixedSizeListMixin):
-        def __repr__(self):
-            return f"FixedSizeList(dim={dim})"
-
         @staticmethod
         def dim() -> int:
             return dim
@@ -119,25 +94,6 @@ def Vector(
                 ),
             )
 
-        @classmethod
-        def __get_validators__(cls) -> Generator[Callable, None, None]:
-            yield cls.validate
-
-        # For pydantic v1
-        @classmethod
-        def validate(cls, v):
-            if not isinstance(v, (list, range, np.ndarray)) or len(v) != dim:
-                raise TypeError("A list of numbers or numpy.ndarray is needed")
-            return cls(v)
-
-        if PYDANTIC_VERSION < (2, 0):
-
-            @classmethod
-            def __modify_schema__(cls, field_schema: Dict[str, Any]):
-                field_schema["items"] = {"type": "number"}
-                field_schema["maxItems"] = dim
-                field_schema["minItems"] = dim
-
     return FixedSizeList
 
 
@@ -164,20 +120,11 @@ def _py_type_to_arrow_type(py_type: Type[Any]) -> pa.DataType:
     )
 
 
-if PYDANTIC_VERSION.major < 2:
-
-    def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
-        return [
-            _pydantic_to_field(name, field) for name, field in model.__fields__.items()
-        ]
-
-else:
-
-    def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
-        return [
-            _pydantic_to_field(name, field)
-            for name, field in model.model_fields.items()
-        ]
+def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
+    fields = []
+    for name, field in model.model_fields.items():
+        fields.append(_pydantic_to_field(name, field))
+    return fields
 
 
 def _pydantic_to_arrow_type(field: pydantic.fields.FieldInfo) -> pa.DataType:
@@ -238,103 +185,28 @@ def pydantic_to_schema(model: Type[pydantic.BaseModel]) -> pa.Schema:
     >>> from typing import List, Optional
     >>> import pydantic
     >>> from lancedb.pydantic import pydantic_to_schema
+    ...
+    >>> class InnerModel(pydantic.BaseModel):
+    ...     a: str
+    ...     b: Optional[float]
+    >>>
     >>> class FooModel(pydantic.BaseModel):
     ...     id: int
-    ...     s: str
+    ...     s: Optional[str] = None
     ...     vec: List[float]
     ...     li: List[int]
-    ...
+    ...     inner: InnerModel
     >>> schema = pydantic_to_schema(FooModel)
     >>> assert schema == pa.schema([
     ...     pa.field("id", pa.int64(), False),
-    ...     pa.field("s", pa.utf8(), False),
+    ...     pa.field("s", pa.utf8(), True),
     ...     pa.field("vec", pa.list_(pa.float64()), False),
     ...     pa.field("li", pa.list_(pa.int64()), False),
+    ...     pa.field("inner", pa.struct([
+    ...         pa.field("a", pa.utf8(), False),
+    ...         pa.field("b", pa.float64(), True),
+    ...     ]), False),
     ... ])
     """
     fields = _pydantic_model_to_fields(model)
     return pa.schema(fields)
-
-
-class LanceModel(pydantic.BaseModel):
-    """
-    A Pydantic Model base class that can be converted to a LanceDB Table.
-
-    Examples
-    --------
-    >>> import lancedb
-    >>> from lancedb.pydantic import LanceModel, Vector
-    >>>
-    >>> class TestModel(LanceModel):
-    ...     name: str
-    ...     vector: Vector(2)
-    ...
-    >>> db = lancedb.connect("/tmp")
-    >>> table = db.create_table("test", schema=TestModel.to_arrow_schema())
-    >>> table.add([
-    ...     TestModel(name="test", vector=[1.0, 2.0])
-    ... ])
-    >>> table.search([0., 0.]).limit(1).to_pydantic(TestModel)
-    [TestModel(name='test', vector=FixedSizeList(dim=2))]
-    """
-
-    @classmethod
-    def to_arrow_schema(cls):
-        """
-        Get the Arrow Schema for this model.
-        """
-        schema = pydantic_to_schema(cls)
-        functions = cls.parse_embedding_functions()
-        if len(functions) > 0:
-            metadata = EmbeddingFunctionRegistry.get_instance().get_table_metadata(
-                functions
-            )
-            schema = schema.with_metadata(metadata)
-        return schema
-
-    @classmethod
-    def field_names(cls) -> List[str]:
-        """
-        Get the field names of this model.
-        """
-        return list(cls.safe_get_fields().keys())
-
-    @classmethod
-    def safe_get_fields(cls):
-        if PYDANTIC_VERSION.major < 2:
-            return cls.__fields__
-        return cls.model_fields
-
-    @classmethod
-    def parse_embedding_functions(cls) -> List["EmbeddingFunctionConfig"]:
-        """
-        Parse the embedding functions from this model.
-        """
-        from .embeddings import EmbeddingFunctionConfig
-
-        vec_and_function = []
-        for name, field_info in cls.safe_get_fields().items():
-            func = get_extras(field_info, "vector_column_for")
-            if func is not None:
-                vec_and_function.append([name, func])
-
-        configs = []
-        for vec, func in vec_and_function:
-            for source, field_info in cls.safe_get_fields().items():
-                src_func = get_extras(field_info, "source_column_for")
-                if src_func == func:
-                    configs.append(
-                        EmbeddingFunctionConfig(
-                            source_column=source, vector_column=vec, function=func
-                        )
-                    )
-        return configs
-
-
-def get_extras(field_info: pydantic.fields.FieldInfo, key: str) -> Any:
-    """
-    Get the extra metadata from a Pydantic FieldInfo.
-    """
-    if PYDANTIC_VERSION.major >= 2:
-        return (field_info.json_schema_extra or {}).get(key)
-    return (field_info.field_info.extra or {}).get("json_schema_extra", {}).get(key)

python/lancedb/query.py

@@ -13,23 +13,17 @@
 
 from __future__ import annotations
 
-from abc import ABC, abstractmethod
-from typing import List, Literal, Optional, Type, Union
+from typing import List, Literal, Optional, Union
 
-import deprecation
 import numpy as np
+import pandas as pd
 import pyarrow as pa
-import pydantic
+from pydantic import BaseModel
 
-from . import __version__
 from .common import VECTOR_COLUMN_NAME
-from .pydantic import LanceModel
-from .util import safe_import_pandas
-
-pd = safe_import_pandas()
 
 
-class Query(pydantic.BaseModel):
+class Query(BaseModel):
     """A Query"""
 
     vector_column: str = VECTOR_COLUMN_NAME
@@ -40,9 +34,6 @@ class Query(pydantic.BaseModel):
     # sql filter to refine the query with
     filter: Optional[str] = None
 
-    # if True then apply the filter before vector search
-    prefilter: bool = False
-
     # top k results to return
     k: int
 
@@ -60,139 +51,44 @@ class Query(pydantic.BaseModel):
     refine_factor: Optional[int] = None
 
 
-class LanceQueryBuilder(ABC):
-    @classmethod
-    def create(
-        cls,
+class LanceQueryBuilder:
+    """
+    A builder for nearest neighbor queries for LanceDB.
+
+    Examples
+    --------
+    >>> import lancedb
+    >>> data = [{"vector": [1.1, 1.2], "b": 2},
+    ...         {"vector": [0.5, 1.3], "b": 4},
+    ...         {"vector": [0.4, 0.4], "b": 6},
+    ...         {"vector": [0.4, 0.4], "b": 10}]
+    >>> db = lancedb.connect("./.lancedb")
+    >>> table = db.create_table("my_table", data=data)
+    >>> (table.search([0.4, 0.4])
+    ...       .metric("cosine")
+    ...       .where("b < 10")
+    ...       .select(["b"])
+    ...       .limit(2)
+    ...       .to_df())
+       b      vector  score
+    0  6  [0.4, 0.4]    0.0
+    """
+
+    def __init__(
+        self,
         table: "lancedb.table.Table",
-        query: Optional[Union[np.ndarray, str, "PIL.Image.Image"]],
-        query_type: str,
-        vector_column_name: str,
-    ) -> LanceQueryBuilder:
-        if query is None:
-            return LanceEmptyQueryBuilder(table)
-
-        # convert "auto" query_type to "vector" or "fts"
-        # and convert the query to vector if needed
-        query, query_type = cls._resolve_query(
-            table, query, query_type, vector_column_name
-        )
-
-        if isinstance(query, str):
-            # fts
-            return LanceFtsQueryBuilder(table, query)
-
-        if isinstance(query, list):
-            query = np.array(query, dtype=np.float32)
-        elif isinstance(query, np.ndarray):
-            query = query.astype(np.float32)
-        else:
-            raise TypeError(f"Unsupported query type: {type(query)}")
-
-        return LanceVectorQueryBuilder(table, query, vector_column_name)
-
-    @classmethod
-    def _resolve_query(cls, table, query, query_type, vector_column_name):
-        # If query_type is fts, then query must be a string.
-        # otherwise raise TypeError
-        if query_type == "fts":
-            if not isinstance(query, str):
-                raise TypeError(f"'fts' queries must be a string: {type(query)}")
-            return query, query_type
-        elif query_type == "vector":
-            if not isinstance(query, (list, np.ndarray)):
-                conf = table.embedding_functions.get(vector_column_name)
-                if conf is not None:
-                    query = conf.function.compute_query_embeddings(query)[0]
-                else:
-                    msg = f"No embedding function for {vector_column_name}"
-                    raise ValueError(msg)
-            return query, query_type
-        elif query_type == "auto":
-            if isinstance(query, (list, np.ndarray)):
-                return query, "vector"
-            else:
-                conf = table.embedding_functions.get(vector_column_name)
-                if conf is not None:
-                    query = conf.function.compute_query_embeddings(query)[0]
-                    return query, "vector"
-                else:
-                    return query, "fts"
-        else:
-            raise ValueError(
-                f"Invalid query_type, must be 'vector', 'fts', or 'auto': {query_type}"
-            )
-
-    def __init__(self, table: "lancedb.table.Table"):
+        query: Union[np.ndarray, str],
+        vector_column: str = VECTOR_COLUMN_NAME,
+    ):
+        self._metric = "L2"
+        self._nprobes = 20
+        self._refine_factor = None
         self._table = table
+        self._query = query
         self._limit = 10
         self._columns = None
         self._where = None
-
-    @deprecation.deprecated(
-        deprecated_in="0.3.1",
-        removed_in="0.4.0",
-        current_version=__version__,
-        details="Use the bar function instead",
-    )
-    def to_df(self) -> "pd.DataFrame":
-        """
-        Deprecated alias for `to_pandas()`. Please use `to_pandas()` instead.
-
-        Execute the query and return the results as a pandas DataFrame.
-        In addition to the selected columns, LanceDB also returns a vector
-        and also the "_distance" column which is the distance between the query
-        vector and the returned vector.
-        """
-        return self.to_pandas()
-
-    def to_pandas(self) -> "pd.DataFrame":
-        """
-        Execute the query and return the results as a pandas DataFrame.
-        In addition to the selected columns, LanceDB also returns a vector
-        and also the "_distance" column which is the distance between the query
-        vector and the returned vector.
-        """
-        return self.to_arrow().to_pandas()
-
-    @abstractmethod
-    def to_arrow(self) -> pa.Table:
-        """
-        Execute the query and return the results as an
-        [Apache Arrow Table](https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table).
-
-        In addition to the selected columns, LanceDB also returns a vector
-        and also the "_distance" column which is the distance between the query
-        vector and the returned vectors.
-        """
-        raise NotImplementedError
-
-    def to_list(self) -> List[dict]:
-        """
-        Execute the query and return the results as a list of dictionaries.
-
-        Each list entry is a dictionary with the selected column names as keys,
-        or all table columns if `select` is not called. The vector and the "_distance"
-        fields are returned whether or not they're explicitly selected.
-        """
-        return self.to_arrow().to_pylist()
-
-    def to_pydantic(self, model: Type[LanceModel]) -> List[LanceModel]:
-        """Return the table as a list of pydantic models.
-
-        Parameters
-        ----------
-        model: Type[LanceModel]
-            The pydantic model to use.
-
-        Returns
-        -------
-        List[LanceModel]
-        """
-        return [
-            model(**{k: v for k, v in row.items() if k in model.field_names()})
-            for row in self.to_arrow().to_pylist()
-        ]
+        self._vector_column = vector_column
 
     def limit(self, limit: int) -> LanceQueryBuilder:
         """Set the maximum number of results to return.
@@ -226,7 +122,7 @@ class LanceQueryBuilder(ABC):
         self._columns = columns
         return self
 
-    def where(self, where) -> LanceQueryBuilder:
+    def where(self, where: str) -> LanceQueryBuilder:
         """Set the where clause.
 
         Parameters
@@ -242,45 +138,7 @@ class LanceQueryBuilder(ABC):
         self._where = where
         return self
 
-
-class LanceVectorQueryBuilder(LanceQueryBuilder):
-    """
-    A builder for nearest neighbor queries for LanceDB.
-
-    Examples
-    --------
-    >>> import lancedb
-    >>> data = [{"vector": [1.1, 1.2], "b": 2},
-    ...         {"vector": [0.5, 1.3], "b": 4},
-    ...         {"vector": [0.4, 0.4], "b": 6},
-    ...         {"vector": [0.4, 0.4], "b": 10}]
-    >>> db = lancedb.connect("./.lancedb")
-    >>> table = db.create_table("my_table", data=data)
-    >>> (table.search([0.4, 0.4])
-    ...       .metric("cosine")
-    ...       .where("b < 10")
-    ...       .select(["b"])
-    ...       .limit(2)
-    ...       .to_pandas())
-       b      vector  _distance
-    0  6  [0.4, 0.4]        0.0
-    """
-
-    def __init__(
-        self,
-        table: "lancedb.table.Table",
-        query: Union[np.ndarray, list, "PIL.Image.Image"],
-        vector_column: str = VECTOR_COLUMN_NAME,
-    ):
-        super().__init__(table)
-        self._query = query
-        self._metric = "L2"
-        self._nprobes = 20
-        self._refine_factor = None
-        self._vector_column = vector_column
-        self._prefilter = False
-
-    def metric(self, metric: Literal["L2", "cosine"]) -> LanceVectorQueryBuilder:
+    def metric(self, metric: Literal["L2", "cosine"]) -> LanceQueryBuilder:
         """Set the distance metric to use.
 
         Parameters
@@ -290,13 +148,13 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
 
         Returns
         -------
-        LanceVectorQueryBuilder
+        LanceQueryBuilder
             The LanceQueryBuilder object.
         """
         self._metric = metric
         return self
 
-    def nprobes(self, nprobes: int) -> LanceVectorQueryBuilder:
+    def nprobes(self, nprobes: int) -> LanceQueryBuilder:
         """Set the number of probes to use.
 
         Higher values will yield better recall (more likely to find vectors if
@@ -312,13 +170,13 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
 
         Returns
         -------
-        LanceVectorQueryBuilder
+        LanceQueryBuilder
             The LanceQueryBuilder object.
         """
         self._nprobes = nprobes
         return self
 
-    def refine_factor(self, refine_factor: int) -> LanceVectorQueryBuilder:
+    def refine_factor(self, refine_factor: int) -> LanceQueryBuilder:
         """Set the refine factor to use, increasing the number of vectors sampled.
 
         As an example, a refine factor of 2 will sample 2x as many vectors as
@@ -334,26 +192,35 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
 
         Returns
         -------
-        LanceVectorQueryBuilder
+        LanceQueryBuilder
             The LanceQueryBuilder object.
         """
         self._refine_factor = refine_factor
         return self
 
+    def to_df(self) -> pd.DataFrame:
+        """
+        Execute the query and return the results as a pandas DataFrame.
+        In addition to the selected columns, LanceDB also returns a vector
+        and also the "score" column which is the distance between the query
+        vector and the returned vector.
+        """
+
+        return self.to_arrow().to_pandas()
+
     def to_arrow(self) -> pa.Table:
         """
         Execute the query and return the results as an
         [Apache Arrow Table](https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table).
 
         In addition to the selected columns, LanceDB also returns a vector
-        and also the "_distance" column which is the distance between the query
+        and also the "score" column which is the distance between the query
         vector and the returned vectors.
         """
         vector = self._query if isinstance(self._query, list) else self._query.tolist()
         query = Query(
             vector=vector,
             filter=self._where,
-            prefilter=self._prefilter,
             k=self._limit,
             metric=self._metric,
             columns=self._columns,
@@ -363,37 +230,9 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
         )
         return self._table._execute_query(query)
 
-    def where(self, where: str, prefilter: bool = False) -> LanceVectorQueryBuilder:
-        """Set the where clause.
-
-        Parameters
-        ----------
-        where: str
-            The where clause.
-        prefilter: bool, default False
-            If True, apply the filter before vector search, otherwise the
-            filter is applied on the result of vector search.
-            This feature is **EXPERIMENTAL** and may be removed and modified
-            without warning in the future. Currently this is only supported
-            in OSS and can only be used with a table that does not have an ANN
-            index.
-
-        Returns
-        -------
-        LanceQueryBuilder
-            The LanceQueryBuilder object.
-        """
-        self._where = where
-        self._prefilter = prefilter
-        return self
-
 
 class LanceFtsQueryBuilder(LanceQueryBuilder):
-    def __init__(self, table: "lancedb.table.Table", query: str):
-        super().__init__(table)
-        self._query = query
-
-    def to_arrow(self) -> pa.Table:
+    def to_arrow(self) -> pd.Table:
         try:
             import tantivy
         except ImportError:
@@ -416,13 +255,3 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
         output_tbl = self._table.to_lance().take(row_ids, columns=self._columns)
         output_tbl = output_tbl.append_column("score", scores)
         return output_tbl
-
-
-class LanceEmptyQueryBuilder(LanceQueryBuilder):
-    def to_arrow(self) -> pa.Table:
-        ds = self._table.to_lance()
-        return ds.to_table(
-            columns=self._columns,
-            filter=self._where,
-            limit=self._limit,
-        )

python/lancedb/remote/__init__.py

@@ -14,7 +14,7 @@
 import abc
 from typing import List, Optional
 
-import attrs
+import attr
 import pyarrow as pa
 from pydantic import BaseModel
 
@@ -44,7 +44,7 @@ class VectorQuery(BaseModel):
     refine_factor: Optional[int] = None
 
 
-@attrs.define
+@attr.define
 class VectorQueryResult:
     # for now the response is directly seralized into a pandas dataframe
     tbl: pa.Table

python/lancedb/remote/client.py

@@ -16,7 +16,7 @@ import functools
 from typing import Any, Callable, Dict, Optional, Union
 
 import aiohttp
-import attrs
+import attr
 import pyarrow as pa
 from pydantic import BaseModel
 
@@ -43,21 +43,16 @@ async def _read_ipc(resp: aiohttp.ClientResponse) -> pa.Table:
         return reader.read_all()
 
 
-@attrs.define(slots=False)
+@attr.define(slots=False)
 class RestfulLanceDBClient:
     db_name: str
     region: str
     api_key: Credential
-    host_override: Optional[str] = attrs.field(default=None)
-
-    closed: bool = attrs.field(default=False, init=False)
+    closed: bool = attr.field(default=False, init=False)
 
     @functools.cached_property
     def session(self) -> aiohttp.ClientSession:
-        url = (
-            self.host_override
-            or f"https://{self.db_name}.{self.region}.api.lancedb.com"
-        )
+        url = f"https://{self.db_name}.{self.region}.api.lancedb.com"
         return aiohttp.ClientSession(url)
 
     async def close(self):
@@ -71,8 +66,6 @@ class RestfulLanceDBClient:
         }
         if self.region == "local":  # Local test mode
             headers["Host"] = f"{self.db_name}.{self.region}.api.lancedb.com"
-        if self.host_override:
-            headers["x-lancedb-database"] = self.db_name
         return headers
 
     @staticmethod
@@ -97,12 +90,7 @@ class RestfulLanceDBClient:
         """Send a GET request and returns the deserialized response payload."""
         if isinstance(params, BaseModel):
             params: Dict[str, Any] = params.dict(exclude_none=True)
-        async with self.session.get(
-            uri,
-            params=params,
-            headers=self.headers,
-            timeout=aiohttp.ClientTimeout(total=30),
-        ) as resp:
+        async with self.session.get(uri, params=params, headers=self.headers) as resp:
             await self._check_status(resp)
             return await resp.json()
 
@@ -110,11 +98,10 @@ class RestfulLanceDBClient:
     async def post(
         self,
         uri: str,
-        data: Optional[Union[Dict[str, Any], BaseModel, bytes]] = None,
+        data: Union[Dict[str, Any], BaseModel, bytes],
         params: Optional[Dict[str, Any]] = None,
         content_type: Optional[str] = None,
         deserialize: Callable = lambda resp: resp.json(),
-        request_id: Optional[str] = None,
     ) -> Dict[str, Any]:
         """Send a POST request and returns the deserialized response payload.
 
@@ -123,8 +110,6 @@ class RestfulLanceDBClient:
         uri : str
             The uri to send the POST request to.
         data: Union[Dict[str, Any], BaseModel]
-        request_id: Optional[str]
-            Optional client side request id to be sent in the request headers.
 
         """
         if isinstance(data, BaseModel):
@@ -137,13 +122,10 @@ class RestfulLanceDBClient:
         headers = self.headers.copy()
         if content_type is not None:
             headers["content-type"] = content_type
-        if request_id is not None:
-            headers["x-request-id"] = request_id
         async with self.session.post(
             uri,
             headers=headers,
             params=params,
-            timeout=aiohttp.ClientTimeout(total=30),
             **req_kwargs,
         ) as resp:
             resp: aiohttp.ClientResponse = resp
@@ -159,7 +141,5 @@ class RestfulLanceDBClient:
     @_check_not_closed
     async def query(self, table_name: str, query: VectorQuery) -> VectorQueryResult:
         """Query a table."""
-        tbl = await self.post(
-            f"/v1/table/{table_name}/query/", query, deserialize=_read_ipc
-        )
+        tbl = await self.post(f"/v1/table/{table_name}/", query, deserialize=_read_ipc)
         return VectorQueryResult(tbl)

python/lancedb/remote/db.py

@@ -13,14 +13,16 @@
 
 import asyncio
 import uuid
-from typing import List, Optional
+from typing import List
 from urllib.parse import urlparse
 
 import pyarrow as pa
 
-from ..common import DATA
-from ..db import DBConnection
-from ..table import Table, _sanitize_data
+from lancedb.common import DATA
+from lancedb.db import DBConnection
+from lancedb.schema import schema_to_json
+from lancedb.table import Table, _sanitize_data
+
 from .arrow import to_ipc_binary
 from .client import ARROW_STREAM_CONTENT_TYPE, RestfulLanceDBClient
 
@@ -28,22 +30,14 @@ from .client import ARROW_STREAM_CONTENT_TYPE, RestfulLanceDBClient
 class RemoteDBConnection(DBConnection):
     """A connection to a remote LanceDB database."""
 
-    def __init__(
-        self,
-        db_url: str,
-        api_key: str,
-        region: str,
-        host_override: Optional[str] = None,
-    ):
+    def __init__(self, db_url: str, api_key: str, region: str):
         """Connect to a remote LanceDB database."""
         parsed = urlparse(db_url)
         if parsed.scheme != "db":
             raise ValueError(f"Invalid scheme: {parsed.scheme}, only accepts db://")
         self.db_name = parsed.netloc
         self.api_key = api_key
-        self._client = RestfulLanceDBClient(
-            self.db_name, region, api_key, host_override
-        )
+        self._client = RestfulLanceDBClient(self.db_name, region, api_key)
         try:
             self._loop = asyncio.get_running_loop()
         except RuntimeError:
@@ -101,24 +95,10 @@ class RemoteDBConnection(DBConnection):
 
         self._loop.run_until_complete(
             self._client.post(
-                f"/v1/table/{name}/create/",
+                f"/v1/table/{name}/create",
                 data=data,
-                request_id=request_id,
+                params={"request_id": request_id},
                 content_type=ARROW_STREAM_CONTENT_TYPE,
             )
         )
         return RemoteTable(self, name)
-
-    def drop_table(self, name: str):
-        """Drop a table from the database.
-
-        Parameters
-        ----------
-        name: str
-            The name of the table.
-        """
-        self._loop.run_until_complete(
-            self._client.post(
-                f"/v1/table/{name}/drop/",
-            )
-        )

python/lancedb/remote/table.py

@@ -13,14 +13,14 @@
 
 import uuid
 from functools import cached_property
-from typing import Optional, Union
+from typing import Union
 
 import pyarrow as pa
-from lance import json_to_schema
 
 from lancedb.common import DATA, VEC, VECTOR_COLUMN_NAME
 
-from ..query import LanceVectorQueryBuilder
+from ..query import LanceQueryBuilder, Query
+from ..schema import json_to_schema
 from ..table import Query, Table, _sanitize_data
 from .arrow import to_ipc_binary
 from .client import ARROW_STREAM_CONTENT_TYPE
@@ -33,27 +33,19 @@ class RemoteTable(Table):
         self._name = name
 
     def __repr__(self) -> str:
-        return f"RemoteTable({self._conn.db_name}.{self._name})"
+        return f"RemoteTable({self._conn.db_name}.{self.name})"
 
     @cached_property
     def schema(self) -> pa.Schema:
         """Return the schema of the table."""
         resp = self._conn._loop.run_until_complete(
-            self._conn._client.post(f"/v1/table/{self._name}/describe/")
+            self._conn._client.get(f"/v1/table/{self._name}/describe")
         )
         schema = json_to_schema(resp["schema"])
         return schema
 
     def to_arrow(self) -> pa.Table:
-        """Return the table as an Arrow table."""
-        raise NotImplementedError("to_arrow() is not supported on the LanceDB cloud")
-
-    def to_pandas(self):
-        """Return the table as a Pandas DataFrame.
-
-        Intercept `to_arrow()` for better error message.
-        """
-        return NotImplementedError("to_pandas() is not supported on the LanceDB cloud")
+        raise NotImplementedError
 
     def create_index(
         self,
@@ -62,7 +54,6 @@ class RemoteTable(Table):
         num_sub_vectors=96,
         vector_column_name: str = VECTOR_COLUMN_NAME,
         replace: bool = True,
-        accelerator: Optional[str] = None,
     ):
         raise NotImplementedError
 
@@ -74,11 +65,7 @@ class RemoteTable(Table):
         fill_value: float = 0.0,
     ) -> int:
         data = _sanitize_data(
-            data,
-            self.schema,
-            metadata=None,
-            on_bad_vectors=on_bad_vectors,
-            fill_value=fill_value,
+            data, self.schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
         )
         payload = to_ipc_binary(data)
 
@@ -86,23 +73,19 @@ class RemoteTable(Table):
 
         self._conn._loop.run_until_complete(
             self._conn._client.post(
-                f"/v1/table/{self._name}/insert/",
+                f"/v1/table/{self._name}/insert",
                 data=payload,
                 params={"request_id": request_id, "mode": mode},
                 content_type=ARROW_STREAM_CONTENT_TYPE,
             )
         )
+        return len(data)
 
     def search(
-        self, query: Union[VEC, str], vector_column_name: str = VECTOR_COLUMN_NAME
-    ) -> LanceVectorQueryBuilder:
-        return LanceVectorQueryBuilder(self, query, vector_column_name)
+        self, query: Union[VEC, str], vector_column: str = VECTOR_COLUMN_NAME
+    ) -> LanceQueryBuilder:
+        return LanceQueryBuilder(self, query, vector_column)
 
     def _execute_query(self, query: Query) -> pa.Table:
-        if query.prefilter:
-            raise NotImplementedError("Cloud support for prefiltering is coming soon")
         result = self._conn._client.query(self._name, query)
         return self._conn._loop.run_until_complete(result).to_arrow()
-
-    def delete(self, predicate: str):
-        raise NotImplementedError