try manylinux again

.github/workflows/node.yml

@@ -67,8 +67,10 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run build
         npm run tsc
+        npm run build
+        npm run pack-build
+        npm install --no-save ./dist/vectordb-*.tgz
     - name: Test
       run: npm run test
   macos:
@@ -94,8 +96,10 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run build
         npm run tsc
+        npm run build
+        npm run pack-build
+        npm install --no-save ./dist/vectordb-*.tgz
     - name: Test
       run: |
         npm run test

.github/workflows/python.yml

@@ -31,8 +31,7 @@ jobs:
     - name: Install lancedb
       run: |
         pip install -e .
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock
+        pip install pytest
     - name: Run tests
       run: pytest -x -v --durations=30 tests
   mac:
@@ -50,11 +49,10 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.11"
+        python-version: "3.10"
     - name: Install lancedb
       run: |
         pip install -e .
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock
+        pip install pytest
     - name: Run tests
       run: pytest -x -v --durations=30 tests

.github/workflows/release.yml

@@ -0,0 +1,205 @@
+name: Prepare Release
+
+# Based on https://github.com/dherman/neon-prebuild-example/blob/eaa4d33d682e5eb7abbc3da7aed153a1b1acb1b3/.github/workflows/publish.yml
+
+on:
+  push:
+    tags:
+      - v*
+
+jobs:
+  draft-release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          prerelease: true # hardcoded on for now
+          generate_release_notes: true
+  
+  rust:
+    runs-on: ubuntu-latest
+    needs: draft-release
+    defaults:
+      run:
+        shell: bash
+        working-directory: rust/vectordb
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Package Rust
+        run: cargo package --all-features
+      - uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          files: target/vectordb-*.crate
+    
+  python:
+    runs-on: ubuntu-latest
+    needs: draft-release
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Build wheel
+        run: |
+          pip install wheel
+          python setup.py sdist bdist_wheel
+      - uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          files: |
+            python/dist/lancedb-*.tar.gz 
+            python/dist/lancedb-*.whl
+
+  node:
+    runs-on: ubuntu-latest
+    needs: draft-release
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          cache: 'npm'
+          cache-dependency-path: node/package-lock.json
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Build
+        run: |
+          npm ci
+          npm run tsc
+          npm pack
+      - uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          files: node/vectordb-*.tgz
+
+  node-macos:
+    runs-on: macos-12
+    needs: draft-release
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64-apple-darwin, aarch64-apple-darwin]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Install system dependencies
+      run: brew install protobuf
+    - name: Install npm dependencies
+      run: |
+        cd node
+        npm ci
+    - name: Build MacOS native node modules
+      run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
+    - uses: softprops/action-gh-release@v1
+      with:
+        draft: true
+        files: node/dist/vectordb-darwin*.tgz
+
+  node-linux:
+    name: Linux ${{ matrix.settings.arch}} native node module
+    runs-on: ubuntu-latest
+    needs: draft-release
+    strategy:
+      fail-fast: false
+      matrix:
+        libc:
+          - gnu
+          - musl
+        settings:
+          - container: quay.io/pypa/manylinux2014_x86_64
+            arch: x86_64
+            protoc_arch: x86_64
+          # TODO: find a way to build aarch64 on GA
+          # - container:  quay.io/pypa/manylinux2014_aarch64
+          #   arch: aarch64
+          #   protoc_arch: aarch_64
+    container:
+      image: ${{ matrix.settings.container }}
+      options: --platform linux/amd64
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Install system dependencies
+      run: |
+        yum install -y openssl-devel unzip
+        # Install new enough protobuf (yum-provided is old)
+        PB_REL=https://github.com/protocolbuffers/protobuf/releases
+        PB_VERSION=23.1
+        curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-${{ matrix.settings.protoc_arch }}.zip
+        unzip protoc-$PB_VERSION-linux-${{ matrix.settings.protoc_arch }}.zip -d /usr/local
+    - name: Setup Node
+      uses: actions/setup-node@v3
+      with:
+        node-version: 17 # Last that support glibc 2.17
+    - name: Setup Rust
+      run: |
+        curl https://sh.rustup.rs -sSf | bash -s -- -y
+        echo "/root/.cargo/bin" >> $GITHUB_PATH
+    - name: Install npm dependencies
+      run: npm ci
+    - name: Build ${{ matrix.settings.libc }} library
+      run: |
+        BUILD_TARGET=${{ matrix.settings.arch }}-unknown-linux-${{ matrix.settings.libc }}
+        rustup target add $BUILD_TARGET
+        export RUSTFLAGS="-C target-feature=-crt-static" # for musl
+        npm run build-release -- --target $BUILD_TARGET
+        npm run pack-build -- --target $BUILD_TARGET
+    - uses: softprops/action-gh-release@v1
+      with:
+        draft: true
+        files: node/dist/vectordb-linux*.tgz
+
+  release:
+    needs: [python, node, node-macos, node-linux, rust]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v3
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: |
+          python -m twine upload --non-interactive \
+            --skip-existing \
+            --repository testpypi python/dist/*
+      - name: Publish to NPM
+        run: |
+          for filename in node/dist/*.tgz; do
+            npm publish --dry-run $filename
+          done
+      - name: Publish to crates.io
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+        run: |
+          cargo publish --dry-run --no-verify rust/target/vectordb-*.crate
+      # - uses: softprops/action-gh-release@v1
+      #   with:
+      #     draft: false

.gitignore

@@ -4,6 +4,8 @@
 **/__pycache__
 .DS_Store
 
+.vscode
+
 rust/target
 rust/Cargo.lock
 
@@ -15,7 +17,7 @@ site
 python/build
 python/dist
 
-**/.ipynb_checkpoints
+notebooks/.ipynb_checkpoints
 
 **/.hypothesis
 

Cargo.lock

@@ -1052,7 +1052,6 @@ dependencies = [
  "paste",
  "petgraph",
  "rand",
- "regex",
  "uuid",
 ]
 
@@ -1646,9 +1645,9 @@ dependencies = [
 
 [[package]]
 name = "lance"
-version = "0.4.17"
+version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86dda8185bd1ffae7b910c1f68035af23be9b717c52e9cc4de176cd30b47f772"
+checksum = "fc96cf89139af6f439a0e28ccd04ddf81be795b79fda3105b7a8952fadeb778e"
 dependencies = [
  "accelerate-src",
  "arrow",
@@ -1685,7 +1684,6 @@ dependencies = [
  "rand",
  "reqwest",
  "shellexpand",
- "snafu",
  "sqlparser-lance",
  "tokio",
  "url",
@@ -3358,22 +3356,18 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
 [[package]]
 name = "vectordb"
-version = "0.0.1"
+version = "0.1.2"
 dependencies = [
  "arrow-array",
- "arrow-data",
  "arrow-schema",
  "lance",
- "object_store",
- "rand",
- "snafu",
  "tempfile",
  "tokio",
 ]
 
 [[package]]
 name = "vectordb-node"
-version = "0.1.0"
+version = "0.1.2"
 dependencies = [
  "arrow-array",
  "arrow-ipc",

Cross.toml

@@ -0,0 +1,39 @@
+# These make sure our builds are compatible with old glibc versions.
+[target.x86_64-unknown-linux-gnu]
+pre-build = [
+    # Install newer gfortran
+    "yum install -y openssl-devel unzip gcc-gfortran",
+    "scl enable devtoolset-11 bash",
+    # protobuf is too old, so we directly download binaries
+    "PB_REL=https://github.com/protocolbuffers/protobuf/releases",
+    "PB_VERSION=23.1",
+    "curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip",
+    "unzip protoc-$PB_VERSION-linux-x86_64.zip -d /usr/local",
+]
+image = "ghcr.io/cross-rs/x86_64-unknown-linux-gnu:main-centos"
+
+[target.aarch64-unknown-linux-gnu]
+pre-build = [
+    "yum install -y openssl-devel unzip",
+    # protobuf is too old, so we directly download binaries
+    "PB_REL=https://github.com/protocolbuffers/protobuf/releases",
+    "PB_VERSION=23.1",
+    "curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip",
+    "unzip protoc-$PB_VERSION-linux-x86_64.zip -d /usr/local",
+]
+# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.aarch64-unknown-linux-gnu.centos
+image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main-centos"
+
+[target.x86_64-unknown-linux-musl]
+# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.x86_64-unknown-linux-musl
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH",
+]
+
+[target.aarch64-unknown-linux-musl]
+# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.aarch64-unknown-linux-musl
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH",
+]

README.md

@@ -10,10 +10,6 @@
 <a href="https://discord.gg/zMM32dvNtd">Discord</a> •
 <a href="https://twitter.com/lancedb">Twitter</a>
 
-</p>
-
-<img max-width="750px" alt="LanceDB Multimodal Search" src="https://github.com/lancedb/lancedb/assets/917119/09c5afc5-7816-4687-bae4-f2ca194426ec">
-
 </p>
 </div>
 
@@ -27,15 +23,13 @@ The key features of LanceDB include:
 
 * Store, query and filter vectors, metadata and multi-modal data (text, images, videos, point clouds, and more).
 
-* Support for vector similarity search, full-text search and SQL.
-
 * Native Python and Javascript/Typescript support.
 
 * Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure.
 
 * Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lanecdb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
 
-LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
+LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/eto-ai/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
 
 ## Quick Start
 

ci/build_linux_artifacts.sh

@@ -0,0 +1,40 @@
+# Builds the Linux artifacts (node binaries).
+# Usage: ./build_linux_artifacts.sh [target]
+# Targets supported: 
+# - x86_64-unknown-linux-gnu:centos
+# - aarch64-unknown-linux-gnu:centos
+# - aarch64-unknown-linux-musl
+# - x86_64-unknown-linux-musl
+
+# On MacOS, need to run in a linux container:
+# docker run -v $(pwd):/io -w /io 
+
+# Must run rustup toolchain install stable-x86_64-unknown-linux-gnu --force-non-host
+
+set -e
+
+build_node_binaries() {
+    pushd node
+    
+    for target in $1
+    do
+        echo "Building node library for $target"
+        # cross doesn't yet pass this down to Docker, so we do it ourselves.
+        export CROSS_CONTAINER_OPTS="--platform linux/amd64"
+        if [[ $target == *musl ]]; then
+            # This is needed for cargo to allow build cdylibs with musl
+            RUSTFLAGS="-C target-feature=-crt-static"
+        fi
+        npm run cross-release -- --target $target
+        npm run pack-build -- --target $target
+    done
+    popd
+}
+
+if [ -n "$1" ]; then
+    targets=$1
+else
+    # targets="x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu aarch64-unknown-linux-musl x86_64-unknown-linux-musl"
+    targets="aarch64-unknown-linux-gnu"
+fi
+build_node_binaries $targets

ci/build_macos_artifacts.sh

@@ -0,0 +1,22 @@
+# Builds the macOS artifacts (node binaries).
+# Usage: ./build_macos_artifacts.sh [target]
+# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
+
+build_node_binaries() {
+    pushd node
+    
+    for target in $1
+    do
+        echo "Building node library for $target"
+        npm run build-release -- --target $target
+        npm run pack-build -- --target $target
+    done
+    popd
+}
+
+if [ -n "$1" ]; then
+    targets=$1
+else
+    targets="x86_64-apple-darwin aarch64-apple-darwin"
+fi
+build_node_binaries $targets

ci/ubuntu_build.dockerfile

@@ -0,0 +1,31 @@
+# On MacOS, need to run in a linux container:
+# cat ci/ubuntu_build.dockerfile | docker build -t lancedb-node-build -
+# docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd):/io -w /io lancedb-node-build bash ci/build_linux_artifacts.sh
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/Moscow
+
+RUN apt update && apt install -y protobuf-compiler libssl-dev build-essential curl \
+    software-properties-common npm docker.io
+
+# Install rust
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install cross
+# https://github.com/cross-rs/cross/issues/1257#issuecomment-1544553706
+RUN cargo install cross --git https://github.com/cross-rs/cross
+
+# Install additional build targets
+RUN rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu aarch64-unknown-linux-musl x86_64-unknown-linux-musl
+
+# Install node
+RUN npm install npm@latest -g && \
+    npm install n -g && \
+    n latest
+
+# set CROSS_CONTAINER_IN_CONTAINER to inform `cross` that it is executed from within a container
+ENV CROSS_CONTAINER_IN_CONTAINER=true
+ENV CROSS_CONTAINER_ENGINE_NO_BUILDKIT=1

docs/mkdocs.yml

@@ -1,16 +1,10 @@
-site_name: LanceDB Docs
-repo_url: https://github.com/lancedb/lancedb
-repo_name: lancedb/lancedb
+site_name: LanceDB Documentation
 docs_dir: src
 
 theme:
   name: "material"
-  logo: assets/logo.png
   features:
     - content.code.copy
-    - content.tabs.link
-  icon:
-    repo: fontawesome/brands/github
 
 plugins:
 - search
@@ -20,36 +14,19 @@ plugins:
         paths: [../python]
 - mkdocs-jupyter
 
+nav:
+- Home: index.md
+- Basics: basic.md
+- Embeddings: embedding.md
+- Indexing: ann_indexes.md
+- Integrations: integrations.md
+- Python API: python.md
+
 markdown_extensions:
-- admonition
-- pymdownx.superfences
-- pymdownx.details
 - pymdownx.highlight:
     anchor_linenums: true
     line_spans: __span
     pygments_lang_class: true
 - pymdownx.inlinehilite
 - pymdownx.snippets
-- pymdownx.superfences
-- pymdownx.tabbed:
-    alternate_style: true
-
-nav:
-- Home: index.md
-- Basics: basic.md
-- Embeddings: embedding.md
-- Python full-text search: fts.md
-- Python integrations: integrations.md
-- Python examples:
-  - YouTube Transcript Search using OpenAI: notebooks/youtube_transcript_search.ipynb
-  - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
-  - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
-- References:
-  - Vector Search: search.md
-  - Indexing: ann_indexes.md
-- API references:
-  - Python API: python/python.md
-  - Javascript API: javascript/modules.md
-
-extra_css:
-  - styles/global.css
+- pymdownx.superfences

docs/src/ann_indexes.md

@@ -12,43 +12,29 @@ In the future we will look to automatically create and configure the ANN index.
 
 ## Creating an ANN Index
 
-=== "Python"
-     Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
+Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
 
-     ```python
-     import lancedb
-     import numpy as np
-     uri = "data/sample-lancedb"
-     db = lancedb.connect(uri)
+```python
+import lancedb
+import numpy as np
+uri = "~/.lancedb"
+db = lancedb.connect(uri)
 
-     # Create 10,000 sample vectors
-     data = [{"vector": row, "item": f"item {i}"}
-        for i, row in enumerate(np.random.random((10_000, 768)).astype('float32'))]
+# Create 10,000 sample vectors
+data = [{"vector": row, "item": f"item {i}"}
+   for i, row in enumerate(np.random.random((10_000, 768)).astype('float32'))]
 
-     # Add the vectors to a table
-     tbl = db.create_table("my_vectors", data=data)
+# Add the vectors to a table
+tbl = db.create_table("my_vectors", data=data)
 
-     # Create and train the index - you need to have enough data in the table for an effective training step
-     tbl.create_index(num_partitions=256, num_sub_vectors=96)
-     ```
-
-=== "Javascript"
-     ```javascript
-     const vectordb = require('vectordb')
-     const db = await vectordb.connect('data/sample-lancedb')
-
-     let data = []
-     for (let i = 0; i < 10_000; i++) {
-         data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
-     }
-     const table = await db.createTable('vectors', data)
-     await table.create_index({ type: 'ivf_pq', column: 'vector', num_partitions: 256, num_sub_vectors: 96 })
-     ```
+# Create and train the index - you need to have enough data in the table for an effective training step
+tbl.create_index(num_partitions=256, num_sub_vectors=96)
+```
 
 Since `create_index` has a training step, it can take a few minutes to finish for large tables. You can control the index
 creation by providing the following parameters:
 
-- **metric** (default: "L2"): The distance metric to use. By default we use euclidean distance. We also support "cosine" distance.
+- **metric** (default: "L2"): The distance metric to use. By default we use euclidean distance. We also support cosine distance.
 - **num_partitions** (default: 256): The number of partitions of the index. The number of partitions should be configured so each partition has 3-5K vectors. For example, a table
 with ~1M vectors should use 256 partitions. You can specify arbitrary number of partitions but powers of 2 is most conventional.
 A higher number leads to faster queries, but it makes index generation slower.
@@ -71,28 +57,18 @@ There are a couple of parameters that can be used to fine-tune the search:
   e.g., for 1M vectors divided into 256 partitions, if you're looking for top 20, then refine_factor=200 reranks the whole partition.<br/>
   Note: refine_factor is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
 
-=== "Python"
-     ```python
-     tbl.search(np.random.random((768))) \
-         .limit(2) \
-         .nprobes(20) \
-         .refine_factor(10) \
-         .to_df()
+
+```python
+tbl.search(np.random.random((768))) \
+    .limit(2) \
+    .nprobes(20) \
+    .refine_factor(10) \
+    .to_df()
 
                                               vector       item       score
-     0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
-     1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
-     ```
-
-=== "Javascript"
-     ```javascript
-     const results = await table
-         .search(Array(768).fill(1.2))
-         .limit(2)
-         .nprobes(20)
-         .refineFactor(10)
-         .execute()
-     ```
+0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
+1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
+```
 
 The search will return the data requested in addition to the score of each item.
 
@@ -102,36 +78,18 @@ The search will return the data requested in addition to the score of each item.
 
 You can further filter the elements returned by a search using a where clause.
 
-=== "Python"
-     ```python
-     tbl.search(np.random.random((768))).where("item != 'item 1141'").to_df()
-     ```
-
-=== "Javascript"
-     ```javascript
-     const results = await table
-         .search(Array(1536).fill(1.2))
-         .where("item != 'item 1141'")
-         .execute()
-     ```
+```python
+tbl.search(np.random.random((768))).where("item != 'item 1141'").to_df()
+```
 
 ### Projections (select clause)
 
 You can select the columns returned by the query using a select clause.
 
-=== "Python"
-     ```python
-     tbl.search(np.random.random((768))).select(["vector"]).to_df()
-     vector      score
-     0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
-     1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
-     ...
-     ```
-
-=== "Javascript"
-     ```javascript
-     const results = await table
-         .search(Array(1536).fill(1.2))
-         .select(["id"])
-         .execute()
-     ```
+```python
+tbl.search(np.random.random((768))).select(["vector"]).to_df()
+                                              vector      score
+0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
+1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
+...
+```

docs/src/basic.md

@@ -1,142 +1,74 @@
 # Basic LanceDB Functionality
 
-We'll cover the basics of using LanceDB on your local machine in this section.
-
-??? info "LanceDB runs embedded on your backend application, so there is no need to run a separate server."
-
-      <img src="../assets/lancedb_embedded_explanation.png" width="650px" />
-
-## Installation
-
-=== "Python"
-      ```shell
-      pip install lancedb
-      ```
-
-=== "Javascript"
-      ```shell
-      npm install vectordb
-      ```
-
 ## How to connect to a database
 
-=== "Python"
-      ```python
-      import lancedb
-      uri = "~/.lancedb"
-      db = lancedb.connect(uri)
-      ```
+In local mode, LanceDB stores data in a directory on your local machine. To connect to a local database, you can use the following code:
+```python
+import lancedb
+uri = "~/.lancedb"
+db = lancedb.connect(uri)
+```
 
-      LanceDB will create the directory if it doesn't exist (including parent directories).
+LanceDB will create the directory if it doesn't exist (including parent directories).
 
-      If you need a reminder of the uri, use the `db.uri` property.
-
-=== "Javascript"
-      ```javascript
-      const lancedb = require("vectordb");
-
-      const uri = "~./lancedb";
-      const db = await lancedb.connect(uri);
-      ```
-      
-      LanceDB will create the directory if it doesn't exist (including parent directories).
-
-      If you need a reminder of the uri, you can call `db.uri()`.
+If you need a reminder of the uri, use the `db.uri` property.
 
 ## How to create a table
 
-=== "Python"
-      ```python
-      tbl = db.create_table("my_table",
-                        data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                              {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      ```
+To create a table, you can use the following code:
+```python
+tbl = db.create_table("my_table",
+                      data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                            {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+```
 
-      If the table already exists, LanceDB will raise an error by default.
-      If you want to overwrite the table, you can pass in `mode="overwrite"`
-      to the `create_table` method.
+Under the hood, LanceDB is converting the input data into an Apache Arrow table
+and persisting it to disk in [Lance format](github.com/eto-ai/lance).
 
-      You can also pass in a pandas DataFrame directly:
-      ```python
-      import pandas as pd
-      df = pd.DataFrame([{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      tbl = db.create_table("table_from_df", data=df)
-      ```
+If the table already exists, LanceDB will raise an error by default.
+If you want to overwrite the table, you can pass in `mode="overwrite"`
+to the `create_table` method.
 
-=== "Javascript"
-      ```javascript
-      const tb = await db.createTable("my_table",
-                        data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                              {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      ```
-      
-!!! warning
-
-      If the table already exists, LanceDB will raise an error by default.
-      If you want to overwrite the table, you can pass in `mode="overwrite"`
-      to the `createTable` function.
-
-??? info "Under the hood, LanceDB is converting the input data into an Apache Arrow table and persisting it to disk in [Lance format](https://www.github.com/lancedb/lance)."
+You can also pass in a pandas DataFrame directly:
+```python
+import pandas as pd
+df = pd.DataFrame([{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                   {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+tbl = db.create_table("table_from_df", data=df)
+```
 
 ## How to open an existing table
 
 Once created, you can open a table using the following code:
+```python
+tbl = db.open_table("my_table")
+```
 
-=== "Python"
-      ```python
-      tbl = db.open_table("my_table")
-      ```
+If you forget the name of your table, you can always get a listing of all table names:
 
-      If you forget the name of your table, you can always get a listing of all table names:
-
-      ```python
-      print(db.table_names())
-      ```
-
-=== "Javascript"
-      ```javascript
-      const tbl = await db.openTable("my_table");
-      ```
-
-      If you forget the name of your table, you can always get a listing of all table names:
-
-      ```javascript
-      console.log(db.tableNames());
-      ```
+```python
+db.table_names()
+```
 
 ## How to add data to a table
 
 After a table has been created, you can always add more data to it using
 
-=== "Python"
-      ```python
-      df = pd.DataFrame([{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
-                        {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}])
-      tbl.add(df)
-      ```
-
-=== "Javascript"
-      ```javascript
-      await tbl.add([vector: [1.3, 1.4], item: "fizz", price: 100.0},
-              {vector: [9.5, 56.2], item: "buzz", price: 200.0}])
-      ```
+```python
+df = pd.DataFrame([{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
+                   {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}])
+tbl.add(df)
+```
 
 ## How to search for (approximate) nearest neighbors
 
 Once you've embedded the query, you can find its nearest neighbors using the following code:
 
-=== "Python"
-      ```python
-      tbl.search([100, 100]).limit(2).to_df()
-      ```
+```python
+tbl.search([100, 100]).limit(2).to_df()
+```
 
-      This returns a pandas DataFrame with the results.
-
-=== "Javascript"
-      ```javascript
-      const query = await tbl.search([100, 100]).limit(2).execute();
-      ```
+This returns a pandas DataFrame with the results.
 
 ## What's next
 

docs/src/embedding.md

@@ -25,88 +25,55 @@ def embed_func(batch):
     return [model.encode(sentence) for sentence in batch]
 ```
 
-Please note that currently HuggingFace is only supported in the Python SDK.
-
 ### OpenAI example
 
 You can also use an external API like OpenAI to generate embeddings
 
-=== "Python"
-      ```python
-        import openai
-        import os
+```python
+import openai
+import os
 
-        # Configuring the environment variable OPENAI_API_KEY
-        if "OPENAI_API_KEY" not in os.environ:
-        # OR set the key here as a variable
-        openai.api_key = "sk-..."
+# Configuring the environment variable OPENAI_API_KEY
+if "OPENAI_API_KEY" not in os.environ:
+    # OR set the key here as a variable
+    openai.api_key = "sk-..."
 
-        # verify that the API key is working
-        assert len(openai.Model.list()["data"]) > 0
+# verify that the API key is working
+assert len(openai.Model.list()["data"]) > 0
 
-        def embed_func(c):
-        rs = openai.Embedding.create(input=c, engine="text-embedding-ada-002")
-        return [record["embedding"] for record in rs["data"]]
-      ```
-
-=== "Javascript"
-      ```javascript
-        const lancedb = require("vectordb");
-
-        // You need to provide an OpenAI API key
-        const apiKey = "sk-..."
-        // The embedding function will create embeddings for the 'text' column
-        const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
-      ```
+def embed_func(c):
+    rs = openai.Embedding.create(input=c, engine="text-embedding-ada-002")
+    return [record["embedding"] for record in rs["data"]]
+```
 
 ## Applying an embedding function
 
-=== "Python"
-    Using an embedding function, you can apply it to raw data
-    to generate embeddings for each row.
+Using an embedding function, you can apply it to raw data
+to generate embeddings for each row.
 
-    Say if you have a pandas DataFrame with a `text` column that you want to be embedded,
-    you can use the [with_embeddings](https://lancedb.github.io/lancedb/python/#lancedb.embeddings.with_embeddings)
-    function to generate embeddings and add create a combined pyarrow table:
+Say if you have a pandas DataFrame with a `text` column that you want to be embedded,
+you can use the [with_embeddings](https://lancedb.github.io/lancedb/python/#lancedb.embeddings.with_embeddings)
+function to generate embeddings and add create a combined pyarrow table:
 
+```python
+import pandas as pd
+from lancedb.embeddings import with_embeddings
 
-     ```python
-     import pandas as pd
-     from lancedb.embeddings import with_embeddings
+df = pd.DataFrame([{"text": "pepperoni"},
+                   {"text": "pineapple"}])
+data = with_embeddings(embed_func, df)
 
-     df = pd.DataFrame([{"text": "pepperoni"},
-                        {"text": "pineapple"}])
-     data = with_embeddings(embed_func, df)
+# The output is used to create / append to a table
+# db.create_table("my_table", data=data)
+```
 
-     # The output is used to create / append to a table
-     # db.create_table("my_table", data=data)
-     ```
+If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
 
-     If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
-
-     By default, LanceDB calls the function with batches of 1000 rows. This can be configured
-     using the `batch_size` parameter to `with_embeddings`.
-
-     LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
-     API call is reliable.
-
-=== "Javascript"
-     Using an embedding function, you can apply it to raw data
-     to generate embeddings for each row.
-
-     You can just pass the embedding function created previously and LanceDB will automatically generate
-     embededings for your data.
-
-      ```javascript
-      const db = await lancedb.connect("/tmp/lancedb");
-      const data = [
-        { text: 'pepperoni'  },
-        { text: 'pineapple' }
-      ]
-
-      const table = await db.createTable('vectors', data, embedding)
-      ```
+By default, LanceDB calls the function with batches of 1000 rows. This can be configured
+using the `batch_size` parameter to `with_embeddings`.
 
+LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
+API call is reliable.
 
 ## Searching with an embedding function
 
@@ -114,25 +81,13 @@ At inference time, you also need the same embedding function to embed your query
 It's important that you use the same model / function otherwise the embedding vectors don't
 belong in the same latent space and your results will be nonsensical.
 
-=== "Python"
-     ```python
-     query = "What's the best pizza topping?"
-     query_vector = embed_func([query])[0]
-     tbl.search(query_vector).limit(10).to_df()
-     ```
-
-     The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
-
-=== "Javascript"
-     ```javascript
-      const results = await table
-        .search('What's the best pizza topping?')
-        .limit(10)
-        .execute()
-     ```
-
-     The above snippet returns an array of records with the 10 closest vectors to the query.
+```python
+query = "What's the best pizza topping?"
+query_vector = embed_func([query])[0]
+tbl.search(query_vector).limit(10).to_df()
+```
 
+The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
 
 ## Roadmap
 

docs/src/examples/langchain.md

@@ -4,4 +4,4 @@
 
 <img id="splash" width="400" alt="langchain" src="https://user-images.githubusercontent.com/917119/236580868-61a246a9-e587-4c2b-8ae5-6fe5f7b7e81e.png">
 
-This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/code_qa_bot.ipynb)
+This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/notebooks/code_qa_bot.ipynb)

docs/src/examples/modal_langchain.md

@@ -1,166 +0,0 @@
-# Serverless QA Bot with Modal and LangChain
-
-## use LanceDB's LangChain integration with Modal to run a serverless app
-
-<img id="splash" width="400" alt="modal" src="https://github.com/lancedb/lancedb/assets/917119/7d80a40f-60d7-48a6-972f-dab05000eccf">
-
-We're going to build a QA bot for your documentation using LanceDB's LangChain integration and use Modal for deployment.
-
-Modal is an end-to-end compute platform for model inference, batch jobs, task queues, web apps and more. It's a great way to deploy your LanceDB models and apps.
-
-To get started, ensure that you have created an account and logged into [Modal](https://modal.com/). To follow along, the full source code is available on Github [here](https://github.com/lancedb/lancedb/blob/main/docs/src/examples/modal_langchain.py).
-
-### Setting up Modal
-
-We'll start by specifying our dependencies and creating a new Modal `Stub`:
-
-```python
-lancedb_image = Image.debian_slim().pip_install(
-    "lancedb",
-    "langchain",
-    "openai",
-    "pandas",
-    "tiktoken",
-    "unstructured",
-    "tabulate"
-)
-
-stub = Stub(
-    name="example-langchain-lancedb",
-    image=lancedb_image,
-    secrets=[Secret.from_name("my-openai-secret")],
-)
-```
-
-We're using Modal's Secrets injection to secure our OpenAI key. To set your own, you can access the Modal UI and enter your key.
-
-### Setting up caches for LanceDB and LangChain
-
-Next, we can setup some globals to cache our LanceDB database, as well as our LangChain docsource:
-
-```python
-docsearch = None
-docs_path = Path("docs.pkl")
-db_path = Path("lancedb")
-```
-
-### Downloading our dataset
-
-We're going use a pregenerated dataset, which stores HTML files of the Pandas 2.0 documentation. 
-You could switch this out for your own dataset.
-
-```python
-def download_docs():
-    pandas_docs = requests.get("https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip")
-    with open(Path("pandas.documentation.zip"), "wb") as f:
-        f.write(pandas_docs.content)
-
-    file = zipfile.ZipFile(Path("pandas.documentation.zip"))
-    file.extractall(path=Path("pandas_docs"))
-```
-
-### Pre-processing the dataset and generating metadata
-
-Once we've downloaded it, we want to parse and pre-process them using LangChain, and then vectorize them and store it in LanceDB.
-Let's first create a function that uses LangChains `UnstructuredHTMLLoader` to parse them.
-We can then add our own metadata to it and store it alongside the data, we'll later be able to use this for filtering metadata.
-
-```python
-def store_docs():
-    docs = []
-
-    if not docs_path.exists():
-        for p in Path("pandas_docs/pandas.documentation").rglob("*.html"):
-            if p.is_dir():
-                continue
-            loader = UnstructuredHTMLLoader(p)
-            raw_document = loader.load()
-
-            m = {}
-            m["title"] = get_document_title(raw_document[0])
-            m["version"] = "2.0rc0"
-            raw_document[0].metadata = raw_document[0].metadata | m
-            raw_document[0].metadata["source"] = str(raw_document[0].metadata["source"])
-            docs = docs + raw_document
-
-        with docs_path.open("wb") as fh:
-            pickle.dump(docs, fh)
-    else:
-        with docs_path.open("rb") as fh:
-            docs = pickle.load(fh)
-
-    return docs
-```
-
-### Simple LangChain chain for a QA bot
-
-Now we can create a simple LangChain chain for our QA bot. We'll use the `RecursiveCharacterTextSplitter` to split our documents into chunks, and then use the `OpenAIEmbeddings` to vectorize them.
-
-Lastly, we'll create a LanceDB table and store the vectorized documents in it, then create a `RetrievalQA` model from the chain and return it.
-
-```python
-def qanda_langchain(query):
-    download_docs()
-    docs = store_docs()
-
-    text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=1000,
-        chunk_overlap=200,
-    )
-    documents = text_splitter.split_documents(docs)
-    embeddings = OpenAIEmbeddings()
-
-    db = lancedb.connect(db_path) 
-    table = db.create_table("pandas_docs", data=[
-        {"vector": embeddings.embed_query("Hello World"), "text": "Hello World", "id": "1"}
-    ], mode="overwrite")
-    docsearch = LanceDB.from_documents(documents, embeddings, connection=table)
-    qa = RetrievalQA.from_chain_type(llm=OpenAI(), chain_type="stuff", retriever=docsearch.as_retriever())
-    return qa.run(query)
-```
-
-### Creating our Modal entry points
-
-Now we can create our Modal entry points for our CLI and web endpoint:
-
-```python
-@stub.function()
-@web_endpoint(method="GET")
-def web(query: str):
-    answer = qanda_langchain(query)
-    return {
-        "answer": answer,
-    }
-    
-@stub.function()
-def cli(query: str):
-    answer = qanda_langchain(query)
-    print(answer)
-```
-
-# Testing it out!
-
-Testing the CLI:
-
-```bash
-modal run modal_langchain.py --query "What are the major differences in pandas 2.0?"
-```
-
-Testing the web endpoint:
-
-```bash
-modal serve modal_langchain.py
-```
-
-In the CLI, Modal will provide you a web endpoint. Copy this endpoint URI for the next step.
-Once this is served, then we can hit it with `curl`. 
-
-Note, the first time this runs, it will take a few minutes to download the dataset and vectorize it.
-An actual production example would pre-cache/load the dataset and vectorized documents prior
-
-```bash
-curl --get --data-urlencode "query=What are the major differences in pandas 2.0?" https://your-modal-endpoint-app.modal.run
-
-{"answer":" The major differences in pandas 2.0 include the ability to use any numpy numeric dtype in a Index, installing optional dependencies with pip extras, and enhancements, bug fixes, and performance improvements."}
-```
-

docs/src/examples/modal_langchain.py

@@ -1,107 +0,0 @@
-import sys
-from modal import Secret, Stub, Image, web_endpoint
-import lancedb
-import re
-import pickle
-import requests
-import zipfile
-from pathlib import Path
-
-from langchain.document_loaders import UnstructuredHTMLLoader
-from langchain.embeddings import OpenAIEmbeddings
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.vectorstores import LanceDB
-from langchain.llms import OpenAI
-from langchain.chains import RetrievalQA
-
-lancedb_image = Image.debian_slim().pip_install(
-    "lancedb",
-    "langchain",
-    "openai",
-    "pandas",
-    "tiktoken",
-    "unstructured",
-    "tabulate"
-)
-
-stub = Stub(
-    name="example-langchain-lancedb",
-    image=lancedb_image,
-    secrets=[Secret.from_name("my-openai-secret")],
-)
-
-docsearch = None
-docs_path = Path("docs.pkl")
-db_path = Path("lancedb")
-
-def get_document_title(document):
-    m = str(document.metadata["source"])
-    title = re.findall("pandas.documentation(.*).html", m)
-    if title[0] is not None:
-        return(title[0])
-    return ''
-
-def download_docs():
-    pandas_docs = requests.get("https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip")
-    with open(Path("pandas.documentation.zip"), "wb") as f:
-        f.write(pandas_docs.content)
-
-    file = zipfile.ZipFile(Path("pandas.documentation.zip"))
-    file.extractall(path=Path("pandas_docs"))
-
-def store_docs():
-    docs = []
-
-    if not docs_path.exists():
-        for p in Path("pandas_docs/pandas.documentation").rglob("*.html"):
-            if p.is_dir():
-                continue
-            loader = UnstructuredHTMLLoader(p)
-            raw_document = loader.load()
-
-            m = {}
-            m["title"] = get_document_title(raw_document[0])
-            m["version"] = "2.0rc0"
-            raw_document[0].metadata = raw_document[0].metadata | m
-            raw_document[0].metadata["source"] = str(raw_document[0].metadata["source"])
-            docs = docs + raw_document
-
-        with docs_path.open("wb") as fh:
-            pickle.dump(docs, fh)
-    else:
-        with docs_path.open("rb") as fh:
-            docs = pickle.load(fh)
-
-    return docs
-
-def qanda_langchain(query):
-    download_docs()
-    docs = store_docs()
-
-    text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=1000,
-        chunk_overlap=200,
-    )
-    documents = text_splitter.split_documents(docs)
-    embeddings = OpenAIEmbeddings()
-
-    db = lancedb.connect(db_path) 
-    table = db.create_table("pandas_docs", data=[
-        {"vector": embeddings.embed_query("Hello World"), "text": "Hello World", "id": "1"}
-    ], mode="overwrite")
-    docsearch = LanceDB.from_documents(documents, embeddings, connection=table)
-    qa = RetrievalQA.from_chain_type(llm=OpenAI(), chain_type="stuff", retriever=docsearch.as_retriever())
-    return qa.run(query)
-
-@stub.function()
-@web_endpoint(method="GET")
-def web(query: str):
-    answer = qanda_langchain(query)
-    return {
-        "answer": answer,
-    }
-
-@stub.function()
-def cli(query: str):
-    answer = qanda_langchain(query)
-    print(answer)

docs/src/examples/multimodal_search.md

@@ -1,7 +0,0 @@
-# Image multimodal search
-
-## Search through an image dataset using natural language, full text and SQL
-
-<img id="splash" width="400" alt="multimodal search" src="https://github.com/lancedb/lancedb/assets/917119/993a7c9f-be01-449d-942e-1ce1d4ed63af">
-
-This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/multimodal_search.ipynb)

docs/src/examples/youtube_transcript_search.md

@@ -4,4 +4,4 @@
 
 <img id="splash" width="400" alt="youtube transcript search" src="https://user-images.githubusercontent.com/917119/236965568-def7394d-171c-45f2-939d-8edfeaadd88c.png">
 
-This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb)
+This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/notebooks/youtube_transcript_search.ipynb)

docs/src/fts.md

@@ -1,51 +0,0 @@
-# [EXPERIMENTAL] Full text search
-
-LanceDB now provides experimental support for full text search.
-This is currently Python only. We plan to push the integration down to Rust in the future
-to make this available for JS as well.
-
-## Installation
-
-To use full text search, you must install optional dependency tantivy-py:
-
-# tantivy 0.19.2
-pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-
-
-## Quickstart
-
-Assume:
-1. `table` is a LanceDB Table
-2. `text` is the name of the Table column that we want to index
-
-To create the index:
-
-```python
-table.create_fts_index("text")
-```
-
-To search:
-
-```python
-df = table.search("puppy").limit(10).select(["text"]).to_df()
-```
-
-LanceDB automatically looks for an FTS index if the input is str.
-
-## Multiple text columns
-
-If you have multiple columns to index, pass them all as a list to `create_fts_index`:
-
-```python
-table.create_fts_index(["text1", "text2"])
-```
-
-Note that the search API call does not change - you can search over all indexed columns at once.
-
-## Current limitations
-
-1. Currently we do not yet support incremental writes.
-If you add data after fts index creation, it won't be reflected
-in search results until you do a full reindex.
-
-2. We currently only support local filesystem paths for the fts index.

docs/src/index.md

@@ -1,6 +1,6 @@
 # Welcome to LanceDB's Documentation
 
-LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrevial, filtering and management of embeddings.
+LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrivial, filtering and management of embeddings.
 
 The key features of LanceDB include:
 
@@ -8,57 +8,42 @@ The key features of LanceDB include:
 
 * Store, query and filter vectors, metadata and multi-modal data (text, images, videos, point clouds, and more).
 
-* Native Python and Javascript/Typescript support.
+* Native Python and Javascript/Typescript support (coming soon).
 
 * Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure.
 
 * Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lanecdb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
 
-LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
+LanceDB's core is written in Rust 🦀 and is built using Lance, an open-source columnar format designed for performant ML workloads.
 
-## Quick Start
 
-=== "Python"
-      ```shell
-      pip install lancedb
-      ```
+## Installation
 
-      ```python
-      import lancedb
+```shell
+pip install lancedb
+```
 
-      uri = "/tmp/lancedb"
-      db = lancedb.connect(uri)
-      table = db.create_table("my_table",
-                              data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                                    {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      result = table.search([100, 100]).limit(2).to_df()
-      ```
+## Quickstart
 
-=== "Javascript"
-      ```shell
-      npm install vectordb
-      ```
+```python
+import lancedb
 
-      ```javascript
-      const lancedb = require("vectordb");
+db = lancedb.connect(".")
+table = db.create_table("my_table",
+                         data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                               {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+result = table.search([100, 100]).limit(2).to_df()
+```
 
-      const uri = "/tmp/lancedb";
-      const db = await lancedb.connect(uri);
-      const table = await db.createTable("my_table", 
-            [{ id: 1, vector: [3.1, 4.1], item: "foo", price: 10.0 },
-            { id: 2, vector: [5.9, 26.5], item: "bar", price: 20.0 }])
-      const results = await table.search([100, 100]).limit(2).execute();
-      ```
+## Complete Demos
+
+We will be adding completed demo apps built using LanceDB.
+- [YouTube Transcript Search](../notebooks/youtube_transcript_search.ipynb)
 
-## Complete Demos (Python)
-- [YouTube Transcript Search](notebooks/youtube_transcript_search.ipynb)
-- [Documentation QA Bot using LangChain](notebooks/code_qa_bot.ipynb)
-- [Multimodal search using CLIP](notebooks/multimodal_search.ipynb)
 
 ## Documentation Quick Links
 * [`Basic Operations`](basic.md) - basic functionality of LanceDB.
 * [`Embedding Functions`](embedding.md) - functions for working with embeddings.
 * [`Indexing`](ann_indexes.md) - create vector indexes to speed up queries.
-* [`Full text search`](fts.md) - [EXPERIMENTAL] full-text search API
 * [`Ecosystem Integrations`](integrations.md) - integrating LanceDB with python data tooling ecosystem.
 * [`API Reference`](python.md) - detailed documentation for the LanceDB Python SDK.

docs/src/javascript/.nojekyll

@@ -1 +0,0 @@
-TypeDoc added this file to prevent GitHub Pages from using Jekyll. You can turn off this behavior by setting the `githubPages` option to false.

docs/src/javascript/README.md

@@ -1,51 +0,0 @@
-vectordb / [Exports](modules.md)
-
-# LanceDB
-
-A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb).
-
-## Installation
-
-```bash
-npm install vectordb
-```
-
-## Usage
-
-### Basic Example
-
-```javascript
-const lancedb = require('vectordb');
-const db = lancedb.connect('<PATH_TO_LANCEDB_DATASET>');
-const table = await db.openTable('my_table');
-const query = await table.search([0.1, 0.3]).setLimit(20).execute();
-console.log(results);
-```
-
-The [examples](./examples) folder contains complete examples.
-
-## Development
-
-The LanceDB javascript is built with npm:
-
-```bash
-npm run tsc
-```
-
-Run the tests with
-
-```bash
-npm test
-```
-
-To run the linter and have it automatically fix all errors
-
-```bash
-npm run lint -- --fix
-```
-
-To build documentation
-
-```bash
-npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
-```

docs/src/javascript/classes/Connection.md

@@ -1,211 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / Connection
-
-# Class: Connection
-
-A connection to a LanceDB database.
-
-## Table of contents
-
-### Constructors
-
-- [constructor](Connection.md#constructor)
-
-### Properties
-
-- [\_db](Connection.md#_db)
-- [\_uri](Connection.md#_uri)
-
-### Accessors
-
-- [uri](Connection.md#uri)
-
-### Methods
-
-- [createTable](Connection.md#createtable)
-- [createTableArrow](Connection.md#createtablearrow)
-- [openTable](Connection.md#opentable)
-- [tableNames](Connection.md#tablenames)
-
-## Constructors
-
-### constructor
-
-• **new Connection**(`db`, `uri`)
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `db` | `any` |
-| `uri` | `string` |
-
-#### Defined in
-
-[index.ts:46](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L46)
-
-## Properties
-
-### \_db
-
-• `Private` `Readonly` **\_db**: `any`
-
-#### Defined in
-
-[index.ts:44](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L44)
-
-___
-
-### \_uri
-
-• `Private` `Readonly` **\_uri**: `string`
-
-#### Defined in
-
-[index.ts:43](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L43)
-
-## Accessors
-
-### uri
-
-• `get` **uri**(): `string`
-
-#### Returns
-
-`string`
-
-#### Defined in
-
-[index.ts:51](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L51)
-
-## Methods
-
-### createTable
-
-▸ **createTable**(`name`, `data`): `Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-Creates a new Table and initialize it with new data.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-#### Defined in
-
-[index.ts:91](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L91)
-
-▸ **createTable**<`T`\>(`name`, `data`, `embeddings`): `Promise`<[`Table`](Table.md)<`T`\>\>
-
-Creates a new Table and initialize it with new data.
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`T`\>\>
-
-#### Defined in
-
-[index.ts:99](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L99)
-
-___
-
-### createTableArrow
-
-▸ **createTableArrow**(`name`, `table`): `Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` |
-| `table` | `Table`<`any`\> |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-#### Defined in
-
-[index.ts:109](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L109)
-
-___
-
-### openTable
-
-▸ **openTable**(`name`): `Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-Open a table in the database.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-#### Defined in
-
-[index.ts:67](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L67)
-
-▸ **openTable**<`T`\>(`name`, `embeddings`): `Promise`<[`Table`](Table.md)<`T`\>\>
-
-Open a table in the database.
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`T`\>\>
-
-#### Defined in
-
-[index.ts:74](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L74)
-
-___
-
-### tableNames
-
-▸ **tableNames**(): `Promise`<`string`[]\>
-
-Get the names of all tables in the database.
-
-#### Returns
-
-`Promise`<`string`[]\>
-
-#### Defined in
-
-[index.ts:58](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L58)

docs/src/javascript/classes/OpenAIEmbeddingFunction.md

@@ -1,105 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / OpenAIEmbeddingFunction
-
-# Class: OpenAIEmbeddingFunction
-
-An embedding function that automatically creates vector representation for a given column.
-
-## Implements
-
-- [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`string`\>
-
-## Table of contents
-
-### Constructors
-
-- [constructor](OpenAIEmbeddingFunction.md#constructor)
-
-### Properties
-
-- [\_modelName](OpenAIEmbeddingFunction.md#_modelname)
-- [\_openai](OpenAIEmbeddingFunction.md#_openai)
-- [sourceColumn](OpenAIEmbeddingFunction.md#sourcecolumn)
-
-### Methods
-
-- [embed](OpenAIEmbeddingFunction.md#embed)
-
-## Constructors
-
-### constructor
-
-• **new OpenAIEmbeddingFunction**(`sourceColumn`, `openAIKey`, `modelName?`)
-
-#### Parameters
-
-| Name | Type | Default value |
-| :------ | :------ | :------ |
-| `sourceColumn` | `string` | `undefined` |
-| `openAIKey` | `string` | `undefined` |
-| `modelName` | `string` | `'text-embedding-ada-002'` |
-
-#### Defined in
-
-[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/openai.ts#L21)
-
-## Properties
-
-### \_modelName
-
-• `Private` `Readonly` **\_modelName**: `string`
-
-#### Defined in
-
-[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/openai.ts#L19)
-
-___
-
-### \_openai
-
-• `Private` `Readonly` **\_openai**: `any`
-
-#### Defined in
-
-[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/openai.ts#L18)
-
-___
-
-### sourceColumn
-
-• **sourceColumn**: `string`
-
-The name of the column that will be used as input for the Embedding Function.
-
-#### Implementation of
-
-[EmbeddingFunction](../interfaces/EmbeddingFunction.md).[sourceColumn](../interfaces/EmbeddingFunction.md#sourcecolumn)
-
-#### Defined in
-
-[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/openai.ts#L50)
-
-## Methods
-
-### embed
-
-▸ **embed**(`data`): `Promise`<`number`[][]\>
-
-Creates a vector representation for the given values.
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `data` | `string`[] |
-
-#### Returns
-
-`Promise`<`number`[][]\>
-
-#### Implementation of
-
-[EmbeddingFunction](../interfaces/EmbeddingFunction.md).[embed](../interfaces/EmbeddingFunction.md#embed)
-
-#### Defined in
-
-[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/openai.ts#L38)

docs/src/javascript/classes/Query.md

@@ -1,299 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / Query
-
-# Class: Query<T\>
-
-A builder for nearest neighbor queries for LanceDB.
-
-## Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-## Table of contents
-
-### Constructors
-
-- [constructor](Query.md#constructor)
-
-### Properties
-
-- [\_columns](Query.md#_columns)
-- [\_embeddings](Query.md#_embeddings)
-- [\_filter](Query.md#_filter)
-- [\_limit](Query.md#_limit)
-- [\_metricType](Query.md#_metrictype)
-- [\_nprobes](Query.md#_nprobes)
-- [\_query](Query.md#_query)
-- [\_queryVector](Query.md#_queryvector)
-- [\_refineFactor](Query.md#_refinefactor)
-- [\_tbl](Query.md#_tbl)
-
-### Methods
-
-- [execute](Query.md#execute)
-- [filter](Query.md#filter)
-- [limit](Query.md#limit)
-- [metricType](Query.md#metrictype)
-- [nprobes](Query.md#nprobes)
-- [refineFactor](Query.md#refinefactor)
-
-## Constructors
-
-### constructor
-
-• **new Query**<`T`\>(`tbl`, `query`, `embeddings?`)
-
-#### Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `tbl` | `any` |
-| `query` | `T` |
-| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
-
-#### Defined in
-
-[index.ts:241](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L241)
-
-## Properties
-
-### \_columns
-
-• `Private` `Optional` `Readonly` **\_columns**: `string`[]
-
-#### Defined in
-
-[index.ts:236](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L236)
-
-___
-
-### \_embeddings
-
-• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
-
-#### Defined in
-
-[index.ts:239](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L239)
-
-___
-
-### \_filter
-
-• `Private` `Optional` **\_filter**: `string`
-
-#### Defined in
-
-[index.ts:237](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L237)
-
-___
-
-### \_limit
-
-• `Private` **\_limit**: `number`
-
-#### Defined in
-
-[index.ts:233](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L233)
-
-___
-
-### \_metricType
-
-• `Private` `Optional` **\_metricType**: [`MetricType`](../enums/MetricType.md)
-
-#### Defined in
-
-[index.ts:238](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L238)
-
-___
-
-### \_nprobes
-
-• `Private` **\_nprobes**: `number`
-
-#### Defined in
-
-[index.ts:235](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L235)
-
-___
-
-### \_query
-
-• `Private` `Readonly` **\_query**: `T`
-
-#### Defined in
-
-[index.ts:231](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L231)
-
-___
-
-### \_queryVector
-
-• `Private` `Optional` **\_queryVector**: `number`[]
-
-#### Defined in
-
-[index.ts:232](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L232)
-
-___
-
-### \_refineFactor
-
-• `Private` `Optional` **\_refineFactor**: `number`
-
-#### Defined in
-
-[index.ts:234](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L234)
-
-___
-
-### \_tbl
-
-• `Private` `Readonly` **\_tbl**: `any`
-
-#### Defined in
-
-[index.ts:230](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L230)
-
-## Methods
-
-### execute
-
-▸ **execute**<`T`\>(): `Promise`<`T`[]\>
-
-Execute the query and return the results as an Array of Objects
-
-#### Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `Record`<`string`, `unknown`\> |
-
-#### Returns
-
-`Promise`<`T`[]\>
-
-#### Defined in
-
-[index.ts:301](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L301)
-
-___
-
-### filter
-
-▸ **filter**(`value`): [`Query`](Query.md)<`T`\>
-
-A filter statement to be applied to this query.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | `string` | A filter in the same format used by a sql WHERE clause. |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:284](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L284)
-
-___
-
-### limit
-
-▸ **limit**(`value`): [`Query`](Query.md)<`T`\>
-
-Sets the number of results that will be returned
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | `number` | number of results |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:257](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L257)
-
-___
-
-### metricType
-
-▸ **metricType**(`value`): [`Query`](Query.md)<`T`\>
-
-The MetricType used for this Query.
-
-**`See`**
-
-MetricType for the different options
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | [`MetricType`](../enums/MetricType.md) | The metric to the. |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:293](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L293)
-
-___
-
-### nprobes
-
-▸ **nprobes**(`value`): [`Query`](Query.md)<`T`\>
-
-The number of probes used. A higher number makes search more accurate but also slower.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | `number` | The number of probes used. |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:275](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L275)
-
-___
-
-### refineFactor
-
-▸ **refineFactor**(`value`): [`Query`](Query.md)<`T`\>
-
-Refine the results by reading extra elements and re-ranking them in memory.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | `number` | refine factor to use in this query. |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:266](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L266)

docs/src/javascript/classes/Table.md

@@ -1,215 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / Table
-
-# Class: Table<T\>
-
-## Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-## Table of contents
-
-### Constructors
-
-- [constructor](Table.md#constructor)
-
-### Properties
-
-- [\_embeddings](Table.md#_embeddings)
-- [\_name](Table.md#_name)
-- [\_tbl](Table.md#_tbl)
-
-### Accessors
-
-- [name](Table.md#name)
-
-### Methods
-
-- [add](Table.md#add)
-- [create\_index](Table.md#create_index)
-- [overwrite](Table.md#overwrite)
-- [search](Table.md#search)
-
-## Constructors
-
-### constructor
-
-• **new Table**<`T`\>(`tbl`, `name`)
-
-#### Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `tbl` | `any` |
-| `name` | `string` |
-
-#### Defined in
-
-[index.ts:121](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L121)
-
-• **new Table**<`T`\>(`tbl`, `name`, `embeddings`)
-
-#### Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `tbl` | `any` |  |
-| `name` | `string` |  |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use when interacting with this table |
-
-#### Defined in
-
-[index.ts:127](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L127)
-
-## Properties
-
-### \_embeddings
-
-• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
-
-#### Defined in
-
-[index.ts:119](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L119)
-
-___
-
-### \_name
-
-• `Private` `Readonly` **\_name**: `string`
-
-#### Defined in
-
-[index.ts:118](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L118)
-
-___
-
-### \_tbl
-
-• `Private` `Readonly` **\_tbl**: `any`
-
-#### Defined in
-
-[index.ts:117](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L117)
-
-## Accessors
-
-### name
-
-• `get` **name**(): `string`
-
-#### Returns
-
-`string`
-
-#### Defined in
-
-[index.ts:134](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L134)
-
-## Methods
-
-### add
-
-▸ **add**(`data`): `Promise`<`number`\>
-
-Insert records into this Table.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
-
-#### Returns
-
-`Promise`<`number`\>
-
-The number of rows added to the table
-
-#### Defined in
-
-[index.ts:152](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L152)
-
-___
-
-### create\_index
-
-▸ **create_index**(`indexParams`): `Promise`<`any`\>
-
-Create an ANN index on this Table vector index.
-
-**`See`**
-
-VectorIndexParams.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `indexParams` | `IvfPQIndexConfig` | The parameters of this Index, |
-
-#### Returns
-
-`Promise`<`any`\>
-
-#### Defined in
-
-[index.ts:171](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L171)
-
-___
-
-### overwrite
-
-▸ **overwrite**(`data`): `Promise`<`number`\>
-
-Insert records into this Table, replacing its contents.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
-
-#### Returns
-
-`Promise`<`number`\>
-
-The number of rows added to the table
-
-#### Defined in
-
-[index.ts:162](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L162)
-
-___
-
-### search
-
-▸ **search**(`query`): [`Query`](Query.md)<`T`\>
-
-Creates a search query to find the nearest neighbors of the given search term
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `query` | `T` | The query search term |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:142](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L142)

docs/src/javascript/enums/MetricType.md

@@ -1,36 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / MetricType
-
-# Enumeration: MetricType
-
-Distance metrics type.
-
-## Table of contents
-
-### Enumeration Members
-
-- [Cosine](MetricType.md#cosine)
-- [L2](MetricType.md#l2)
-
-## Enumeration Members
-
-### Cosine
-
-• **Cosine** = ``"cosine"``
-
-Cosine distance
-
-#### Defined in
-
-[index.ts:341](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L341)
-
-___
-
-### L2
-
-• **L2** = ``"l2"``
-
-Euclidean distance
-
-#### Defined in
-
-[index.ts:336](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L336)

docs/src/javascript/enums/WriteMode.md

@@ -1,30 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / WriteMode
-
-# Enumeration: WriteMode
-
-## Table of contents
-
-### Enumeration Members
-
-- [Append](WriteMode.md#append)
-- [Overwrite](WriteMode.md#overwrite)
-
-## Enumeration Members
-
-### Append
-
-• **Append** = ``"append"``
-
-#### Defined in
-
-[index.ts:326](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L326)
-
-___
-
-### Overwrite
-
-• **Overwrite** = ``"overwrite"``
-
-#### Defined in
-
-[index.ts:325](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L325)

docs/src/javascript/interfaces/EmbeddingFunction.md

@@ -1,60 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / EmbeddingFunction
-
-# Interface: EmbeddingFunction<T\>
-
-An embedding function that automatically creates vector representation for a given column.
-
-## Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-## Implemented by
-
-- [`OpenAIEmbeddingFunction`](../classes/OpenAIEmbeddingFunction.md)
-
-## Table of contents
-
-### Properties
-
-- [embed](EmbeddingFunction.md#embed)
-- [sourceColumn](EmbeddingFunction.md#sourcecolumn)
-
-## Properties
-
-### embed
-
-• **embed**: (`data`: `T`[]) => `Promise`<`number`[][]\>
-
-#### Type declaration
-
-▸ (`data`): `Promise`<`number`[][]\>
-
-Creates a vector representation for the given values.
-
-##### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `data` | `T`[] |
-
-##### Returns
-
-`Promise`<`number`[][]\>
-
-#### Defined in
-
-[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/embedding_function.ts#L27)
-
-___
-
-### sourceColumn
-
-• **sourceColumn**: `string`
-
-The name of the column that will be used as input for the Embedding Function.
-
-#### Defined in
-
-[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/embedding_function.ts#L22)

docs/src/javascript/modules.md

@@ -1,61 +0,0 @@
-[vectordb](README.md) / Exports
-
-# vectordb
-
-## Table of contents
-
-### Enumerations
-
-- [MetricType](enums/MetricType.md)
-- [WriteMode](enums/WriteMode.md)
-
-### Classes
-
-- [Connection](classes/Connection.md)
-- [OpenAIEmbeddingFunction](classes/OpenAIEmbeddingFunction.md)
-- [Query](classes/Query.md)
-- [Table](classes/Table.md)
-
-### Interfaces
-
-- [EmbeddingFunction](interfaces/EmbeddingFunction.md)
-
-### Type Aliases
-
-- [VectorIndexParams](modules.md#vectorindexparams)
-
-### Functions
-
-- [connect](modules.md#connect)
-
-## Type Aliases
-
-### VectorIndexParams
-
-Ƭ **VectorIndexParams**: `IvfPQIndexConfig`
-
-#### Defined in
-
-[index.ts:224](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L224)
-
-## Functions
-
-### connect
-
-▸ **connect**(`uri`): `Promise`<[`Connection`](classes/Connection.md)\>
-
-Connect to a LanceDB instance at the given URI
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `uri` | `string` | The uri of the database. |
-
-#### Returns
-
-`Promise`<[`Connection`](classes/Connection.md)\>
-
-#### Defined in
-
-[index.ts:34](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L34)

docs/src/notebooks/diffusiondb/datagen.py

@@ -1,108 +0,0 @@
-#!/usr/bin/env python
-#
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-"""Dataset hf://poloclub/diffusiondb
-"""
-
-import io
-from argparse import ArgumentParser
-from multiprocessing import Pool
-
-import lance
-import lancedb
-import pyarrow as pa
-from datasets import load_dataset
-from PIL import Image
-from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast
-
-MODEL_ID = "openai/clip-vit-base-patch32"
-
-device = "cuda"
-
-tokenizer = CLIPTokenizerFast.from_pretrained(MODEL_ID)
-model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32").to(device)
-processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
-
-schema = pa.schema(
-    [
-        pa.field("prompt", pa.string()),
-        pa.field("seed", pa.uint32()),
-        pa.field("step", pa.uint16()),
-        pa.field("cfg", pa.float32()),
-        pa.field("sampler", pa.string()),
-        pa.field("width", pa.uint16()),
-        pa.field("height", pa.uint16()),
-        pa.field("timestamp", pa.timestamp("s")),
-        pa.field("image_nsfw", pa.float32()),
-        pa.field("prompt_nsfw", pa.float32()),
-        pa.field("vector", pa.list_(pa.float32(), 512)),
-        pa.field("image", pa.binary()),
-    ]
-)
-
-
-def pil_to_bytes(img) -> list[bytes]:
-    buf = io.BytesIO()
-    img.save(buf, format="PNG")
-    return buf.getvalue()
-
-
-def generate_clip_embeddings(batch) -> pa.RecordBatch:
-    image = processor(text=None, images=batch["image"], return_tensors="pt")[
-        "pixel_values"
-    ].to(device)
-    img_emb = model.get_image_features(image)
-    batch["vector"] = img_emb.cpu().tolist()
-
-    with Pool() as p:
-        batch["image_bytes"] = p.map(pil_to_bytes, batch["image"])
-    return batch
-
-
-def datagen(args):
-    """Generate DiffusionDB dataset, and use CLIP model to generate image embeddings."""
-    dataset = load_dataset("poloclub/diffusiondb", args.subset)
-    data = []
-    for b in dataset.map(
-        generate_clip_embeddings, batched=True, batch_size=256, remove_columns=["image"]
-    )["train"]:
-        b["image"] = b["image_bytes"]
-        del b["image_bytes"]
-        data.append(b)
-    tbl = pa.Table.from_pylist(data, schema=schema)
-    return tbl
-
-
-def main():
-    parser = ArgumentParser()
-    parser.add_argument(
-        "-o", "--output", metavar="DIR", help="Output lance directory", required=True
-    )
-    parser.add_argument(
-        "-s",
-        "--subset",
-        choices=["2m_all", "2m_first_10k", "2m_first_100k"],
-        default="2m_first_10k",
-        help="subset of the hg dataset",
-    )
-
-    args = parser.parse_args()
-
-    batches = datagen(args)
-    lance.write_dataset(batches, args.output)
-
-
-if __name__ == "__main__":
-    main()

docs/src/notebooks/diffusiondb/requirements.txt

@@ -1,9 +0,0 @@
-datasets
-Pillow
-lancedb
-isort
-black
-transformers
---index-url https://download.pytorch.org/whl/cu118
-torch
-torchvision

docs/src/notebooks/multimodal_search.ipynb

@@ -1,269 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
-      "\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
-     ]
-    }
-   ],
-   "source": [
-    "!pip install --quiet -U lancedb\n",
-    "!pip install --quiet gradio transformers torch torchvision"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import io\n",
-    "import PIL\n",
-    "import duckdb\n",
-    "import lancedb"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## First run setup: Download data and pre-process"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 30,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "<lance.dataset.LanceDataset at 0x3045db590>"
-      ]
-     },
-     "execution_count": 30,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "# remove null prompts\n",
-    "import lance\n",
-    "import pyarrow.compute as pc\n",
-    "\n",
-    "# download s3://eto-public/datasets/diffusiondb/small_10k.lance to this uri\n",
-    "data = lance.dataset(\"~/datasets/rawdata.lance\").to_table()\n",
-    "\n",
-    "# First data processing and full-text-search index\n",
-    "db = lancedb.connect(\"~/datasets/demo\")\n",
-    "tbl = db.create_table(\"diffusiondb\", data.filter(~pc.field(\"prompt\").is_null()))\n",
-    "tbl = tbl.create_fts_index([\"prompt\"])"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Create / Open LanceDB Table"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "db = lancedb.connect(\"~/datasets/demo\")\n",
-    "tbl = db.open_table(\"diffusiondb\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Create CLIP embedding function for the text"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast\n",
-    "\n",
-    "MODEL_ID = \"openai/clip-vit-base-patch32\"\n",
-    "\n",
-    "tokenizer = CLIPTokenizerFast.from_pretrained(MODEL_ID)\n",
-    "model = CLIPModel.from_pretrained(MODEL_ID)\n",
-    "processor = CLIPProcessor.from_pretrained(MODEL_ID)\n",
-    "\n",
-    "def embed_func(query):\n",
-    "    inputs = tokenizer([query], padding=True, return_tensors=\"pt\")\n",
-    "    text_features = model.get_text_features(**inputs)\n",
-    "    return text_features.detach().numpy()[0]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Search functions for Gradio"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def find_image_vectors(query):\n",
-    "    emb = embed_func(query)\n",
-    "    code = (\n",
-    "        \"import lancedb\\n\"\n",
-    "        \"db = lancedb.connect('~/datasets/demo')\\n\"\n",
-    "        \"tbl = db.open_table('diffusiondb')\\n\\n\"\n",
-    "        f\"embedding = embed_func('{query}')\\n\"\n",
-    "        \"tbl.search(embedding).limit(9).to_df()\"\n",
-    "    )\n",
-    "    return (_extract(tbl.search(emb).limit(9).to_df()), code)\n",
-    "\n",
-    "def find_image_keywords(query):\n",
-    "    code = (\n",
-    "        \"import lancedb\\n\"\n",
-    "        \"db = lancedb.connect('~/datasets/demo')\\n\"\n",
-    "        \"tbl = db.open_table('diffusiondb')\\n\\n\"\n",
-    "        f\"tbl.search('{query}').limit(9).to_df()\"\n",
-    "    )\n",
-    "    return (_extract(tbl.search(query).limit(9).to_df()), code)\n",
-    "\n",
-    "def find_image_sql(query):\n",
-    "    code = (\n",
-    "        \"import lancedb\\n\"\n",
-    "        \"import duckdb\\n\"\n",
-    "        \"db = lancedb.connect('~/datasets/demo')\\n\"\n",
-    "        \"tbl = db.open_table('diffusiondb')\\n\\n\"\n",
-    "        \"diffusiondb = tbl.to_lance()\\n\"\n",
-    "        f\"duckdb.sql('{query}').to_df()\"\n",
-    "    )    \n",
-    "    diffusiondb = tbl.to_lance()\n",
-    "    return (_extract(duckdb.sql(query).to_df()), code)\n",
-    "\n",
-    "def _extract(df):\n",
-    "    image_col = \"image\"\n",
-    "    return [(PIL.Image.open(io.BytesIO(row[image_col])), row[\"prompt\"]) for _, row in df.iterrows()]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Setup Gradio interface"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 28,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Running on local URL:  http://127.0.0.1:7881\n",
-      "\n",
-      "To create a public link, set `share=True` in `launch()`.\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<div><iframe src=\"http://127.0.0.1:7881/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
-      ],
-      "text/plain": [
-       "<IPython.core.display.HTML object>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/plain": []
-     },
-     "execution_count": 28,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import gradio as gr\n",
-    "\n",
-    "\n",
-    "with gr.Blocks() as demo:\n",
-    "    with gr.Row():\n",
-    "        with gr.Tab(\"Embeddings\"):\n",
-    "            vector_query = gr.Textbox(value=\"portraits of a person\", show_label=False)\n",
-    "            b1 = gr.Button(\"Submit\")\n",
-    "        with gr.Tab(\"Keywords\"):\n",
-    "            keyword_query = gr.Textbox(value=\"ninja turtle\", show_label=False)\n",
-    "            b2 = gr.Button(\"Submit\")\n",
-    "        with gr.Tab(\"SQL\"):\n",
-    "            sql_query = gr.Textbox(value=\"SELECT * from diffusiondb WHERE image_nsfw >= 2 LIMIT 9\", show_label=False)\n",
-    "            b3 = gr.Button(\"Submit\")\n",
-    "    with gr.Row():\n",
-    "        code = gr.Code(label=\"Code\", language=\"python\")\n",
-    "    with gr.Row():\n",
-    "        gallery = gr.Gallery(\n",
-    "                label=\"Found images\", show_label=False, elem_id=\"gallery\"\n",
-    "            ).style(columns=[3], rows=[3], object_fit=\"contain\", height=\"auto\")   \n",
-    "        \n",
-    "    b1.click(find_image_vectors, inputs=vector_query, outputs=[gallery, code])\n",
-    "    b2.click(find_image_keywords, inputs=keyword_query, outputs=[gallery, code])\n",
-    "    b3.click(find_image_sql, inputs=sql_query, outputs=[gallery, code])\n",
-    "    \n",
-    "demo.launch()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.3"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 1
-}

docs/src/search.md

@@ -1,85 +0,0 @@
-# Vector Search
-
-`Vector Search` finds the nearest vectors from the database.
-In a recommendation system or search engine, you can find similar products from
-the one you searched.
-In LLM and other AI applications,
-each data point can be [presented by the embeddings generated from some models](embedding.md),
-it returns the most relevant features.
-
-A search in high-dimensional vector space, is to find `K-Nearest-Neighbors (KNN)` of the query vector.
-
-## Metric
-
-In LanceDB, a `Metric` is the way to describe the distance between a pair of vectors.
-Currently, we support the following metrics:
-
-| Metric      | Description                          |
-| ----------- | ------------------------------------ |
-| `L2`        | [Euclidean / L2 distance](https://en.wikipedia.org/wiki/Euclidean_distance) |
-| `Cosine`    | [Cosine Similarity](https://en.wikipedia.org/wiki/Cosine_similarity)|
-
-
-## Search
-
-### Flat Search
-
-
-If there is no [vector index is created](ann_indexes.md), LanceDB will just brute-force scan
-the vector column and compute the distance.
-
-=== "Python"
-
-    ```python
-    import lancedb
-    db = lancedb.connect("data/sample-lancedb")
-
-    tbl = db.open_table("my_vectors")
-
-    df = tbl.search(np.random.random((768)))
-        .limit(10)
-        .to_df()
-    ```
-
-=== "JavaScript"
-
-    ```javascript
-    const vectordb = require('vectordb')
-    const db = await vectordb.connect('data/sample-lancedb')
-
-    tbl = db.open_table("my_vectors")
-
-    const results = await tbl.search(Array(768))
-        .limit(20)
-        .execute()
-    ```
-
-By default, `l2` will be used as `Metric` type. You can customize the metric type
-as well.
-
-=== "Python"
-
-    ```python
-    df = tbl.search(np.random.random((768)))
-        .metric("cosine")
-        .limit(10)
-        .to_df()
-    ```
-
-=== "JavaScript"
-
-    ```javascript
-    const vectordb = require('vectordb')
-    const db = await vectordb.connect('data/sample-lancedb')
-
-    tbl = db.open_table("my_vectors")
-
-    const results = await tbl.search(Array(768))
-        .metric("cosine")
-        .limit(20)
-        .execute()
-    ```
-
-### Search with Vector Index.
-
-See [ANN Index](ann_indexes.md) for more details.

docs/src/styles/global.css

@@ -1,6 +0,0 @@
-:root {
-    --md-primary-fg-color:        #625eff;
-    --md-primary-fg-color--dark:  #4338ca;
-    --md-text-font: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
-    --md-code-font: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
-}

node/.npmignore

@@ -0,0 +1,2 @@
+gen_test_data.py
+index.node

node/CHANGELOG.md

@@ -1,58 +0,0 @@
-# Changelog
-
-All notable changes to this project will be documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-## [0.1.4] - 2023-06-03
-
-### Added
-
-- Select / Project query API
-
-### Changed
-
--  Deprecated created_index in favor of createIndex
-
-## [0.1.3] - 2023-06-01
-
-### Added
-
-- Support S3 and Google Cloud Storage
-- Embedding functions support
-- OpenAI embedding function
-
-## [0.1.2] - 2023-05-27
-
-### Added
-
-- Append records API
-- Extra query params to to nodejs client
-- Create_index API
- 
-### Fixed
-
-- bugfix: string columns should be converted to Utf8Array (#94)
-
-## [0.1.1] - 2023-05-16
-
-### Added
-
-- create_table API
-- limit parameter for queries
-- Typescript / JavaScript examples
-- Linux support
-
-## [0.1.0] - 2023-05-16
-
-### Added
-
-- Initial  JavaScript / Node.js library for LanceDB
-- Read-only api to query LanceDB datasets
-- Supports macOS arm only
-
-## [pre-0.1.0]
-
-- Various prototypes / test builds
-

node/README.md

@@ -8,6 +8,9 @@ A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb)
 npm install vectordb
 ```
 
+This will download the appropriate native library for your platform. We currently
+support x86_64 Linux, Intel MacOS, and ARM (M1/M2) MacOS.
+
 ## Usage
 
 ### Basic Example
@@ -24,6 +27,19 @@ The [examples](./examples) folder contains complete examples.
 
 ## Development
 
+Build and install the rust library with:
+
+```bash
+npm run build
+npm run pack-build
+npm install --no-save ./dist/vectordb-*.tgz
+```
+
+`npm run build` builds the Rust library, `npm run pack-build` packages the Rust
+binary into an npm module called `@vectordb/<platform>` (for example, 
+`@vectordb/darwin-arm64.node`), and then `npm run install ...` installs that
+module.
+
 The LanceDB javascript is built with npm:
 
 ```bash
@@ -41,9 +57,3 @@ To run the linter and have it automatically fix all errors
 ```bash
 npm run lint -- --fix
 ```
-
-To build documentation
-
-```bash
-npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
-```

node/examples/js-openai/index.js

@@ -1,41 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-'use strict'
-
-async function example () {
-  const lancedb = require('vectordb')
-  // You need to provide an OpenAI API key, here we read it from the OPENAI_API_KEY environment variable
-  const apiKey = process.env.OPENAI_API_KEY
-  // The embedding function will create embeddings for the 'text' column(text in this case)
-  const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
-
-  const db = await lancedb.connect('data/sample-lancedb')
-
-  const data = [
-    { id: 1, text: 'Black T-Shirt', price: 10 },
-    { id: 2, text: 'Leather Jacket', price: 50 }
-  ]
-
-  const table = await db.createTable('vectors', data, embedding)
-  console.log(await db.tableNames())
-
-  const results = await table
-    .search('keeps me warm')
-    .limit(1)
-    .execute()
-  console.log(results[0].text)
-}
-
-example().then(_ => { console.log('All done!') })

node/examples/js-openai/package.json

@@ -1,15 +0,0 @@
-{
-  "name": "vectordb-example-js-openai",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "author": "Lance Devs",
-  "license": "Apache-2.0",
-  "dependencies": {
-    "vectordb": "file:../..",
-    "openai": "^3.2.1"
-  }
-}

node/examples/js/package.json

@@ -9,6 +9,6 @@
   "author": "Lance Devs",
   "license": "Apache-2.0",
   "dependencies": {
-    "vectordb": "file:../.."
+    "vectordb": "^0.1.0"
   }
 }

node/examples/ts/package.json

@@ -17,6 +17,6 @@
     "typescript": "*"
   },
   "dependencies": {
-    "vectordb": "file:../.."
+    "vectordb": "^0.1.0"
   }
 }

node/gen_test_data.py

@@ -0,0 +1,8 @@
+import lancedb
+
+uri = "sample-lancedb"
+db = lancedb.connect(uri)
+table = db.create_table("my_table",
+                         data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                               {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+

node/native.js

@@ -12,29 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+const { currentTarget } = require('@neon-rs/load');
+
 let nativeLib;
 
-function getPlatformLibrary() {
-    if (process.platform === "darwin" && process.arch == "arm64") {
-        return require('./aarch64-apple-darwin.node');
-    } else if (process.platform === "darwin" && process.arch == "x64") {
-        return require('./x86_64-apple-darwin.node');
-    } else if (process.platform === "linux" && process.arch == "x64") {
-        return require('./x86_64-unknown-linux-gnu.node');
-    } else {
-        throw new Error(`vectordb: unsupported platform ${process.platform}_${process.arch}. Please file a bug report at https://github.com/lancedb/lancedb/issues`)
-    }
-}
-
 try {
-    nativeLib = require('./index.node')
+    nativeLib = require(`@vectordb/${currentTarget()}`);
 } catch (e) {
-    if (e.code === "MODULE_NOT_FOUND") {
-        nativeLib = getPlatformLibrary();
-    } else {
-        throw new Error('vectordb: failed to load native library. Please file a bug report at https://github.com/lancedb/lancedb/issues');
-    }
+    throw new Error(`vectordb: failed to load native library.
+  You may need to run \`npm install @vectordb/${currentTarget()}\`.
+
+  If that does not work, please file a bug report at https://github.com/lancedb/lancedb/issues
+
+  Source error: ${e}`);
 }
 
-module.exports = nativeLib
-
+// Dynamic require for runtime.
+module.exports = nativeLib;

node/package.json

@@ -1,16 +1,17 @@
 {
   "name": "vectordb",
-  "version": "0.1.4",
+  "version": "0.1.2",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
     "tsc": "tsc -b",
-    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json-render-diagnostics",
+    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json",
     "build-release": "npm run build -- --release",
+    "cross-release": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cross build --message-format=json --release -p vectordb-node",
     "test": "mocha -recursive dist/test",
     "lint": "eslint src --ext .js,.ts",
-    "clean": "rm -rf node_modules *.node dist/"
+    "pack-build": "neon pack-build"
   },
   "repository": {
     "type": "git",
@@ -25,10 +26,10 @@
   "author": "Lance Devs",
   "license": "Apache-2.0",
   "devDependencies": {
+    "@neon-rs/cli": "^0.0.74",
     "@types/chai": "^4.3.4",
     "@types/mocha": "^10.0.1",
     "@types/node": "^18.16.2",
-    "@types/sinon": "^10.0.15",
     "@types/temp": "^0.9.1",
     "@typescript-eslint/eslint-plugin": "^5.59.1",
     "cargo-cp-artifact": "^0.1",
@@ -39,17 +40,40 @@
     "eslint-plugin-n": "^15.7.0",
     "eslint-plugin-promise": "^6.1.1",
     "mocha": "^10.2.0",
-    "openai": "^3.2.1",
-    "sinon": "^15.1.0",
     "temp": "^0.9.4",
     "ts-node": "^10.9.1",
     "ts-node-dev": "^2.0.0",
-    "typedoc": "^0.24.7",
-    "typedoc-plugin-markdown": "^3.15.3",
     "typescript": "*"
   },
   "dependencies": {
     "@apache-arrow/ts": "^12.0.0",
+    "@neon-rs/load": "^0.0.74",
     "apache-arrow": "^12.0.0"
+  },
+  "os": [
+    "darwin",
+    "linux"
+  ],
+  "cpu": [
+    "x64",
+    "arm64"
+  ],
+  "neon": {
+    "targets": {
+      "x86_64-apple-darwin": "@vectordb/darwin-x64",
+      "aarch64-apple-darwin": "@vectordb/darwin-arm64",
+      "x86_64-unknown-linux-gnu": "@vectordb/linux-x64-gnu",
+      "x86_64-unknown-linux-musl": "@vectordb/linux-x64-musl",
+      "aarch64-unknown-linux-gnu": "@vectordb/linux-arm64-gnu",
+      "aarch64-unknown-linux-musl": "@vectordb/linux-arm64-musl"
+    }
+  },
+  "optionalDependencies": {
+    "@vectordb/darwin-arm64": "0.1.2",
+    "@vectordb/darwin-x64": "0.1.2",
+    "@vectordb/linux-x64-gnu": "0.1.2",
+    "@vectordb/linux-x64-musl": "0.1.2",
+    "@vectordb/linux-arm64-gnu": "0.1.2",
+    "@vectordb/linux-arm64-musl": "0.1.2"
   }
 }

node/src/arrow.ts

@@ -15,16 +15,15 @@
 import {
   Field,
   Float32,
-  List, type ListBuilder,
+  List,
   makeBuilder,
   RecordBatchFileWriter,
-  Table, Utf8,
+  Table,
   type Vector,
   vectorFromArray
 } from 'apache-arrow'
-import { type EmbeddingFunction } from './index'
 
-export async function convertToTable<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Table> {
+export function convertToTable (data: Array<Record<string, unknown>>): Table {
   if (data.length === 0) {
     throw new Error('At least one record needs to be provided')
   }
@@ -34,7 +33,11 @@ export async function convertToTable<T> (data: Array<Record<string, unknown>>, e
 
   for (const columnsKey of columns) {
     if (columnsKey === 'vector') {
-      const listBuilder = newVectorListBuilder()
+      const children = new Field<Float32>('item', new Float32())
+      const list = new List(children)
+      const listBuilder = makeBuilder({
+        type: list
+      })
       const vectorSize = (data[0].vector as any[]).length
       for (const datum of data) {
         if ((datum[columnsKey] as any[]).length !== vectorSize) {
@@ -49,37 +52,15 @@ export async function convertToTable<T> (data: Array<Record<string, unknown>>, e
       for (const datum of data) {
         values.push(datum[columnsKey])
       }
-
-      if (columnsKey === embeddings?.sourceColumn) {
-        const vectors = await embeddings.embed(values as T[])
-        const listBuilder = newVectorListBuilder()
-        vectors.map(v => listBuilder.append(v))
-        records.vector = listBuilder.finish().toVector()
-      }
-
-      if (typeof values[0] === 'string') {
-        // `vectorFromArray` converts strings into dictionary vectors, forcing it back to a string column
-        records[columnsKey] = vectorFromArray(values, new Utf8())
-      } else {
-        records[columnsKey] = vectorFromArray(values)
-      }
+      records[columnsKey] = vectorFromArray(values)
     }
   }
 
   return new Table(records)
 }
 
-// Creates a new Arrow ListBuilder that stores a Vector column
-function newVectorListBuilder (): ListBuilder<Float32, any> {
-  const children = new Field<Float32>('item', new Float32())
-  const list = new List(children)
-  return makeBuilder({
-    type: list
-  })
-}
-
-export async function fromRecordsToBuffer<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Buffer> {
-  const table = await convertToTable(data, embeddings)
+export async function fromRecordsToBuffer (data: Array<Record<string, unknown>>): Promise<Buffer> {
+  const table = convertToTable(data)
   const writer = RecordBatchFileWriter.writeAll(table)
   return Buffer.from(await writer.toUint8Array())
 }

node/src/embedding/embedding_function.ts

@@ -1,28 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-/**
- * An embedding function that automatically creates vector representation for a given column.
- */
-export interface EmbeddingFunction<T> {
-  /**
-     * The name of the column that will be used as input for the Embedding Function.
-     */
-  sourceColumn: string
-
-  /**
-     * Creates a vector representation for the given values.
-     */
-  embed: (data: T[]) => Promise<number[][]>
-}

node/src/embedding/openai.ts

@@ -1,51 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { type EmbeddingFunction } from '../index'
-
-export class OpenAIEmbeddingFunction implements EmbeddingFunction<string> {
-  private readonly _openai: any
-  private readonly _modelName: string
-
-  constructor (sourceColumn: string, openAIKey: string, modelName: string = 'text-embedding-ada-002') {
-    let openai
-    try {
-      // eslint-disable-next-line @typescript-eslint/no-var-requires
-      openai = require('openai')
-    } catch {
-      throw new Error('please install openai using npm install openai')
-    }
-
-    this.sourceColumn = sourceColumn
-    const configuration = new openai.Configuration({
-      apiKey: openAIKey
-    })
-    this._openai = new openai.OpenAIApi(configuration)
-    this._modelName = modelName
-  }
-
-  async embed (data: string[]): Promise<number[][]> {
-    const response = await this._openai.createEmbedding({
-      model: this._modelName,
-      input: data
-    })
-    const embeddings: number[][] = []
-    for (let i = 0; i < response.data.data.length; i++) {
-      embeddings.push(response.data.data[i].embedding as number[])
-    }
-    return embeddings
-  }
-
-  sourceColumn: string
-}

node/src/index.ts

@@ -19,21 +19,16 @@ import {
   Vector
 } from 'apache-arrow'
 import { fromRecordsToBuffer } from './arrow'
-import type { EmbeddingFunction } from './embedding/embedding_function'
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const { databaseNew, databaseTableNames, databaseOpenTable, tableCreate, tableSearch, tableAdd, tableCreateVectorIndex } = require('../native.js')
-
-export type { EmbeddingFunction }
-export { OpenAIEmbeddingFunction } from './embedding/openai'
+const { databaseNew, databaseTableNames, databaseOpenTable, tableCreate, tableSearch, tableAdd } = require('../native.js')
 
 /**
  * Connect to a LanceDB instance at the given URI
  * @param uri The uri of the database.
  */
 export async function connect (uri: string): Promise<Connection> {
-  const db = await databaseNew(uri)
-  return new Connection(db, uri)
+  return new Connection(uri)
 }
 
 /**
@@ -43,9 +38,9 @@ export class Connection {
   private readonly _uri: string
   private readonly _db: any
 
-  constructor (db: any, uri: string) {
+  constructor (uri: string) {
     this._uri = uri
-    this._db = db
+    this._db = databaseNew(uri)
   }
 
   get uri (): string {
@@ -60,50 +55,17 @@ export class Connection {
   }
 
   /**
-   * Open a table in the database.
-   *
-   * @param name The name of the table.
-   */
-  async openTable (name: string): Promise<Table>
-  /**
-   * Open a table in the database.
-   *
-   * @param name The name of the table.
-   * @param embeddings An embedding function to use on this Table
-   */
-  async openTable<T> (name: string, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
-  async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
+     * Open a table in the database.
+     * @param name The name of the table.
+     */
+  async openTable (name: string): Promise<Table> {
     const tbl = await databaseOpenTable.call(this._db, name)
-    if (embeddings !== undefined) {
-      return new Table(tbl, name, embeddings)
-    } else {
-      return new Table(tbl, name)
-    }
+    return new Table(tbl, name)
   }
 
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param name The name of the table.
-   * @param data Non-empty Array of Records to be inserted into the Table
-   */
-
-  async createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table>
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param name The name of the table.
-   * @param data Non-empty Array of Records to be inserted into the Table
-   * @param embeddings An embedding function to use on this Table
-   */
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
-    const tbl = await tableCreate.call(this._db, name, await fromRecordsToBuffer(data, embeddings))
-    if (embeddings !== undefined) {
-      return new Table(tbl, name, embeddings)
-    } else {
-      return new Table(tbl, name)
-    }
+  async createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table> {
+    await tableCreate.call(this._db, name, await fromRecordsToBuffer(data))
+    return await this.openTable(name)
   }
 
   async createTableArrow (name: string, table: ArrowTable): Promise<Table> {
@@ -113,22 +75,16 @@ export class Connection {
   }
 }
 
-export class Table<T = number[]> {
+/**
+ * A table in a LanceDB database.
+ */
+export class Table {
   private readonly _tbl: any
   private readonly _name: string
-  private readonly _embeddings?: EmbeddingFunction<T>
 
-  constructor (tbl: any, name: string)
-  /**
-   * @param tbl
-   * @param name
-   * @param embeddings An embedding function to use when interacting with this table
-   */
-  constructor (tbl: any, name: string, embeddings: EmbeddingFunction<T>)
-  constructor (tbl: any, name: string, embeddings?: EmbeddingFunction<T>) {
+  constructor (tbl: any, name: string) {
     this._tbl = tbl
     this._name = name
-    this._embeddings = embeddings
   }
 
   get name (): string {
@@ -136,192 +92,72 @@ export class Table<T = number[]> {
   }
 
   /**
-   * Creates a search query to find the nearest neighbors of the given search term
-   * @param query The query search term
-   */
-  search (query: T): Query<T> {
-    return new Query(this._tbl, query, this._embeddings)
+     * Create a search query to find the nearest neighbors of the given query vector.
+     * @param queryVector The query vector.
+     */
+  search (queryVector: number[]): Query {
+    return new Query(this._tbl, queryVector)
   }
 
   /**
-   * Insert records into this Table.
-   *
+   * Insert records into this Table
    * @param data Records to be inserted into the Table
+   *
+   * @param mode Append / Overwrite existing records. Default: Append
    * @return The number of rows added to the table
    */
   async add (data: Array<Record<string, unknown>>): Promise<number> {
-    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Append.toString())
+    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data), WriteMode.Append.toString())
   }
 
-  /**
-   * Insert records into this Table, replacing its contents.
-   *
-   * @param data Records to be inserted into the Table
-   * @return The number of rows added to the table
-   */
   async overwrite (data: Array<Record<string, unknown>>): Promise<number> {
-    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Overwrite.toString())
-  }
-
-  /**
-   * Create an ANN index on this Table vector index.
-   *
-   * @param indexParams The parameters of this Index, @see VectorIndexParams.
-   */
-  async createIndex (indexParams: VectorIndexParams): Promise<any> {
-    return tableCreateVectorIndex.call(this._tbl, indexParams)
-  }
-
-  /**
-   * @deprecated Use [Table.createIndex]
-   */
-  async create_index (indexParams: VectorIndexParams): Promise<any> {
-    return await this.createIndex(indexParams)
+    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data), WriteMode.Overwrite.toString())
   }
 }
 
-interface IvfPQIndexConfig {
-  /**
-   * The column to be indexed
-   */
-  column?: string
-
-  /**
-   * A unique name for the index
-   */
-  index_name?: string
-
-  /**
-   * Metric type, L2 or Cosine
-   */
-  metric_type?: MetricType
-
-  /**
-   * The number of partitions this index
-   */
-  num_partitions?: number
-
-  /**
-   * The max number of iterations for kmeans training.
-   */
-  max_iters?: number
-
-  /**
-   * Train as optimized product quantization.
-   */
-  use_opq?: boolean
-
-  /**
-   * Number of subvectors to build PQ code
-   */
-  num_sub_vectors?: number
-  /**
-   * The number of bits to present one PQ centroid.
-   */
-  num_bits?: number
-
-  /**
-   * Max number of iterations to train OPQ, if `use_opq` is true.
-   */
-  max_opq_iters?: number
-
-  type: 'ivf_pq'
-}
-
-export type VectorIndexParams = IvfPQIndexConfig
-
 /**
  * A builder for nearest neighbor queries for LanceDB.
  */
-export class Query<T = number[]> {
+export class Query {
   private readonly _tbl: any
-  private readonly _query: T
-  private _queryVector?: number[]
+  private readonly _query_vector: number[]
   private _limit: number
-  private _refineFactor?: number
-  private _nprobes: number
-  private _select?: string[]
+  private readonly _refine_factor?: number
+  private readonly _nprobes: number
+  private readonly _columns?: string[]
   private _filter?: string
-  private _metricType?: MetricType
-  private readonly _embeddings?: EmbeddingFunction<T>
+  private readonly _metric = 'L2'
 
-  constructor (tbl: any, query: T, embeddings?: EmbeddingFunction<T>) {
+  constructor (tbl: any, queryVector: number[]) {
     this._tbl = tbl
-    this._query = query
+    this._query_vector = queryVector
     this._limit = 10
     this._nprobes = 20
-    this._refineFactor = undefined
-    this._select = undefined
+    this._refine_factor = undefined
+    this._columns = undefined
     this._filter = undefined
-    this._metricType = undefined
-    this._embeddings = embeddings
   }
 
-  /***
-   * Sets the number of results that will be returned
-   * @param value number of results
-   */
-  limit (value: number): Query<T> {
+  limit (value: number): Query {
     this._limit = value
     return this
   }
 
-  /**
-   * Refine the results by reading extra elements and re-ranking them in memory.
-   * @param value refine factor to use in this query.
-   */
-  refineFactor (value: number): Query<T> {
-    this._refineFactor = value
-    return this
-  }
-
-  /**
-   * The number of probes used. A higher number makes search more accurate but also slower.
-   * @param value The number of probes used.
-   */
-  nprobes (value: number): Query<T> {
-    this._nprobes = value
-    return this
-  }
-
-  /**
-   * A filter statement to be applied to this query.
-   * @param value A filter in the same format used by a sql WHERE clause.
-   */
-  filter (value: string): Query<T> {
+  filter (value: string): Query {
     this._filter = value
     return this
   }
 
-  /** Return only the specified columns.
-   *
-   * @param value Only select the specified columns. If not specified, all columns will be returned.
-   */
-  select (value: string[]): Query<T> {
-    this._select = value
-    return this
-  }
-
   /**
-   * The MetricType used for this Query.
-   * @param value The metric to the. @see MetricType for the different options
-   */
-  metricType (value: MetricType): Query<T> {
-    this._metricType = value
-    return this
-  }
-
-  /**
-   * Execute the query and return the results as an Array of Objects
-   */
+     * Execute the query and return the results as an Array of Objects
+     */
   async execute<T = Record<string, unknown>> (): Promise<T[]> {
-    if (this._embeddings !== undefined) {
-      this._queryVector = (await this._embeddings.embed([this._query]))[0]
+    let buffer
+    if (this._filter != null) {
+      buffer = await tableSearch.call(this._tbl, this._query_vector, this._limit, this._filter)
     } else {
-      this._queryVector = this._query as number[]
+      buffer = await tableSearch.call(this._tbl, this._query_vector, this._limit)
     }
-
-    const buffer = await tableSearch.call(this._tbl, this)
     const data = tableFromIPC(buffer)
     return data.toArray().map((entry: Record<string, unknown>) => {
       const newObject: Record<string, unknown> = {}
@@ -341,18 +177,3 @@ export enum WriteMode {
   Overwrite = 'overwrite',
   Append = 'append'
 }
-
-/**
- * Distance metrics type.
- */
-export enum MetricType {
-  /**
-   * Euclidean distance
-   */
-  L2 = 'l2',
-
-  /**
-   * Cosine distance
-   */
-  Cosine = 'cosine'
-}

node/src/test/embedding/openai.ts

@@ -1,50 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { describe } from 'mocha'
-import { assert } from 'chai'
-
-import { OpenAIEmbeddingFunction } from '../../embedding/openai'
-
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const { OpenAIApi } = require('openai')
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const { stub } = require('sinon')
-
-describe('OpenAPIEmbeddings', function () {
-  const stubValue = {
-    data: {
-      data: [
-        {
-          embedding: Array(1536).fill(1.0)
-        },
-        {
-          embedding: Array(1536).fill(2.0)
-        }
-      ]
-    }
-  }
-
-  describe('#embed', function () {
-    it('should create vector embeddings', async function () {
-      const openAIStub = stub(OpenAIApi.prototype, 'createEmbedding').returns(stubValue)
-      const f = new OpenAIEmbeddingFunction('text', 'sk-key')
-      const vectors = await f.embed(['abc', 'def'])
-      assert.isTrue(openAIStub.calledOnce)
-      assert.equal(vectors.length, 2)
-      assert.deepEqual(vectors[0], stubValue.data.data[0].embedding)
-      assert.deepEqual(vectors[1], stubValue.data.data[1].embedding)
-    })
-  })
-})

node/src/test/io.ts

@@ -1,52 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// IO tests
-
-import { describe } from 'mocha'
-import { assert } from 'chai'
-
-import * as lancedb from '../index'
-
-describe('LanceDB S3 client', function () {
-  if (process.env.TEST_S3_BASE_URL != null) {
-    const baseUri = process.env.TEST_S3_BASE_URL
-    it('should have a valid url', async function () {
-      const uri = `${baseUri}/valid_url`
-      const table = await createTestDB(uri, 2, 20)
-      const con = await lancedb.connect(uri)
-      assert.equal(con.uri, uri)
-
-      const results = await table.search([0.1, 0.3]).limit(5).execute()
-      assert.equal(results.length, 5)
-    })
-  } else {
-    describe.skip('Skip S3 test', function () {})
-  }
-})
-
-async function createTestDB (uri: string, numDimensions: number = 2, numRows: number = 2): Promise<lancedb.Table> {
-  const con = await lancedb.connect(uri)
-
-  const data = []
-  for (let i = 0; i < numRows; i++) {
-    const vector = []
-    for (let j = 0; j < numDimensions; j++) {
-      vector.push(i + (j * 0.1))
-    }
-    data.push({ id: i + 1, name: `name_${i}`, price: i + 10, is_active: (i % 2 === 0), vector })
-  }
-
-  return await con.createTable('vectors', data)
-}

node/src/test/test.ts

@@ -17,7 +17,6 @@ import { assert } from 'chai'
 import { track } from 'temp'
 
 import * as lancedb from '../index'
-import { type EmbeddingFunction, MetricType, Query } from '../index'
 
 describe('LanceDB client', function () {
   describe('when creating a connection to lancedb', function () {
@@ -68,26 +67,10 @@ describe('LanceDB client', function () {
       const uri = await createTestDB()
       const con = await lancedb.connect(uri)
       const table = await con.openTable('vectors')
-      const results = await table.search([0.1, 0.1]).filter('id == 2').execute()
+      const results = await table.search([0.1, 0.3]).filter('id == 2').execute()
       assert.equal(results.length, 1)
       assert.equal(results[0].id, 2)
     })
-
-    it('select only a subset of columns', async function () {
-      const uri = await createTestDB()
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-      const results = await table.search([0.1, 0.1]).select(['is_active']).execute()
-      assert.equal(results.length, 2)
-      // vector and score are always returned
-      assert.isDefined(results[0].vector)
-      assert.isDefined(results[0].score)
-      assert.isDefined(results[0].is_active)
-
-      assert.isUndefined(results[0].id)
-      assert.isUndefined(results[0].name)
-      assert.isUndefined(results[0].price)
-    })
   })
 
   describe('when creating a new dataset', function () {
@@ -113,8 +96,8 @@ describe('LanceDB client', function () {
       const con = await lancedb.connect(dir)
 
       const data = [
-        { id: 1, vector: [0.1, 0.2], price: 10, name: 'a' },
-        { id: 2, vector: [1.1, 1.2], price: 50, name: 'b' }
+        { id: 1, vector: [0.1, 0.2], price: 10 },
+        { id: 2, vector: [1.1, 1.2], price: 50 }
       ]
 
       const table = await con.createTable('vectors', data)
@@ -122,8 +105,8 @@ describe('LanceDB client', function () {
       assert.equal(results.length, 2)
 
       const dataAdd = [
-        { id: 3, vector: [2.1, 2.2], price: 10, name: 'c' },
-        { id: 4, vector: [3.1, 3.2], price: 50, name: 'd' }
+        { id: 3, vector: [2.1, 2.2], price: 10 },
+        { id: 4, vector: [3.1, 3.2], price: 50 }
       ]
       await table.add(dataAdd)
       const resultsAdd = await table.search([0.1, 0.3]).execute()
@@ -147,78 +130,16 @@ describe('LanceDB client', function () {
       assert.equal(resultsAdd.length, 2)
     })
   })
-
-  describe('when creating a vector index', function () {
-    it('overwrite all records in a table', async function () {
-      const uri = await createTestDB(32, 300)
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2 })
-    }).timeout(10_000) // Timeout is high partially because GH macos runner is pretty slow
-  })
-
-  describe('when using a custom embedding function', function () {
-    class TextEmbedding implements EmbeddingFunction<string> {
-      sourceColumn: string
-
-      constructor (targetColumn: string) {
-        this.sourceColumn = targetColumn
-      }
-
-      _embedding_map = new Map<string, number[]>([
-        ['foo', [2.1, 2.2]],
-        ['bar', [3.1, 3.2]]
-      ])
-
-      async embed (data: string[]): Promise<number[][]> {
-        return data.map(datum => this._embedding_map.get(datum) ?? [0.0, 0.0])
-      }
-    }
-
-    it('should encode the original data into embeddings', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-      const embeddings = new TextEmbedding('name')
-
-      const data = [
-        { price: 10, name: 'foo' },
-        { price: 50, name: 'bar' }
-      ]
-      const table = await con.createTable('vectors', data, embeddings)
-      const results = await table.search('foo').execute()
-      assert.equal(results.length, 2)
-    })
-  })
 })
 
-describe('Query object', function () {
-  it('sets custom parameters', async function () {
-    const query = new Query(undefined, [0.1, 0.3])
-      .limit(1)
-      .metricType(MetricType.Cosine)
-      .refineFactor(100)
-      .select(['a', 'b'])
-      .nprobes(20) as Record<string, any>
-    assert.equal(query._limit, 1)
-    assert.equal(query._metricType, MetricType.Cosine)
-    assert.equal(query._refineFactor, 100)
-    assert.equal(query._nprobes, 20)
-    assert.deepEqual(query._select, ['a', 'b'])
-  })
-})
-
-async function createTestDB (numDimensions: number = 2, numRows: number = 2): Promise<string> {
+async function createTestDB (): Promise<string> {
   const dir = await track().mkdir('lancejs')
   const con = await lancedb.connect(dir)
 
-  const data = []
-  for (let i = 0; i < numRows; i++) {
-    const vector = []
-    for (let j = 0; j < numDimensions; j++) {
-      vector.push(i + (j * 0.1))
-    }
-    data.push({ id: i + 1, name: `name_${i}`, price: i + 10, is_active: (i % 2 === 0), vector })
-  }
+  const data = [
+    { id: 1, vector: [0.1, 0.2], name: 'foo', price: 10, is_active: true },
+    { id: 2, vector: [1.1, 1.2], name: 'bar', price: 50, is_active: false }
+  ]
 
   await con.createTable('vectors', data)
   return dir

notebooks/code_qa_bot.ipynb

@@ -72,8 +72,6 @@
     "import lancedb\n",
     "import re\n",
     "import pickle\n",
-    "import requests\n",
-    "import zipfile\n",
     "from pathlib import Path\n",
     "\n",
     "from langchain.document_loaders import UnstructuredHTMLLoader\n",
@@ -87,25 +85,10 @@
   {
    "attachments": {},
    "cell_type": "markdown",
-   "id": "56cc6d50",
+   "id": "6ccf9b2b",
    "metadata": {},
    "source": [
-    "To make this easier, we've downloaded Pandas documentation and stored the raw HTML files for you to download. We'll download them and then use LangChain's HTML document readers to parse them and store them in LanceDB as a vector store, along with relevant metadata."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7da77e75",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "pandas_docs = requests.get(\"https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip\")\n",
-    "with open('/tmp/pandas.documentation.zip', 'wb') as f:\n",
-    "    f.write(pandas_docs.content)\n",
-    "\n",
-    "file = zipfile.ZipFile(\"/tmp/pandas.documentation.zip\")\n",
-    "file.extractall(path=\"/tmp/pandas_docs\")"
+    "You can download the Pandas documentation from https://pandas.pydata.org/docs/. To make sure we're not littering our repo with docs, we won't include it in the LanceDB repo, so download this and store it locally first."
    ]
   },
   {
@@ -154,8 +137,7 @@
     "docs = []\n",
     "\n",
     "if not docs_path.exists():\n",
-    "    for p in Path(\"/tmp/pandas_docs/pandas.documentation\").rglob(\"*.html\"):\n",
-    "        print(p)\n",
+    "    for p in Path(\"./pandas.documentation\").rglob(\"*.html\"):\n",
     "        if p.is_dir():\n",
     "            continue\n",
     "        loader = UnstructuredHTMLLoader(p)\n",

python/lancedb/db.py

@@ -13,16 +13,13 @@
 
 from __future__ import annotations
 
-import os
 from pathlib import Path
-import os
 
 import pyarrow as pa
-from pyarrow import fs
 
 from .common import DATA, URI
 from .table import LanceTable
-from .util import get_uri_scheme, get_uri_location
+from .util import get_uri_scheme
 
 
 class LanceDBConnection:
@@ -50,20 +47,11 @@ class LanceDBConnection:
         -------
         A list of table names.
         """
-        try:
-            filesystem, path = fs.FileSystem.from_uri(self.uri)
-        except pa.ArrowInvalid:
-            raise NotImplementedError(
-                "Unsupported scheme: " + self.uri
-            )
-
-        try:
-            paths = filesystem.get_file_info(fs.FileSelector(get_uri_location(self.uri)))
-        except FileNotFoundError:
-            # It is ok if the file does not exist since it will be created
-            paths = []
-        tables = [os.path.splitext(file_info.base_name)[0] for file_info in paths if file_info.extension == 'lance']
-        return tables
+        if get_uri_scheme(self.uri) == "file":
+            return [p.stem for p in Path(self.uri).glob("*.lance")]
+        raise NotImplementedError(
+            "List table_names is only supported for local filesystem for now"
+        )
 
     def __len__(self) -> int:
         return len(self.table_names())
@@ -124,15 +112,3 @@ class LanceDBConnection:
         A LanceTable object representing the table.
         """
         return LanceTable(self, name)
-
-    def drop_table(self, name: str):
-        """Drop a table from the database.
-
-        Parameters
-        ----------
-        name: str
-            The name of the table.
-        """
-        filesystem, path = pa.fs.FileSystem.from_uri(self.uri)
-        table_path = os.path.join(path, name + ".lance")
-        filesystem.delete_dir(table_path)

python/lancedb/fts.py

@@ -1,130 +0,0 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-"""Full text search index using tantivy-py"""
-import os
-from typing import List, Tuple
-
-import pyarrow as pa
-
-try:
-    import tantivy
-except ImportError:
-    raise ImportError(
-        "Please install tantivy-py `pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985` to use the full text search feature."
-    )
-
-from .table import LanceTable
-
-
-def create_index(index_path: str, text_fields: List[str]) -> tantivy.Index:
-    """
-    Create a new Index (not populated)
-
-    Parameters
-    ----------
-    index_path : str
-        Path to the index directory
-    text_fields : List[str]
-        List of text fields to index
-
-    Returns
-    -------
-    index : tantivy.Index
-        The index object (not yet populated)
-    """
-    # Declaring our schema.
-    schema_builder = tantivy.SchemaBuilder()
-    # special field that we'll populate with row_id
-    schema_builder.add_integer_field("doc_id", stored=True)
-    # data fields
-    for name in text_fields:
-        schema_builder.add_text_field(name, stored=True)
-    schema = schema_builder.build()
-    os.makedirs(index_path, exist_ok=True)
-    index = tantivy.Index(schema, path=index_path)
-    return index
-
-
-def populate_index(index: tantivy.Index, table: LanceTable, fields: List[str]) -> int:
-    """
-    Populate an index with data from a LanceTable
-
-    Parameters
-    ----------
-    index : tantivy.Index
-        The index object
-    table : LanceTable
-        The table to index
-    fields : List[str]
-        List of fields to index
-    """
-    # first check the fields exist and are string or large string type
-    for name in fields:
-        f = table.schema.field(name)  # raises KeyError if not found
-        if not pa.types.is_string(f.type) and not pa.types.is_large_string(f.type):
-            raise TypeError(f"Field {name} is not a string type")
-
-    # create a tantivy writer
-    writer = index.writer()
-    # write data into index
-    dataset = table.to_lance()
-    row_id = 0
-    for b in dataset.to_batches(columns=fields):
-        for i in range(b.num_rows):
-            doc = tantivy.Document()
-            doc.add_integer("doc_id", row_id)
-            for name in fields:
-                doc.add_text(name, b[name][i].as_py())
-            writer.add_document(doc)
-            row_id += 1
-    # commit changes
-    writer.commit()
-    return row_id
-
-
-def search_index(
-    index: tantivy.Index, query: str, limit: int = 10
-) -> Tuple[Tuple[int], Tuple[float]]:
-    """
-    Search an index for a query
-
-    Parameters
-    ----------
-    index : tantivy.Index
-        The index object
-    query : str
-        The query string
-    limit : int
-        The maximum number of results to return
-
-    Returns
-    -------
-    ids_and_score: list[tuple[int], tuple[float]]
-        A tuple of two tuples, the first containing the document ids
-        and the second containing the scores
-    """
-    searcher = index.searcher()
-    query = index.parse_query(query)
-    # get top results
-    results = searcher.search(query, limit)
-    if results.count == 0:
-        return tuple(), tuple()
-    return tuple(
-        zip(
-            *[
-                (searcher.doc(doc_address)["doc_id"][0], score)
-                for score, doc_address in results.hits
-            ]
-        )
-    )

python/lancedb/query.py

@@ -14,7 +14,6 @@ from __future__ import annotations
 
 import numpy as np
 import pandas as pd
-import pyarrow as pa
 
 from .common import VECTOR_COLUMN_NAME
 
@@ -132,6 +131,7 @@ class LanceQueryBuilder:
         vector and the returned vector.
         """
         ds = self._table.to_lance()
+        # TODO indexed search
         tbl = ds.to_table(
             columns=self._columns,
             filter=self._where,
@@ -145,28 +145,3 @@ class LanceQueryBuilder:
             },
         )
         return tbl.to_pandas()
-
-
-class LanceFtsQueryBuilder(LanceQueryBuilder):
-    def to_df(self) -> pd.DataFrame:
-        try:
-            import tantivy
-        except ImportError:
-            raise ImportError(
-                "Please install tantivy-py `pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985` to use the full text search feature."
-            )
-
-        from .fts import search_index
-
-        # get the index path
-        index_path = self._table._get_fts_index_path()
-        # open the index
-        index = tantivy.Index.open(index_path)
-        # get the scores and doc ids
-        row_ids, scores = search_index(index, self._query, self._limit)
-        if len(row_ids) == 0:
-            return pd.DataFrame()
-        scores = pa.array(scores)
-        output_tbl = self._table.to_lance().take(row_ids, columns=self._columns)
-        output_tbl = output_tbl.append_column("score", scores)
-        return output_tbl.to_pandas()

python/lancedb/table.py

@@ -14,9 +14,7 @@
 from __future__ import annotations
 
 import os
-import shutil
 from functools import cached_property
-from typing import List, Union
 
 import lance
 import numpy as np
@@ -26,8 +24,7 @@ from lance import LanceDataset
 from lance.vector import vec_to_table
 
 from .common import DATA, VEC, VECTOR_COLUMN_NAME
-from .query import LanceFtsQueryBuilder, LanceQueryBuilder
-from .util import get_uri_scheme
+from .query import LanceQueryBuilder
 
 
 def _sanitize_data(data, schema):
@@ -133,27 +130,6 @@ class LanceTable:
         )
         self._reset_dataset()
 
-    def create_fts_index(self, field_names: Union[str, List[str]]):
-        """Create a full-text search index on the table.
-
-        Warning - this API is highly experimental and is highly likely to change
-        in the future.
-
-        Parameters
-        ----------
-        field_names: str or list of str
-            The name(s) of the field to index.
-        """
-        from .fts import create_index, populate_index
-
-        if isinstance(field_names, str):
-            field_names = [field_names]
-        index = create_index(self._get_fts_index_path(), field_names)
-        populate_index(index, self, field_names)
-
-    def _get_fts_index_path(self):
-        return os.path.join(self._dataset_uri, "_indices", "tantivy")
-
     @cached_property
     def _dataset(self) -> LanceDataset:
         return lance.dataset(self._dataset_uri, version=self._version)
@@ -182,7 +158,7 @@ class LanceTable:
         self._reset_dataset()
         return len(self)
 
-    def search(self, query: Union[VEC, str]) -> LanceQueryBuilder:
+    def search(self, query: VEC) -> LanceQueryBuilder:
         """Create a search query to find the nearest neighbors
         of the given query vector.
 
@@ -198,10 +174,6 @@ class LanceTable:
         and also the "score" column which is the distance between the query
         vector and the returned vector.
         """
-        if isinstance(query, str):
-            # fts
-            return LanceFtsQueryBuilder(self, query)
-
         if isinstance(query, list):
             query = np.array(query)
         if isinstance(query, np.ndarray):
@@ -253,7 +225,8 @@ def _sanitize_vector_column(data: pa.Table, vector_column_name: str) -> pa.Table
     vector_column_name: str
         The name of the vector column.
     """
-    if vector_column_name not in data.column_names:
+    i = data.column_names.index(vector_column_name)
+    if i < 0:
         raise ValueError(f"Missing vector column: {vector_column_name}")
     vec_arr = data[vector_column_name].combine_chunks()
     if pa.types.is_fixed_size_list(vec_arr.type):
@@ -265,4 +238,4 @@ def _sanitize_vector_column(data: pa.Table, vector_column_name: str) -> pa.Table
         values = values.cast(pa.float32())
     list_size = len(values) / len(data)
     vec_arr = pa.FixedSizeListArray.from_arrays(values, list_size)
-    return data.set_column(data.column_names.index(vector_column_name), vector_column_name, vec_arr)
+    return data.set_column(i, vector_column_name, vec_arr)

python/lancedb/util.py

@@ -41,23 +41,3 @@ def get_uri_scheme(uri: str) -> str:
         # So we add special handling here for schemes that are a single character
         scheme = "file"
     return scheme
-
-
-def get_uri_location(uri: str) -> str:
-    """
-    Get the location of a URI. If the parameter is not a url, assumes it is just a path
-
-    Parameters
-    ----------
-    uri : str
-        The URI to parse.
-
-    Returns
-    -------
-    str: Location part of the URL, without scheme
-    """
-    parsed = urlparse(uri)
-    if not parsed.netloc:
-        return parsed.path
-    else:
-        return parsed.netloc + parsed.path

python/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "lancedb"
-version = "0.1.6"
-dependencies = ["pylance>=0.4.17", "ratelimiter", "retry", "tqdm"]
+version = "0.1.2"
+dependencies = ["pylance>=0.4.6", "ratelimiter", "retry", "tqdm"]
 description = "lancedb"
 authors = [
     { name = "LanceDB Devs", email = "dev@lancedb.com" },
@@ -33,11 +33,11 @@ classifiers = [
 ]
 
 [project.urls]
-repository = "https://github.com/lancedb/lancedb"
+repository = "https://github.com/eto-ai/lancedb"
 
 [project.optional-dependencies]
 tests = [
-    "pytest", "pytest-mock"
+    "pytest"
 ]
 dev = [
     "ruff", "pre-commit", "black"

python/tests/test_db.py

@@ -97,26 +97,3 @@ def test_create_mode(tmp_path):
     )
     tbl = db.create_table("test", data=new_data, mode="overwrite")
     assert tbl.to_pandas().item.tolist() == ["fizz", "buzz"]
-
-
-def test_delete_table(tmp_path):
-    db = lancedb.connect(tmp_path)
-    data = pd.DataFrame(
-        {
-            "vector": [[3.1, 4.1], [5.9, 26.5]],
-            "item": ["foo", "bar"],
-            "price": [10.0, 20.0],
-        }
-    )
-    db.create_table("test", data=data)
-
-    with pytest.raises(Exception):
-        db.create_table("test", data=data)
-
-    assert db.table_names() == ["test"]
-
-    db.drop_table("test")
-    assert db.table_names() == []
-
-    db.create_table("test", data=data)
-    assert db.table_names() == ["test"]

python/tests/test_embeddings.py

@@ -14,6 +14,7 @@ import sys
 
 import numpy as np
 import pyarrow as pa
+
 from lancedb.embeddings import with_embeddings
 
 

python/tests/test_fts.py

@@ -1,91 +0,0 @@
-# Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-import os
-import random
-
-import lancedb.fts
-import numpy as np
-import pandas as pd
-import pytest
-import tantivy
-
-import lancedb as ldb
-
-
-@pytest.fixture
-def table(tmp_path) -> ldb.table.LanceTable:
-    db = ldb.connect(tmp_path)
-    vectors = [np.random.randn(128) for _ in range(100)]
-
-    nouns = ("puppy", "car", "rabbit", "girl", "monkey")
-    verbs = ("runs", "hits", "jumps", "drives", "barfs")
-    adv = ("crazily.", "dutifully.", "foolishly.", "merrily.", "occasionally.")
-    adj = ("adorable", "clueless", "dirty", "odd", "stupid")
-    text = [
-        " ".join(
-            [
-                nouns[random.randrange(0, 5)],
-                verbs[random.randrange(0, 5)],
-                adv[random.randrange(0, 5)],
-                adj[random.randrange(0, 5)],
-            ]
-        )
-        for _ in range(100)
-    ]
-    table = db.create_table(
-        "test", data=pd.DataFrame({"vector": vectors, "text": text, "text2": text})
-    )
-    return table
-
-
-def test_create_index(tmp_path):
-    index = ldb.fts.create_index(str(tmp_path / "index"), ["text"])
-    assert isinstance(index, tantivy.Index)
-    assert os.path.exists(str(tmp_path / "index"))
-
-
-def test_populate_index(tmp_path, table):
-    index = ldb.fts.create_index(str(tmp_path / "index"), ["text"])
-    assert ldb.fts.populate_index(index, table, ["text"]) == len(table)
-
-
-def test_search_index(tmp_path, table):
-    index = ldb.fts.create_index(str(tmp_path / "index"), ["text"])
-    ldb.fts.populate_index(index, table, ["text"])
-    index.reload()
-    results = ldb.fts.search_index(index, query="puppy", limit=10)
-    assert len(results) == 2
-    assert len(results[0]) == 10  # row_ids
-    assert len(results[1]) == 10  # scores
-
-
-def test_create_index_from_table(tmp_path, table):
-    table.create_fts_index("text")
-    df = table.search("puppy").limit(10).select(["text"]).to_df()
-    assert len(df) == 10
-    assert "text" in df.columns
-
-
-def test_create_index_multiple_columns(tmp_path, table):
-    table.create_fts_index(["text", "text2"])
-    df = table.search("puppy").limit(10).to_df()
-    assert len(df) == 10
-    assert "text" in df.columns
-    assert "text2" in df.columns
-
-
-def test_empty_rs(tmp_path, table, mocker):
-    table.create_fts_index(["text", "text2"])
-    mocker.patch("lancedb.fts.search_index", return_value=([], []))
-    df = table.search("puppy").limit(10).to_df()
-    assert len(df) == 0

python/tests/test_io.py

@@ -1,49 +0,0 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-import os
-import pytest
-
-import lancedb
-
-# You need to setup AWS credentials an a base path to run this test. Example
-#    AWS_PROFILE=default TEST_S3_BASE_URL=s3://my_bucket/dataset pytest tests/test_io.py
-
-@pytest.mark.skipif(
-    (os.environ.get("TEST_S3_BASE_URL") is None),
-    reason="please setup s3 base url",
-)
-def test_s3_io():
-    db = lancedb.connect(os.environ.get("TEST_S3_BASE_URL"))
-    assert db.table_names() == []
-
-    table = db.create_table(
-        "test",
-        data=[
-            {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-            {"vector": [5.9, 26.5], "item": "bar", "price": 20.0},
-        ],
-    )
-    rs = table.search([100, 100]).limit(1).to_df()
-    assert len(rs) == 1
-    assert rs["item"].iloc[0] == "bar"
-
-    rs = table.search([100, 100]).where("price < 15").limit(2).to_df()
-    assert len(rs) == 1
-    assert rs["item"].iloc[0] == "foo"
-
-    assert db.table_names() == ["test"]
-    assert "test" in db
-    assert len(db) == 1
-
-    assert db.open_table("test").name == db["test"].name

python/tests/test_query.py

@@ -17,6 +17,7 @@ import pandas as pd
 import pandas.testing as tm
 import pyarrow as pa
 import pytest
+
 from lancedb.query import LanceQueryBuilder
 
 

python/tests/test_table.py

@@ -16,6 +16,7 @@ from pathlib import Path
 import pandas as pd
 import pyarrow as pa
 import pytest
+
 from lancedb.table import LanceTable
 
 

rust/ffi/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb-node"
-version = "0.1.0"
+version = "0.1.2"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
 edition = "2018"
@@ -15,7 +15,7 @@ arrow-ipc = "37.0"
 arrow-schema = "37.0"
 once_cell = "1"
 futures = "0.3"
-lance = "0.4.17"
+lance = "0.4.3"
 vectordb = { path = "../../vectordb" }
 tokio = { version = "1.23", features = ["rt-multi-thread"] }
 neon = {version = "0.10.1", default-features = false, features = ["channel-api", "napi-6", "promise-api", "task-api"] }

rust/ffi/node/release_process.md

@@ -0,0 +1,85 @@
+
+How to release the node module
+
+### 1. Bump the versions
+
+<!-- TODO: we also need to bump the optional dependencies for node! -->
+
+```shell
+pushd rust/vectordb
+cargo bump minor
+popd
+
+pushd rust/ffi/node
+cargo bump minor
+popd
+
+pushd python
+cargo bump minor
+popd
+
+pushd node
+npm version minor
+popd
+
+git add -u
+git commit -m "Bump versions"
+git push
+```
+
+### 2. Push a new tag
+
+```shell
+git tag vX.X.X
+git push --tag vX.X.X
+```
+
+When the tag is pushed, GitHub actions will start building the libraries and
+will upload them to a draft release. Wait for those jobs to complete.
+
+### 3. Publish the release
+
+Once the jobs are complete, you can edit the 
+
+2. Push a tag, such as vX.X.X. Once the tag is pushrf, GitHub actions will start
+   building the native libraries and uploading them to a draft release. Wait for
+   those jobs to complete.
+3. If the libraries are successful, edit the changelog and then publish the
+   release. Once you publish, a new action will start and upload all the 
+   release artifacts to npm.
+
+## Manual process
+
+You can build the artifacts locally on a MacOS machine.
+
+### Build the MacOS release libraries
+
+One-time setup:
+
+```shell
+rustup target add x86_64-apple-darwin aarch64-apple-darwin
+```
+
+To build:
+
+```shell
+bash ci/build_macos_artifacts.sh
+```
+
+### Build the Linux release libraries
+
+One-time setup, building the Docker container
+
+```shell
+cat ci/ubuntu_build.dockerfile | docker build -t lancedb-node-build -
+```
+
+To build:
+
+```shell
+docker run \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -v $(pwd):/io -w /io \
+    lancedb-node-build \
+    bash ci/build_linux_artifacts.sh
+```

rust/ffi/node/src/index.rs

@@ -1,15 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-pub mod vector;

rust/ffi/node/src/index/vector.rs

@@ -1,128 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::convert::TryFrom;
-
-use lance::index::vector::ivf::IvfBuildParams;
-use lance::index::vector::pq::PQBuildParams;
-use lance::index::vector::MetricType;
-use neon::context::FunctionContext;
-use neon::prelude::*;
-
-use vectordb::index::vector::{IvfPQIndexBuilder, VectorIndexBuilder};
-
-use crate::{runtime, JsTable};
-
-pub(crate) fn table_create_vector_index(mut cx: FunctionContext) -> JsResult<JsPromise> {
-    let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
-    let index_params = cx.argument::<JsObject>(0)?;
-    let index_params_builder = get_index_params_builder(&mut cx, index_params).unwrap();
-
-    let rt = runtime(&mut cx)?;
-    let channel = cx.channel();
-
-    let (deferred, promise) = cx.promise();
-    let table = js_table.table.clone();
-
-    rt.block_on(async move {
-        let add_result = table
-            .lock()
-            .unwrap()
-            .create_index(&index_params_builder)
-            .await;
-
-        deferred.settle_with(&channel, move |mut cx| {
-            add_result
-                .map(|_| cx.undefined())
-                .or_else(|err| cx.throw_error(err.to_string()))
-        });
-    });
-    Ok(promise)
-}
-
-fn get_index_params_builder(
-    cx: &mut FunctionContext,
-    obj: Handle<JsObject>,
-) -> Result<impl VectorIndexBuilder, String> {
-    let idx_type = obj
-        .get::<JsString, _, _>(cx, "type")
-        .map_err(|t| t.to_string())?
-        .value(cx);
-
-    match idx_type.as_str() {
-        "ivf_pq" => {
-            let mut index_builder: IvfPQIndexBuilder = IvfPQIndexBuilder::new();
-            let mut pq_params = PQBuildParams::default();
-
-            obj.get_opt::<JsString, _, _>(cx, "column")
-                .map_err(|t| t.to_string())?
-                .map(|s| index_builder.column(s.value(cx)));
-
-            obj.get_opt::<JsString, _, _>(cx, "index_name")
-                .map_err(|t| t.to_string())?
-                .map(|s| index_builder.index_name(s.value(cx)));
-
-            obj.get_opt::<JsString, _, _>(cx, "metric_type")
-                .map_err(|t| t.to_string())?
-                .map(|s| MetricType::try_from(s.value(cx).as_str()))
-                .map(|mt| {
-                    let metric_type = mt.unwrap();
-                    index_builder.metric_type(metric_type);
-                    pq_params.metric_type = metric_type;
-                });
-
-            let num_partitions = obj
-                .get_opt::<JsNumber, _, _>(cx, "num_partitions")
-                .map_err(|t| t.to_string())?
-                .map(|s| s.value(cx) as usize);
-
-            let max_iters = obj
-                .get_opt::<JsNumber, _, _>(cx, "max_iters")
-                .map_err(|t| t.to_string())?
-                .map(|s| s.value(cx) as usize);
-
-            num_partitions.map(|np| {
-                let max_iters = max_iters.unwrap_or(50);
-                let ivf_params = IvfBuildParams {
-                    num_partitions: np,
-                    max_iters,
-                };
-                index_builder.ivf_params(ivf_params)
-            });
-
-            obj.get_opt::<JsBoolean, _, _>(cx, "use_opq")
-                .map_err(|t| t.to_string())?
-                .map(|s| pq_params.use_opq = s.value(cx));
-
-            obj.get_opt::<JsNumber, _, _>(cx, "num_sub_vectors")
-                .map_err(|t| t.to_string())?
-                .map(|s| pq_params.num_sub_vectors = s.value(cx) as usize);
-
-            obj.get_opt::<JsNumber, _, _>(cx, "num_bits")
-                .map_err(|t| t.to_string())?
-                .map(|s| pq_params.num_bits = s.value(cx) as usize);
-
-            obj.get_opt::<JsNumber, _, _>(cx, "max_iters")
-                .map_err(|t| t.to_string())?
-                .map(|s| pq_params.max_iters = s.value(cx) as usize);
-
-            obj.get_opt::<JsNumber, _, _>(cx, "max_opq_iters")
-                .map_err(|t| t.to_string())?
-                .map(|s| pq_params.max_opq_iters = s.value(cx) as usize);
-
-            Ok(index_builder)
-        }
-        t => Err(format!("{} is not a valid index type", t).to_string()),
-    }
-}

rust/ffi/node/src/lib.rs

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use std::collections::HashMap;
-use std::convert::TryFrom;
 use std::ops::Deref;
 use std::sync::{Arc, Mutex};
 
@@ -22,7 +21,6 @@ use arrow_ipc::writer::FileWriter;
 use futures::{TryFutureExt, TryStreamExt};
 use lance::arrow::RecordBatchBuffer;
 use lance::dataset::WriteMode;
-use lance::index::vector::MetricType;
 use neon::prelude::*;
 use neon::types::buffer::TypedArray;
 use once_cell::sync::OnceCell;
@@ -36,18 +34,17 @@ use crate::arrow::arrow_buffer_to_record_batch;
 
 mod arrow;
 mod convert;
-mod index;
 
 struct JsDatabase {
     database: Arc<Database>,
 }
 
-impl Finalize for JsDatabase {}
-
 struct JsTable {
     table: Arc<Mutex<Table>>,
 }
 
+impl Finalize for JsDatabase {}
+
 impl Finalize for JsTable {}
 
 fn runtime<'a, C: Context<'a>>(cx: &mut C) -> NeonResult<&'static Runtime> {
@@ -56,46 +53,23 @@ fn runtime<'a, C: Context<'a>>(cx: &mut C) -> NeonResult<&'static Runtime> {
     RUNTIME.get_or_try_init(|| Runtime::new().or_else(|err| cx.throw_error(err.to_string())))
 }
 
-fn database_new(mut cx: FunctionContext) -> JsResult<JsPromise> {
+fn database_new(mut cx: FunctionContext) -> JsResult<JsBox<JsDatabase>> {
     let path = cx.argument::<JsString>(0)?.value(&mut cx);
-
-    let rt = runtime(&mut cx)?;
-    let channel = cx.channel();
-    let (deferred, promise) = cx.promise();
-
-    rt.spawn(async move {
-        let database = Database::connect(&path).await;
-
-        deferred.settle_with(&channel, move |mut cx| {
-            let db = JsDatabase {
-                database: Arc::new(database.or_else(|err| cx.throw_error(err.to_string()))?),
-            };
-            Ok(cx.boxed(db))
-        });
-    });
-    Ok(promise)
+    let db = JsDatabase {
+        database: Arc::new(Database::connect(path).or_else(|err| cx.throw_error(err.to_string()))?),
+    };
+    Ok(cx.boxed(db))
 }
 
-fn database_table_names(mut cx: FunctionContext) -> JsResult<JsPromise> {
+fn database_table_names(mut cx: FunctionContext) -> JsResult<JsArray> {
     let db = cx
         .this()
         .downcast_or_throw::<JsBox<JsDatabase>, _>(&mut cx)?;
-
-    let rt = runtime(&mut cx)?;
-    let (deferred, promise) = cx.promise();
-    let channel = cx.channel();
-    let database = db.database.clone();
-
-    rt.spawn(async move {
-        let tables_rst = database.table_names().await;
-
-        deferred.settle_with(&channel, move |mut cx| {
-            let tables = tables_rst.or_else(|err| cx.throw_error(err.to_string()))?;
-            let table_names = convert::vec_str_to_array(&tables, &mut cx);
-            table_names
-        });
-    });
-    Ok(promise)
+    let tables = db
+        .database
+        .table_names()
+        .or_else(|err| cx.throw_error(err.to_string()))?;
+    convert::vec_str_to_array(&tables, &mut cx)
 }
 
 fn database_open_table(mut cx: FunctionContext) -> JsResult<JsPromise> {
@@ -110,12 +84,10 @@ fn database_open_table(mut cx: FunctionContext) -> JsResult<JsPromise> {
 
     let (deferred, promise) = cx.promise();
     rt.spawn(async move {
-        let table_rst = database.open_table(&table_name).await;
+        let table_rst = database.open_table(table_name).await;
 
         deferred.settle_with(&channel, move |mut cx| {
-            let table = Arc::new(Mutex::new(
-                table_rst.or_else(|err| cx.throw_error(err.to_string()))?,
-            ));
+            let table = Arc::new(Mutex::new(table_rst.or_else(|err| cx.throw_error(err.to_string()))?));
             Ok(cx.boxed(JsTable { table }))
         });
     });
@@ -124,43 +96,15 @@ fn database_open_table(mut cx: FunctionContext) -> JsResult<JsPromise> {
 
 fn table_search(mut cx: FunctionContext) -> JsResult<JsPromise> {
     let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
-    let query_obj = cx.argument::<JsObject>(0)?;
-
-    let limit = query_obj
-        .get::<JsNumber, _, _>(&mut cx, "_limit")?
-        .value(&mut cx);
-    let select = query_obj
-        .get_opt::<JsArray, _, _>(&mut cx, "_select")?
-        .map(|arr| {
-            let js_array = arr.deref();
-            let mut projection_vec: Vec<String> = Vec::new();
-            for i in 0..js_array.len(&mut cx) {
-                let entry: Handle<JsString> = js_array.get(&mut cx, i).unwrap();
-                projection_vec.push(entry.value(&mut cx));
-            }
-            projection_vec
-        });
-    let filter = query_obj
-        .get_opt::<JsString, _, _>(&mut cx, "_filter")?
-        .map(|s| s.value(&mut cx));
-    let refine_factor = query_obj
-        .get_opt::<JsNumber, _, _>(&mut cx, "_refineFactor")?
-        .map(|s| s.value(&mut cx))
-        .map(|i| i as u32);
-    let nprobes = query_obj
-        .get::<JsNumber, _, _>(&mut cx, "_nprobes")?
-        .value(&mut cx) as usize;
-    let metric_type = query_obj
-        .get_opt::<JsString, _, _>(&mut cx, "_metricType")?
-        .map(|s| s.value(&mut cx))
-        .map(|s| MetricType::try_from(s.as_str()).unwrap());
+    let query_vector = cx.argument::<JsArray>(0)?; //. .as_value(&mut cx);
+    let limit = cx.argument::<JsNumber>(1)?.value(&mut cx);
+    let filter = cx.argument_opt(2).map(|f| f.downcast_or_throw::<JsString, _>(&mut cx).unwrap().value(&mut cx));
 
     let rt = runtime(&mut cx)?;
     let channel = cx.channel();
 
     let (deferred, promise) = cx.promise();
     let table = js_table.table.clone();
-    let query_vector = query_obj.get::<JsArray, _, _>(&mut cx, "_queryVector")?;
     let query = convert::js_array_to_vec(query_vector.deref(), &mut cx);
 
     rt.spawn(async move {
@@ -169,11 +113,7 @@ fn table_search(mut cx: FunctionContext) -> JsResult<JsPromise> {
             .unwrap()
             .search(Float32Array::from(query))
             .limit(limit as usize)
-            .refine_factor(refine_factor)
-            .nprobes(nprobes)
-            .filter(filter)
-            .metric_type(metric_type)
-            .select(select);
+            .filter(filter);
         let record_batch_stream = builder.execute();
         let results = record_batch_stream
             .and_then(|stream| stream.try_collect::<Vec<_>>().map_err(Error::from))
@@ -221,12 +161,10 @@ fn table_create(mut cx: FunctionContext) -> JsResult<JsPromise> {
 
     rt.block_on(async move {
         let batch_reader: Box<dyn RecordBatchReader> = Box::new(RecordBatchBuffer::new(batches));
-        let table_rst = database.create_table(&table_name, batch_reader).await;
+        let table_rst = database.create_table(table_name, batch_reader).await;
 
         deferred.settle_with(&channel, move |mut cx| {
-            let table = Arc::new(Mutex::new(
-                table_rst.or_else(|err| cx.throw_error(err.to_string()))?,
-            ));
+            let table = Arc::new(Mutex::new(table_rst.or_else(|err| cx.throw_error(err.to_string()))?));
             Ok(cx.boxed(JsTable { table }))
         });
     });
@@ -240,7 +178,9 @@ fn table_add(mut cx: FunctionContext) -> JsResult<JsPromise> {
         ("overwrite", WriteMode::Overwrite),
     ]);
 
-    let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
+    let js_table = cx
+        .this()
+        .downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
     let buffer = cx.argument::<JsBuffer>(0)?;
     let write_mode = cx.argument::<JsString>(1)?.value(&mut cx);
     let batches = arrow_buffer_to_record_batch(buffer.as_slice(&mut cx));
@@ -264,6 +204,7 @@ fn table_add(mut cx: FunctionContext) -> JsResult<JsPromise> {
     Ok(promise)
 }
 
+
 #[neon::main]
 fn main(mut cx: ModuleContext) -> NeonResult<()> {
     cx.export_function("databaseNew", database_new)?;
@@ -272,9 +213,5 @@ fn main(mut cx: ModuleContext) -> NeonResult<()> {
     cx.export_function("tableSearch", table_search)?;
     cx.export_function("tableCreate", table_create)?;
     cx.export_function("tableAdd", table_add)?;
-    cx.export_function(
-        "tableCreateVectorIndex",
-        index::vector::table_create_vector_index,
-    )?;
     Ok(())
 }

rust/vectordb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb"
-version = "0.0.1"
+version = "0.1.2"
 edition = "2021"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
@@ -10,13 +10,9 @@ repository = "https://github.com/lancedb/lancedb"
 
 [dependencies]
 arrow-array = "37.0"
-arrow-data = "37.0"
 arrow-schema = "37.0"
-object_store = "0.5.6"
-snafu = "0.7.4"
-lance = "0.4.17"
+lance = "0.4.3"
 tokio = { version = "1.23", features = ["rt-multi-thread"] }
 
 [dev-dependencies]
 tempfile = "3.5.0"
-rand = { version = "0.8.3", features = ["small_rng"] }

rust/vectordb/src/database.rs

@@ -12,20 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::fs::create_dir_all;
-use std::path::Path;
-
 use arrow_array::RecordBatchReader;
-use lance::io::object_store::ObjectStore;
-use snafu::prelude::*;
+use std::fs::create_dir_all;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
 
-use crate::error::{CreateDirSnafu, Result};
+use crate::error::Result;
 use crate::table::Table;
 
 pub struct Database {
-    object_store: ObjectStore,
-
-    pub(crate) uri: String,
+    pub(crate) path: Arc<PathBuf>,
 }
 
 const LANCE_EXTENSION: &str = "lance";
@@ -41,24 +37,13 @@ impl Database {
     /// # Returns
     ///
     /// * A [Database] object.
-    pub async fn connect(uri: &str) -> Result<Database> {
-        let object_store = ObjectStore::new(uri).await?;
-        if object_store.is_local() {
-            Self::try_create_dir(uri).context(CreateDirSnafu { path: uri })?;
-        }
-        Ok(Database {
-            uri: uri.to_string(),
-            object_store,
-        })
-    }
-
-    /// Try to create a local directory to store the lancedb dataset
-    fn try_create_dir(path: &str) -> core::result::Result<(), std::io::Error> {
-        let path = Path::new(path);
-        if !path.try_exists()? {
+    pub fn connect<P: AsRef<Path>>(path: P) -> Result<Database> {
+        if !path.as_ref().try_exists()? {
             create_dir_all(&path)?;
         }
-        Ok(())
+        Ok(Database {
+            path: Arc::new(path.as_ref().to_path_buf()),
+        })
     }
 
     /// Get the names of all tables in the database.
@@ -66,13 +51,12 @@ impl Database {
     /// # Returns
     ///
     /// * A [Vec<String>] with all table names.
-    pub async fn table_names(&self) -> Result<Vec<String>> {
+    pub fn table_names(&self) -> Result<Vec<String>> {
         let f = self
-            .object_store
-            .read_dir("/")
-            .await?
-            .iter()
-            .map(|fname| Path::new(fname))
+            .path
+            .read_dir()?
+            .flatten()
+            .map(|dir_entry| dir_entry.path())
             .filter(|path| {
                 let is_lance = path
                     .extension()
@@ -92,10 +76,10 @@ impl Database {
 
     pub async fn create_table(
         &self,
-        name: &str,
+        name: String,
         batches: Box<dyn RecordBatchReader>,
     ) -> Result<Table> {
-        Table::create(&self.uri, name, batches).await
+        Table::create(self.path.clone(), name, batches).await
     }
 
     /// Open a table in the database.
@@ -106,8 +90,8 @@ impl Database {
     /// # Returns
     ///
     /// * A [Table] object.
-    pub async fn open_table(&self, name: &str) -> Result<Table> {
-        Table::open(&self.uri, name).await
+    pub async fn open_table(&self, name: String) -> Result<Table> {
+        Table::open(self.path.clone(), name).await
     }
 }
 
@@ -121,10 +105,10 @@ mod tests {
     #[tokio::test]
     async fn test_connect() {
         let tmp_dir = tempdir().unwrap();
-        let uri = tmp_dir.path().to_str().unwrap();
-        let db = Database::connect(uri).await.unwrap();
+        let path_buf = tmp_dir.into_path();
+        let db = Database::connect(&path_buf);
 
-        assert_eq!(db.uri, uri);
+        assert_eq!(db.unwrap().path.as_path(), path_buf.as_path())
     }
 
     #[tokio::test]
@@ -134,16 +118,10 @@ mod tests {
         create_dir_all(tmp_dir.path().join("table2.lance")).unwrap();
         create_dir_all(tmp_dir.path().join("invalidlance")).unwrap();
 
-        let uri = tmp_dir.path().to_str().unwrap();
-        let db = Database::connect(uri).await.unwrap();
-        let tables = db.table_names().await.unwrap();
+        let db = Database::connect(&tmp_dir.into_path()).unwrap();
+        let tables = db.table_names().unwrap();
         assert_eq!(tables.len(), 2);
         assert!(tables.contains(&String::from("table1")));
         assert!(tables.contains(&String::from("table2")));
     }
-
-    #[tokio::test]
-    async fn test_connect_s3() {
-        // let db = Database::connect("s3://bucket/path/to/database").await.unwrap();
-    }
 }

rust/vectordb/src/error.rs

@@ -12,50 +12,32 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use snafu::Snafu;
-
-#[derive(Debug, Snafu)]
-#[snafu(visibility(pub(crate)))]
+#[derive(Debug)]
 pub enum Error {
-    #[snafu(display("LanceDBError: Invalid table name: {name}"))]
-    InvalidTableName { name: String },
-    #[snafu(display("LanceDBError: Table '{name}' was not found"))]
-    TableNotFound { name: String },
-    #[snafu(display("LanceDBError: Table '{name}' already exists"))]
-    TableAlreadyExists { name: String },
-    #[snafu(display("LanceDBError: Unable to created lance dataset at {path}: {source}"))]
-    CreateDir {
-        path: String,
-        source: std::io::Error,
-    },
-    #[snafu(display("LanceDBError: {message}"))]
-    Store { message: String },
-    #[snafu(display("LanceDBError: {message}"))]
-    Lance { message: String },
+    IO(String),
+    Lance(String),
+}
+
+impl std::fmt::Display for Error {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let (catalog, message) = match self {
+            Self::IO(s) => ("I/O", s.as_str()),
+            Self::Lance(s) => ("Lance", s.as_str()),
+        };
+        write!(f, "LanceDBError({catalog}): {message}")
+    }
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
 
+impl From<std::io::Error> for Error {
+    fn from(e: std::io::Error) -> Self {
+        Self::IO(e.to_string())
+    }
+}
+
 impl From<lance::Error> for Error {
     fn from(e: lance::Error) -> Self {
-        Self::Lance {
-            message: e.to_string(),
-        }
-    }
-}
-
-impl From<object_store::Error> for Error {
-    fn from(e: object_store::Error) -> Self {
-        Self::Store {
-            message: e.to_string(),
-        }
-    }
-}
-
-impl From<object_store::path::Error> for Error {
-    fn from(e: object_store::path::Error) -> Self {
-        Self::Store {
-            message: e.to_string(),
-        }
+        Self::Lance(e.to_string())
     }
 }

rust/vectordb/src/index.rs

@@ -1,15 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-pub mod vector;

rust/vectordb/src/index/vector.rs

@@ -1,163 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use lance::index::vector::ivf::IvfBuildParams;
-use lance::index::vector::pq::PQBuildParams;
-use lance::index::vector::{MetricType, VectorIndexParams};
-
-pub trait VectorIndexBuilder {
-    fn get_column(&self) -> Option<String>;
-    fn get_index_name(&self) -> Option<String>;
-    fn build(&self) -> VectorIndexParams;
-}
-
-pub struct IvfPQIndexBuilder {
-    column: Option<String>,
-    index_name: Option<String>,
-    metric_type: Option<MetricType>,
-    ivf_params: Option<IvfBuildParams>,
-    pq_params: Option<PQBuildParams>,
-}
-
-impl IvfPQIndexBuilder {
-    pub fn new() -> IvfPQIndexBuilder {
-        IvfPQIndexBuilder {
-            column: None,
-            index_name: None,
-            metric_type: None,
-            ivf_params: None,
-            pq_params: None,
-        }
-    }
-}
-
-impl IvfPQIndexBuilder {
-    pub fn column(&mut self, column: String) -> &mut IvfPQIndexBuilder {
-        self.column = Some(column);
-        self
-    }
-
-    pub fn index_name(&mut self, index_name: String) -> &mut IvfPQIndexBuilder {
-        self.index_name = Some(index_name);
-        self
-    }
-
-    pub fn metric_type(&mut self, metric_type: MetricType) -> &mut IvfPQIndexBuilder {
-        self.metric_type = Some(metric_type);
-        self
-    }
-
-    pub fn ivf_params(&mut self, ivf_params: IvfBuildParams) -> &mut IvfPQIndexBuilder {
-        self.ivf_params = Some(ivf_params);
-        self
-    }
-
-    pub fn pq_params(&mut self, pq_params: PQBuildParams) -> &mut IvfPQIndexBuilder {
-        self.pq_params = Some(pq_params);
-        self
-    }
-}
-
-impl VectorIndexBuilder for IvfPQIndexBuilder {
-    fn get_column(&self) -> Option<String> {
-        self.column.clone()
-    }
-
-    fn get_index_name(&self) -> Option<String> {
-        self.index_name.clone()
-    }
-
-    fn build(&self) -> VectorIndexParams {
-        let ivf_params = self.ivf_params.clone().unwrap_or(IvfBuildParams::default());
-        let pq_params = self.pq_params.clone().unwrap_or(PQBuildParams::default());
-
-        VectorIndexParams::with_ivf_pq_params(pq_params.metric_type, ivf_params, pq_params)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use lance::index::vector::ivf::IvfBuildParams;
-    use lance::index::vector::pq::PQBuildParams;
-    use lance::index::vector::{MetricType, StageParams};
-
-    use crate::index::vector::{IvfPQIndexBuilder, VectorIndexBuilder};
-
-    #[test]
-    fn test_builder_no_params() {
-        let index_builder = IvfPQIndexBuilder::new();
-        assert!(index_builder.get_column().is_none());
-        assert!(index_builder.get_index_name().is_none());
-
-        let index_params = index_builder.build();
-        assert_eq!(index_params.stages.len(), 2);
-        if let StageParams::Ivf(ivf_params) = index_params.stages.get(0).unwrap() {
-            let default = IvfBuildParams::default();
-            assert_eq!(ivf_params.num_partitions, default.num_partitions);
-            assert_eq!(ivf_params.max_iters, default.max_iters);
-        } else {
-            panic!("Expected first stage to be ivf")
-        }
-
-        if let StageParams::PQ(pq_params) = index_params.stages.get(1).unwrap() {
-            assert_eq!(pq_params.use_opq, false);
-        } else {
-            panic!("Expected second stage to be pq")
-        }
-    }
-
-    #[test]
-    fn test_builder_all_params() {
-        let mut index_builder = IvfPQIndexBuilder::new();
-
-        index_builder
-            .column("c".to_owned())
-            .metric_type(MetricType::Cosine)
-            .index_name("index".to_owned());
-
-        assert_eq!(index_builder.column.clone().unwrap(), "c");
-        assert_eq!(index_builder.metric_type.unwrap(), MetricType::Cosine);
-        assert_eq!(index_builder.index_name.clone().unwrap(), "index");
-
-        let ivf_params = IvfBuildParams::new(500);
-        let mut pq_params = PQBuildParams::default();
-        pq_params.use_opq = true;
-        pq_params.max_iters = 1;
-        pq_params.num_bits = 8;
-        pq_params.num_sub_vectors = 50;
-        pq_params.metric_type = MetricType::Cosine;
-        pq_params.max_opq_iters = 2;
-        index_builder.ivf_params(ivf_params);
-        index_builder.pq_params(pq_params);
-
-        let index_params = index_builder.build();
-        assert_eq!(index_params.stages.len(), 2);
-        if let StageParams::Ivf(ivf_params) = index_params.stages.get(0).unwrap() {
-            assert_eq!(ivf_params.num_partitions, 500);
-        } else {
-            assert!(false, "Expected first stage to be ivf")
-        }
-
-        if let StageParams::PQ(pq_params) = index_params.stages.get(1).unwrap() {
-            assert_eq!(pq_params.use_opq, true);
-            assert_eq!(pq_params.max_iters, 1);
-            assert_eq!(pq_params.num_bits, 8);
-            assert_eq!(pq_params.num_sub_vectors, 50);
-            assert_eq!(pq_params.metric_type, MetricType::Cosine);
-            assert_eq!(pq_params.max_opq_iters, 2);
-        } else {
-            assert!(false, "Expected second stage to be pq")
-        }
-    }
-}

rust/vectordb/src/lib.rs

@@ -14,6 +14,5 @@
 
 pub mod database;
 pub mod error;
-pub mod index;
 pub mod query;
 pub mod table;

rust/vectordb/src/query.rs

@@ -27,10 +27,9 @@ pub struct Query {
     pub query_vector: Float32Array,
     pub limit: usize,
     pub filter: Option<String>,
-    pub select: Option<Vec<String>>,
     pub nprobes: usize,
     pub refine_factor: Option<u32>,
-    pub metric_type: Option<MetricType>,
+    pub metric_type: MetricType,
     pub use_index: bool,
 }
 
@@ -52,10 +51,9 @@ impl Query {
             limit: 10,
             nprobes: 20,
             refine_factor: None,
-            metric_type: None,
+            metric_type: MetricType::L2,
             use_index: false,
-            filter: None,
-            select: None,
+            filter: None
         }
     }
 
@@ -73,13 +71,10 @@ impl Query {
             self.limit,
         )?;
         scanner.nprobs(self.nprobes);
+        scanner.distance_metric(self.metric_type);
         scanner.use_index(self.use_index);
-        self.select
-            .as_ref()
-            .map(|p| scanner.project(p.as_slice()));
         self.filter.as_ref().map(|f| scanner.filter(f));
         self.refine_factor.map(|rf| scanner.refine(rf));
-        self.metric_type.map(|mt| scanner.distance_metric(mt));
         Ok(scanner.try_into_stream().await?)
     }
 
@@ -128,7 +123,7 @@ impl Query {
     /// # Arguments
     ///
     /// * `metric_type` - The distance metric to use. By default [MetricType::L2] is used.
-    pub fn metric_type(mut self, metric_type: Option<MetricType>) -> Query {
+    pub fn metric_type(mut self, metric_type: MetricType) -> Query {
         self.metric_type = metric_type;
         self
     }
@@ -143,23 +138,10 @@ impl Query {
         self
     }
 
-    ///  A filter statement to be applied to this query.
-    ///
-    /// # Arguments
-    ///
-    /// * `filter` -  value A filter in the same format used by a sql WHERE clause.
     pub fn filter(mut self, filter: Option<String>) -> Query {
         self.filter = filter;
         self
     }
-
-    /// Return only the specified columns.
-    ///
-    /// Only select the specified columns. If not specified, all columns will be returned.
-    pub fn select(mut self, columns: Option<Vec<String>>) -> Query {
-        self.select = columns;
-        self
-    }
 }
 
 #[cfg(test)]
@@ -192,14 +174,14 @@ mod tests {
             .limit(100)
             .nprobes(1000)
             .use_index(true)
-            .metric_type(Some(MetricType::Cosine))
+            .metric_type(MetricType::Cosine)
             .refine_factor(Some(999));
 
         assert_eq!(query.query_vector, new_vector);
         assert_eq!(query.limit, 100);
         assert_eq!(query.nprobes, 1000);
         assert_eq!(query.use_index, true);
-        assert_eq!(query.metric_type, Some(MetricType::Cosine));
+        assert_eq!(query.metric_type, MetricType::Cosine);
         assert_eq!(query.refine_factor, Some(999));
     }
 

rust/vectordb/src/table.rs

@@ -12,35 +12,26 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::path::Path;
+use std::path::PathBuf;
 use std::sync::Arc;
 
 use arrow_array::{Float32Array, RecordBatchReader};
 use lance::dataset::{Dataset, WriteMode, WriteParams};
-use lance::index::IndexType;
-use snafu::prelude::*;
 
-use crate::error::{Error, InvalidTableNameSnafu, Result};
-use crate::index::vector::VectorIndexBuilder;
+use crate::error::{Error, Result};
 use crate::query::Query;
 
 pub const VECTOR_COLUMN_NAME: &str = "vector";
+
 pub const LANCE_FILE_EXTENSION: &str = "lance";
 
 /// A table in a LanceDB database.
-#[derive(Debug)]
 pub struct Table {
     name: String,
-    uri: String,
+    path: String,
     dataset: Arc<Dataset>,
 }
 
-impl std::fmt::Display for Table {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "Table({})", self.name)
-    }
-}
-
 impl Table {
     /// Opens an existing Table
     ///
@@ -52,28 +43,18 @@ impl Table {
     /// # Returns
     ///
     /// * A [Table] object.
-    pub async fn open(base_uri: &str, name: &str) -> Result<Self> {
-        let path = Path::new(base_uri);
-
-        let table_uri = path.join(format!("{}.{}", name, LANCE_FILE_EXTENSION));
-        let uri = table_uri
-            .as_path()
+    pub async fn open(base_path: Arc<PathBuf>, name: String) -> Result<Self> {
+        let ds_path = base_path.join(format!("{}.{}", name, LANCE_FILE_EXTENSION));
+        let ds_uri = ds_path
             .to_str()
-            .context(InvalidTableNameSnafu { name })?;
-
-        let dataset = Dataset::open(&uri).await.map_err(|e| match e {
-            lance::Error::DatasetNotFound { .. } => Error::TableNotFound {
-                name: name.to_string(),
-            },
-            e => Error::Lance {
-                message: e.to_string(),
-            },
-        })?;
-        Ok(Table {
-            name: name.to_string(),
-            uri: uri.to_string(),
+            .ok_or(Error::IO(format!("Unable to find table {}", name)))?;
+        let dataset = Dataset::open(ds_uri).await?;
+        let table = Table {
+            name,
+            path: ds_uri.to_string(),
             dataset: Arc::new(dataset),
-        })
+        };
+        Ok(table)
     }
 
     /// Creates a new Table
@@ -88,52 +69,18 @@ impl Table {
     ///
     /// * A [Table] object.
     pub async fn create(
-        base_uri: &str,
-        name: &str,
+        base_path: Arc<PathBuf>,
+        name: String,
         mut batches: Box<dyn RecordBatchReader>,
     ) -> Result<Self> {
-        let base_path = Path::new(base_uri);
-        let table_uri = base_path.join(format!("{}.{}", name, LANCE_FILE_EXTENSION));
-        let uri = table_uri
-            .as_path()
+        let ds_path = base_path.join(format!("{}.{}", name, LANCE_FILE_EXTENSION));
+        let path = ds_path
             .to_str()
-            .context(InvalidTableNameSnafu { name })?
-            .to_string();
-        let dataset = Dataset::write(&mut batches, &uri, Some(WriteParams::default()))
-            .await
-            .map_err(|e| match e {
-                lance::Error::DatasetAlreadyExists { .. } => Error::TableAlreadyExists {
-                    name: name.to_string(),
-                },
-                e => Error::Lance {
-                    message: e.to_string(),
-                },
-            })?;
-        Ok(Table {
-            name: name.to_string(),
-            uri,
-            dataset: Arc::new(dataset),
-        })
-    }
+            .ok_or(Error::IO(format!("Unable to find table {}", name)))?;
 
-    /// Create index on the table.
-    pub async fn create_index(&mut self, index_builder: &impl VectorIndexBuilder) -> Result<()> {
-        use lance::index::DatasetIndexExt;
-
-        let dataset = self
-            .dataset
-            .create_index(
-                &[index_builder
-                    .get_column()
-                    .unwrap_or(VECTOR_COLUMN_NAME.to_string())
-                    .as_str()],
-                IndexType::Vector,
-                index_builder.get_index_name(),
-                &index_builder.build(),
-            )
-            .await?;
-        self.dataset = Arc::new(dataset);
-        Ok(())
+        let dataset =
+            Arc::new(Dataset::write(&mut batches, path, Some(WriteParams::default())).await?);
+        Ok(Table { name, path: path.to_string(), dataset })
     }
 
     /// Insert records into this Table
@@ -148,12 +95,12 @@ impl Table {
     pub async fn add(
         &mut self,
         mut batches: Box<dyn RecordBatchReader>,
-        write_mode: Option<WriteMode>,
+        write_mode: Option<WriteMode>
     ) -> Result<usize> {
         let mut params = WriteParams::default();
         params.mode = write_mode.unwrap_or(WriteMode::Append);
 
-        self.dataset = Arc::new(Dataset::write(&mut batches, &self.uri, Some(params)).await?);
+        self.dataset = Arc::new(Dataset::write(&mut batches, self.path.as_str(), Some(params)).await?);
         Ok(batches.count())
     }
 
@@ -178,88 +125,60 @@ impl Table {
 
 #[cfg(test)]
 mod tests {
-    use std::sync::Arc;
-
-    use arrow_array::{
-        Array, FixedSizeListArray, Float32Array, Int32Array, RecordBatch, RecordBatchReader,
-    };
-    use arrow_data::ArrayDataBuilder;
+    use arrow_array::{Float32Array, Int32Array, RecordBatch, RecordBatchReader};
     use arrow_schema::{DataType, Field, Schema};
     use lance::arrow::RecordBatchBuffer;
     use lance::dataset::{Dataset, WriteMode};
-    use lance::index::vector::ivf::IvfBuildParams;
-    use lance::index::vector::pq::PQBuildParams;
-    use rand::Rng;
+    use std::sync::Arc;
     use tempfile::tempdir;
 
-    use super::*;
-    use crate::index::vector::IvfPQIndexBuilder;
+    use crate::table::Table;
+
+    #[tokio::test]
+    async fn test_new_table_not_exists() {
+        let tmp_dir = tempdir().unwrap();
+        let path_buf = tmp_dir.into_path();
+
+        let table = Table::open(Arc::new(path_buf), "test".to_string()).await;
+        assert!(table.is_err());
+    }
 
     #[tokio::test]
     async fn test_open() {
         let tmp_dir = tempdir().unwrap();
-        let dataset_path = tmp_dir.path().join("test.lance");
-        let uri = tmp_dir.path().to_str().unwrap();
+        let path_buf = tmp_dir.into_path();
 
         let mut batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
-        Dataset::write(&mut batches, dataset_path.to_str().unwrap(), None)
+        Dataset::write(
+            &mut batches,
+            path_buf.join("test.lance").to_str().unwrap(),
+            None,
+        )
+        .await
+        .unwrap();
+
+        let table = Table::open(Arc::new(path_buf), "test".to_string())
             .await
             .unwrap();
 
-        let table = Table::open(uri, "test").await.unwrap();
-
         assert_eq!(table.name, "test")
     }
 
-    #[tokio::test]
-    async fn test_open_not_found() {
-        let tmp_dir = tempdir().unwrap();
-        let uri = tmp_dir.path().to_str().unwrap();
-        let table = Table::open(uri, "test").await;
-        assert!(matches!(table.unwrap_err(), Error::TableNotFound { .. }));
-    }
-
-    #[test]
-    fn test_object_store_path() {
-        use std::path::Path as StdPath;
-        let p = StdPath::new("s3://bucket/path/to/file");
-        let c = p.join("subfile");
-        assert_eq!(c.to_str().unwrap(), "s3://bucket/path/to/file/subfile");
-    }
-
-    #[tokio::test]
-    async fn test_create_already_exists() {
-        let tmp_dir = tempdir().unwrap();
-        let uri = tmp_dir.path().to_str().unwrap();
-
-        let batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
-        let schema = batches.schema().clone();
-        Table::create(&uri, "test", batches).await.unwrap();
-
-        let batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
-        let result = Table::create(&uri, "test", batches).await;
-        assert!(matches!(
-            result.unwrap_err(),
-            Error::TableAlreadyExists { .. }
-        ));
-    }
-
     #[tokio::test]
     async fn test_add() {
         let tmp_dir = tempdir().unwrap();
-        let uri = tmp_dir.path().to_str().unwrap();
+        let path_buf = tmp_dir.into_path();
 
         let batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
         let schema = batches.schema().clone();
-        let mut table = Table::create(&uri, "test", batches).await.unwrap();
+        let mut table = Table::create(Arc::new(path_buf), "test".to_string(), batches).await.unwrap();
         assert_eq!(table.count_rows().await.unwrap(), 10);
 
-        let new_batches: Box<dyn RecordBatchReader> =
-            Box::new(RecordBatchBuffer::new(vec![RecordBatch::try_new(
-                schema,
-                vec![Arc::new(Int32Array::from_iter_values(100..110))],
-            )
-            .unwrap()]));
+        let new_batches: Box<dyn RecordBatchReader> = Box::new(RecordBatchBuffer::new(vec![RecordBatch::try_new(
+            schema,
+            vec![Arc::new(Int32Array::from_iter_values(100..110))],
+        )
+       .unwrap()]));
 
         table.add(new_batches, None).await.unwrap();
         assert_eq!(table.count_rows().await.unwrap(), 20);
@@ -269,24 +188,19 @@ mod tests {
     #[tokio::test]
     async fn test_add_overwrite() {
         let tmp_dir = tempdir().unwrap();
-        let uri = tmp_dir.path().to_str().unwrap();
+        let path_buf = tmp_dir.into_path();
 
         let batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
         let schema = batches.schema().clone();
-        let mut table = Table::create(uri, "test", batches).await.unwrap();
+        let mut table = Table::create(Arc::new(path_buf), "test".to_string(), batches).await.unwrap();
         assert_eq!(table.count_rows().await.unwrap(), 10);
 
-        let new_batches: Box<dyn RecordBatchReader> =
-            Box::new(RecordBatchBuffer::new(vec![RecordBatch::try_new(
-                schema,
-                vec![Arc::new(Int32Array::from_iter_values(100..110))],
-            )
-            .unwrap()]));
+        let new_batches: Box<dyn RecordBatchReader> = Box::new(RecordBatchBuffer::new(vec![RecordBatch::try_new(
+            schema,
+            vec![Arc::new(Int32Array::from_iter_values(100..110))],
+        ).unwrap()]));
 
-        table
-            .add(new_batches, Some(WriteMode::Overwrite))
-            .await
-            .unwrap();
+        table.add(new_batches, Some(WriteMode::Overwrite)).await.unwrap();
         assert_eq!(table.count_rows().await.unwrap(), 10);
         assert_eq!(table.name, "test");
     }
@@ -294,16 +208,21 @@ mod tests {
     #[tokio::test]
     async fn test_search() {
         let tmp_dir = tempdir().unwrap();
-        let dataset_path = tmp_dir.path().join("test.lance");
-        let uri = tmp_dir.path().to_str().unwrap();
+        let path_buf = tmp_dir.into_path();
 
         let mut batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
-        Dataset::write(&mut batches, dataset_path.to_str().unwrap(), None)
+        Dataset::write(
+            &mut batches,
+            path_buf.join("test.lance").to_str().unwrap(),
+            None,
+        )
+        .await
+        .unwrap();
+
+        let table = Table::open(Arc::new(path_buf), "test".to_string())
             .await
             .unwrap();
 
-        let table = Table::open(uri, "test").await.unwrap();
-
         let vector = Float32Array::from_iter_values([0.1, 0.2]);
         let query = table.search(vector.clone());
         assert_eq!(vector, query.query_vector);
@@ -317,72 +236,4 @@ mod tests {
         )
         .unwrap()])
     }
-
-    #[tokio::test]
-    async fn test_create_index() {
-        use arrow_array::RecordBatch;
-        use arrow_schema::{DataType, Field, Schema as ArrowSchema};
-        use rand;
-        use std::iter::repeat_with;
-
-        use arrow_array::Float32Array;
-
-        let tmp_dir = tempdir().unwrap();
-        let uri = tmp_dir.path().to_str().unwrap();
-
-        let dimension = 16;
-        let schema = Arc::new(ArrowSchema::new(vec![Field::new(
-            "embeddings",
-            DataType::FixedSizeList(
-                Arc::new(Field::new("item", DataType::Float32, true)),
-                dimension,
-            ),
-            false,
-        )]));
-
-        let mut rng = rand::thread_rng();
-        let float_arr = Float32Array::from(
-            repeat_with(|| rng.gen::<f32>())
-                .take(512 * dimension as usize)
-                .collect::<Vec<f32>>(),
-        );
-
-        let vectors = Arc::new(create_fixed_size_list(float_arr, dimension).unwrap());
-        let batches = RecordBatchBuffer::new(vec![RecordBatch::try_new(
-            schema.clone(),
-            vec![vectors.clone()],
-        )
-        .unwrap()]);
-
-        let reader: Box<dyn RecordBatchReader + Send> = Box::new(batches);
-        let mut table = Table::create(uri, "test", reader).await.unwrap();
-
-        let mut i = IvfPQIndexBuilder::new();
-
-        let index_builder = i
-            .column("embeddings".to_string())
-            .index_name("my_index".to_string())
-            .ivf_params(IvfBuildParams::new(256))
-            .pq_params(PQBuildParams::default());
-
-        table.create_index(index_builder).await.unwrap();
-
-        assert_eq!(table.dataset.load_indices().await.unwrap().len(), 1);
-        assert_eq!(table.count_rows().await.unwrap(), 512);
-        assert_eq!(table.name, "test");
-    }
-
-    fn create_fixed_size_list<T: Array>(values: T, list_size: i32) -> Result<FixedSizeListArray> {
-        let list_type = DataType::FixedSizeList(
-            Arc::new(Field::new("item", values.data_type().clone(), true)),
-            list_size,
-        );
-        let data = ArrayDataBuilder::new(list_type)
-            .len(values.len() / list_size as usize)
-            .add_child_data(values.into_data())
-            .build()
-            .unwrap();
-
-        Ok(FixedSizeListArray::from(data))
-    }
 }