move tantivy-py installation to be separate from wheel (#97)

pypi does not allow packages to be uploaded that has a direct reference

for now we'll just ask the user to install tantivy separately

---------

Co-authored-by: Chang She <chang@lancedb.com>

.github/workflows/make_release_commit.yml

@@ -1,70 +0,0 @@
-name: Create release commit
-
-on:
-  workflow_dispatch:
-    inputs:
-      dry_run:
-        description: 'Just create the local commit/tags but do not push it'
-        required: true
-        default: "false"
-        type: choice
-        options:
-          - "true"
-          - "false"
-      part:
-        description: 'What kind of release is this?'
-        required: true
-        default: 'patch'
-        type: choice
-        options:
-          - patch
-          - minor
-          - major
-
-jobs:
-  bump-version:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out main
-      uses: actions/checkout@v3
-      with:
-        ref: main
-        persist-credentials: false
-        fetch-depth: 0
-        lfs: true
-    - name: Install cargo utils
-      run: cargo install cargo-bump cargo-get
-    - name: Bump vectordb
-      working-directory: rust/vectordb
-      run: |
-        cargo bump ${{ inputs.part }}
-        echo "CRATE_VERSION=$(cargo get version)" >> $GITHUB_ENV
-    - name: Bump rust/ffi/node
-      working-directory: rust/ffi/node
-      run: |
-        cargo bump ${{ inputs.part }}
-        echo "FFI_CRATE_VERSION=$(cargo get version)" >> $GITHUB_ENV 
-    - name: Bump node
-      working-directory: node
-      run: |
-        npm version ${{ inputs.part }}
-        echo "NPM_PACKAGE_VERSION=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV 
-    - name: Create tag
-      run: |
-        if [ "$CRATE_VERSION" != "$FFI_CRATE_VERSION" ]; then
-          echo "Version mismatch between rust/vectordb and rust/ffi/node"
-          exit 1
-        fi
-        if [ "$CRATE_VERSION" != "$NPM_PACKAGE_VERSION" ]; then
-          echo "Version mismatch between rust/vectordb and node"
-          exit 1
-        fi
-        export TAG="v$CRATE_VERSION'"
-        git tag $TAG
-    - name: Push new version and tag
-      if: ${{ inputs.dry_run }} == "false"
-      uses: ad-m/github-push-action@master
-      with:
-        github_token: ${{ secrets.RELEASE_TOKEN }}
-        branch: main
-        tags: true

.github/workflows/node.yml

@@ -67,10 +67,8 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run tsc
         npm run build
-        npm run pack-build
-        npm install --no-save ./dist/vectordb-*.tgz
+        npm run tsc
     - name: Test
       run: npm run test
   macos:
@@ -96,10 +94,8 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run tsc
         npm run build
-        npm run pack-build
-        npm install --no-save ./dist/vectordb-*.tgz
+        npm run tsc
     - name: Test
       run: |
         npm run test

.github/workflows/python.yml

@@ -30,7 +30,8 @@ jobs:
         python-version: 3.${{ matrix.python-minor-version }}
     - name: Install lancedb
       run: |
-        pip install -e ".[fts]"
+        pip install -e .
+        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
         pip install pytest
     - name: Run tests
       run: pytest -x -v --durations=30 tests
@@ -52,7 +53,8 @@ jobs:
         python-version: "3.11"
     - name: Install lancedb
       run: |
-        pip install -e ".[fts]"
+        pip install -e .
+        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
         pip install pytest
     - name: Run tests
       run: pytest -x -v --durations=30 tests

.github/workflows/release.yml

@@ -1,174 +0,0 @@
-name: Prepare Release
-
-# TODO: bump versions in CI
-# NOTE: Python is a separate release for now.
-
-on:
-  push:
-    tags:
-      - v*
-
-jobs:
-  draft-release:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: softprops/action-gh-release@v1
-        with:
-          draft: true
-          prerelease: true # hardcoded on for now
-          generate_release_notes: true
-  
-  rust:
-    runs-on: ubuntu-latest
-    needs: draft-release
-    defaults:
-      run:
-        shell: bash
-        working-directory: rust/vectordb
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Package Rust
-        run: cargo package --all-features
-      - uses: softprops/action-gh-release@v1
-        with:
-          draft: true
-          files: target/package/vectordb-*.crate
-          fail_on_unmatched_files: true
-
-  node:
-    runs-on: ubuntu-latest
-    needs: draft-release
-    defaults:
-      run:
-        shell: bash
-        working-directory: node
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 20
-          cache: 'npm'
-          cache-dependency-path: node/package-lock.json
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Build
-        run: |
-          npm ci
-          npm run tsc
-          npm pack
-      - uses: softprops/action-gh-release@v1
-        with:
-          draft: true
-          files: node/vectordb-*.tgz
-          fail_on_unmatched_files: true
-
-  node-macos:
-    runs-on: macos-12
-    needs: draft-release
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64-apple-darwin, aarch64-apple-darwin]
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-    - name: Install system dependencies
-      run: brew install protobuf
-    - name: Install npm dependencies
-      run: |
-        cd node
-        npm ci
-    - name: Install rustup target
-      if: ${{ matrix.target == 'aarch64-apple-darwin' }}
-      run: rustup target add aarch64-apple-darwin
-    - name: Build MacOS native node modules
-      run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
-    - uses: softprops/action-gh-release@v1
-      with:
-        draft: true
-        files: node/dist/vectordb-darwin*.tgz
-        fail_on_unmatched_files: true
-
-  node-linux:
-    name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }})
-    runs-on: ubuntu-latest
-    needs: draft-release
-    strategy:
-      fail-fast: false
-      matrix:
-        libc:
-          - gnu
-          # TODO: re-enable musl once we have refactored to pre-built containers
-          # Right now we have to build node from source which is too expensive.
-          # - musl
-        arch:
-          - x86_64
-          - aarch64
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-    - name: Change owner to root (for npm)
-      # The docker container is run as root, so we need the files to be owned by root
-      # Otherwise npm is a nightmare: https://github.com/npm/cli/issues/3773
-      run: sudo chown -R root:root .
-    - name: Set up QEMU
-      if: ${{ matrix.arch == 'aarch64' }}
-      uses: docker/setup-qemu-action@v2  
-      with:
-        platforms: arm64
-    - name: Build Linux GNU native node modules
-      if: ${{ matrix.libc == 'gnu' }}
-      run: |
-        docker run \
-          -v $(pwd):/io -w /io \
-          quay.io/pypa/manylinux2014_${{ matrix.arch }} \
-          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu
-    - name: Build musl Linux native node modules
-      if: ${{ matrix.libc == 'musl' }}
-      run: |
-        docker run --platform linux/arm64/v8 \
-          -v $(pwd):/io -w /io \
-          quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
-          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
-    - uses: softprops/action-gh-release@v1
-      with:
-        draft: true
-        files: node/dist/vectordb-linux*.tgz
-        fail_on_unmatched_files: true
-
-  release:
-    needs: [rust, node, node-macos, node-linux]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/download-artifact@v3
-      - name: Publish to PyPI
-        env:
-          TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
-        run: |
-          python -m twine upload --non-interactive \
-            --skip-existing \
-            --repository testpypi python/dist/*
-      - name: Publish to NPM
-        run: |
-          for filename in node/dist/*.tgz; do
-            npm publish --dry-run $filename
-          done
-      - name: Publish to crates.io
-        env:
-          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-        run: |
-          cargo publish --dry-run --no-verify rust/target/vectordb-*.crate
-      # - uses: softprops/action-gh-release@v1
-      #   with:
-      #     draft: false

.gitignore

@@ -4,8 +4,6 @@
 **/__pycache__
 .DS_Store
 
-.vscode
-
 rust/target
 rust/Cargo.lock
 

Cargo.lock

@@ -3356,7 +3356,7 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
 [[package]]
 name = "vectordb"
-version = "0.1.2"
+version = "0.0.1"
 dependencies = [
  "arrow-array",
  "arrow-data",
@@ -3369,7 +3369,7 @@ dependencies = [
 
 [[package]]
 name = "vectordb-node"
-version = "0.1.2"
+version = "0.1.0"
 dependencies = [
  "arrow-array",
  "arrow-ipc",

ci/build_linux_artifacts.sh

@@ -1,86 +0,0 @@
-#!/bin/bash
-# Builds the Linux artifacts (node binaries).
-# Usage: ./build_linux_artifacts.sh [target]
-# Targets supported: 
-# - x86_64-unknown-linux-gnu:centos
-# - aarch64-unknown-linux-gnu:centos
-# - aarch64-unknown-linux-musl
-# - x86_64-unknown-linux-musl
-
-# TODO: refactor this into a Docker container we can pull
-
-set -e
-
-setup_dependencies() {
-    echo "Installing system dependencies..."
-    if [[ $1 == *musl ]]; then
-        # musllinux
-        apk add openssl-dev
-    else
-        # manylinux2014
-        yum install -y openssl-devel unzip
-    fi
-
-    if [[ $1 == x86_64* ]]; then
-        ARCH=x86_64
-    else
-        # gnu target
-        ARCH=aarch_64
-    fi
-    
-    # Install new enough protobuf (yum-provided is old)
-    PB_REL=https://github.com/protocolbuffers/protobuf/releases
-    PB_VERSION=23.1
-    curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
-    unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local
-}
-
-install_node() {
-    echo "Installing node..."
-    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
-    source "$HOME"/.bashrc
-
-    if [[ $1 == *musl ]]; then
-        # This node version is 15, we need 16 or higher:
-        # apk add nodejs-current npm 
-        # So instead we install from source (nvm doesn't provide binaries for musl):
-        nvm install -s --no-progress 17
-    else
-        nvm install --no-progress 17 # latest that supports glibc 2.17
-    fi
-}
-
-install_rust() {
-    echo "Installing rust..."
-    curl https://sh.rustup.rs -sSf | bash -s -- -y
-    export PATH="$PATH:/root/.cargo/bin"
-}
-
-build_node_binary() {
-    echo "Building node library for $1..."
-    pushd node
-
-    npm ci
-    
-    if [[ $1 == *musl ]]; then
-        # This is needed for cargo to allow build cdylibs with musl
-        export RUSTFLAGS="-C target-feature=-crt-static"
-    fi
-    # We don't pass in target, since the native target here already matches
-    # and openblas-src doesn't do well with cross-compilation.
-    npm run build-release
-    npm run pack-build
-
-    popd
-}
-
-TARGET=${1:-x86_64-unknown-linux-gnu}
-# Others:
-# aarch64-unknown-linux-gnu
-# x86_64-unknown-linux-musl
-# aarch64-unknown-linux-musl
-
-setup_dependencies $TARGET
-install_node $TARGET
-install_rust
-build_node_binary $TARGET

ci/build_macos_artifacts.sh

@@ -1,38 +0,0 @@
-# Builds the macOS artifacts (node binaries).
-# Usage: ./build_macos_artifacts.sh [target]
-# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
-
-prebuild_rust() {
-    # Building here for the sake of easier debugging.
-    pushd rust/ffi/node
-
-    for target in $1
-    do
-        echo "Building rust library for $target"
-        export RUST_BACKTRACE=1
-        cargo build --release --target $target
-    done
-
-    popd
-}
-
-build_node_binaries() {
-    pushd node
-    
-    for target in $1
-    do
-        echo "Building node library for $target"
-        npm run build-release -- --target $target
-        npm run pack-build -- --target $target
-    done
-    popd
-}
-
-if [ -n "$1" ]; then
-    targets=$1
-else
-    targets="x86_64-apple-darwin aarch64-apple-darwin"
-fi
-
-prebuild_rust $targets
-build_node_binaries $targets

ci/release_process.md

@@ -1,90 +0,0 @@
-# How to release
-
-This is for the Rust crate and Node module. For now, the Python module is
-released separately.
-
-The release is started by bumping the versions and pushing a new tag. To do this
-automatically, use the `make_release_commit` GitHub action.
-
-When the tag is pushed, GitHub actions will start building the libraries and
-will upload them to a draft release.
-
-While those jobs are running, edit the release notes as needed. For example, 
-bring relevant new features and bugfixes to the top of the notes and the testing
-and CI changes to the bottom.
-
-Once the jobs have finished, the release will be marked as not draft and the
-artifacts will be released to crates.io, NPM, and PyPI.
-
-## Manual process
-
-You can also build the artifacts locally on a MacOS machine.
-
-### Build the MacOS release libraries
-
-One-time setup:
-
-```shell
-rustup target add x86_64-apple-darwin aarch64-apple-darwin
-```
-
-To build:
-
-```shell
-bash ci/build_macos_artifacts.sh
-```
-
-### Build the Linux release libraries
-
-To build a Linux library, we need to use docker with a different build script:
-
-```shell
-ARCH=aarch64
-docker run \
-    -v $(pwd):/io -w /io \
-    quay.io/pypa/manylinux2014_$ARCH \
-    bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-gnu
-```
-
-You can change `ARCH` to `x86_64`.
-
-Similar script for musl binaries (not yet working):
-
-```shell
-ARCH=aarch64
-docker run \
-    --user $(id -u) \
-    -v $(pwd):/io -w /io \
-    quay.io/pypa/musllinux_1_1_$ARCH \
-    bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-musl
-```
-
-<!--
-
-For debugging, use these snippets:
-
-```shell
-ARCH=aarch64
-docker run -it \
-    -v $(pwd):/io -w /io \
-    quay.io/pypa/manylinux2014_$ARCH \
-    bash
-```
-
-```shell
-ARCH=aarch64
-docker run -it \
-    -v $(pwd):/io -w /io \
-    quay.io/pypa/musllinux_1_1_$ARCH \
-    bash
-```
-
-Note: musllinux_1_1 is Alpine Linux 3.12
--->
-
-```
-docker run \
-    -v $(pwd):/io -w /io \
-    quay.io/pypa/musllinux_1_1_aarch64 \
-    bash alpine_repro.sh
-```

docs/src/fts.md

@@ -6,9 +6,10 @@ to make this available for JS as well.
 
 ## Installation
 
-To use full text search, you must install the fts optional dependencies:
+To use full text search, you must install optional dependency tantivy-py:
 
-`pip install lancedb[fts]`
+# tantivy 0.19.2
+pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
 
 
 ## Quickstart

node/.npmignore

@@ -1,2 +0,0 @@
-gen_test_data.py
-index.node

node/README.md

@@ -8,10 +8,6 @@ A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb)
 npm install vectordb
 ```
 
-This will download the appropriate native library for your platform. We currently
-support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
-yet support Windows or musl-based Linux (such as Alpine Linux).
-
 ## Usage
 
 ### Basic Example
@@ -28,19 +24,6 @@ The [examples](./examples) folder contains complete examples.
 
 ## Development
 
-Build and install the rust library with:
-
-```bash
-npm run build
-npm run pack-build
-npm install --no-save ./dist/vectordb-*.tgz
-```
-
-`npm run build` builds the Rust library, `npm run pack-build` packages the Rust
-binary into an npm module called `@vectordb/<platform>` (for example, 
-`@vectordb/darwin-arm64.node`), and then `npm run install ...` installs that
-module.
-
 The LanceDB javascript is built with npm:
 
 ```bash

node/native.js

@@ -12,20 +12,29 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-const { currentTarget } = require('@neon-rs/load');
-
 let nativeLib;
 
-try {
-    nativeLib = require(`@vectordb/${currentTarget()}`);
-} catch (e) {
-    throw new Error(`vectordb: failed to load native library.
-  You may need to run \`npm install @vectordb/${currentTarget()}\`.
-
-  If that does not work, please file a bug report at https://github.com/lancedb/lancedb/issues
-
-  Source error: ${e}`);
+function getPlatformLibrary() {
+    if (process.platform === "darwin" && process.arch == "arm64") {
+        return require('./aarch64-apple-darwin.node');
+    } else if (process.platform === "darwin" && process.arch == "x64") {
+        return require('./x86_64-apple-darwin.node');
+    } else if (process.platform === "linux" && process.arch == "x64") {
+        return require('./x86_64-unknown-linux-gnu.node');
+    } else {
+        throw new Error(`vectordb: unsupported platform ${process.platform}_${process.arch}. Please file a bug report at https://github.com/lancedb/lancedb/issues`)
+    }
 }
 
-// Dynamic require for runtime.
-module.exports = nativeLib;
+try {
+    nativeLib = require('./index.node')
+} catch (e) {
+    if (e.code === "MODULE_NOT_FOUND") {
+        nativeLib = getPlatformLibrary();
+    } else {
+        throw new Error('vectordb: failed to load native library. Please file a bug report at https://github.com/lancedb/lancedb/issues');
+    }
+}
+
+module.exports = nativeLib
+

node/package-lock.json

@@ -7,26 +7,12 @@
     "": {
       "name": "vectordb",
       "version": "0.1.1",
-      "cpu": [
-        "x64",
-        "arm64"
-      ],
       "license": "Apache-2.0",
-      "os": [
-        "darwin",
-        "linux"
-      ],
       "dependencies": {
         "@apache-arrow/ts": "^12.0.0",
-        "@neon-rs/load": "^0.0.74",
-        "@vectordb/darwin-arm64": "0.1.1",
-        "@vectordb/darwin-x64": "0.1.1",
-        "@vectordb/linux-x64-gnu": "0.1.1",
-        "@vectordb/linux-x64-musl": "0.1.1",
         "apache-arrow": "^12.0.0"
       },
       "devDependencies": {
-        "@neon-rs/cli": "^0.0.74",
         "@types/chai": "^4.3.4",
         "@types/mocha": "^10.0.1",
         "@types/node": "^18.16.2",
@@ -44,12 +30,6 @@
         "ts-node": "^10.9.1",
         "ts-node-dev": "^2.0.0",
         "typescript": "*"
-      },
-      "optionalDependencies": {
-        "@vectordb/darwin-arm64": "0.1.1",
-        "@vectordb/darwin-x64": "0.1.1",
-        "@vectordb/linux-x64-gnu": "0.1.1",
-        "@vectordb/linux-x64-musl": "0.1.1"
       }
     },
     "node_modules/@apache-arrow/ts": {
@@ -217,20 +197,6 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
-    "node_modules/@neon-rs/cli": {
-      "version": "0.0.74",
-      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
-      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
-      "dev": true,
-      "bin": {
-        "neon": "index.js"
-      }
-    },
-    "node_modules/@neon-rs/load": {
-      "version": "0.0.74",
-      "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
-      "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
-    },
     "node_modules/@nodelib/fs.scandir": {
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -4225,17 +4191,6 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
-    "@neon-rs/cli": {
-      "version": "0.0.74",
-      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
-      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
-      "dev": true
-    },
-    "@neon-rs/load": {
-      "version": "0.0.74",
-      "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
-      "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
-    },
     "@nodelib/fs.scandir": {
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",

node/package.json

@@ -1,18 +1,15 @@
 {
   "name": "vectordb",
-  "version": "0.1.2",
+  "version": "0.1.1",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
     "tsc": "tsc -b",
-    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json",
+    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json-render-diagnostics",
     "build-release": "npm run build -- --release",
-    "cross-release": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cross build --message-format=json --release -p vectordb-node",
     "test": "mocha -recursive dist/test",
-    "lint": "eslint src --ext .js,.ts",
-    "pack-build": "neon pack-build",
-    "check-npm": "printenv && which node && which npm && npm --version"
+    "lint": "eslint src --ext .js,.ts"
   },
   "repository": {
     "type": "git",
@@ -27,7 +24,6 @@
   "author": "Lance Devs",
   "license": "Apache-2.0",
   "devDependencies": {
-    "@neon-rs/cli": "^0.0.74",
     "@types/chai": "^4.3.4",
     "@types/mocha": "^10.0.1",
     "@types/node": "^18.16.2",
@@ -48,33 +44,6 @@
   },
   "dependencies": {
     "@apache-arrow/ts": "^12.0.0",
-    "@neon-rs/load": "^0.0.74",
     "apache-arrow": "^12.0.0"
-  },
-  "os": [
-    "darwin",
-    "linux"
-  ],
-  "cpu": [
-    "x64",
-    "arm64"
-  ],
-  "neon": {
-    "targets": {
-      "x86_64-apple-darwin": "@vectordb/darwin-x64",
-      "aarch64-apple-darwin": "@vectordb/darwin-arm64",
-      "x86_64-unknown-linux-gnu": "@vectordb/linux-x64-gnu",
-      "x86_64-unknown-linux-musl": "@vectordb/linux-x64-musl",
-      "aarch64-unknown-linux-gnu": "@vectordb/linux-arm64-gnu",
-      "aarch64-unknown-linux-musl": "@vectordb/linux-arm64-musl"
-    }
-  },
-  "optionalDependencies": {
-    "@vectordb/darwin-arm64": "0.1.2",
-    "@vectordb/darwin-x64": "0.1.2",
-    "@vectordb/linux-x64-gnu": "0.1.2",
-    "@vectordb/linux-x64-musl": "0.1.2",
-    "@vectordb/linux-arm64-gnu": "0.1.2",
-    "@vectordb/linux-arm64-musl": "0.1.2"
   }
 }

python/lancedb/fts.py

@@ -16,7 +16,13 @@ import os
 from typing import List, Tuple
 
 import pyarrow as pa
-import tantivy
+
+try:
+    import tantivy
+except ImportError:
+    raise ImportError(
+        "Please install tantivy-py `pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985` to use the full text search feature."
+    )
 
 from .table import LanceTable
 

python/lancedb/query.py

@@ -153,7 +153,7 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
             import tantivy
         except ImportError:
             raise ImportError(
-                "You need to install the `lancedb[fts]` extra to use this method."
+                "Please install tantivy-py `pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985` to use the full text search feature."
             )
 
         from .fts import search_index

python/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "lancedb"
-version = "0.1.2"
-dependencies = ["pylance>=0.4.6", "ratelimiter", "retry", "tqdm"]
+version = "0.1.3"
+dependencies = ["pylance>=0.4.15", "ratelimiter", "retry", "tqdm"]
 description = "lancedb"
 authors = [
     { name = "LanceDB Devs", email = "dev@lancedb.com" },
@@ -45,10 +45,6 @@ dev = [
 docs = [
     "mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings[python]"
 ]
-fts = [
-    # tantivy 0.19.2
-    "tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985"
-]
 
 [build-system]
 requires = [

python/tests/test_embeddings.py

@@ -14,7 +14,6 @@ import sys
 
 import numpy as np
 import pyarrow as pa
-
 from lancedb.embeddings import with_embeddings
 
 

python/tests/test_fts.py

@@ -13,13 +13,13 @@
 import os
 import random
 
+import lancedb.fts
 import numpy as np
 import pandas as pd
 import pytest
 import tantivy
 
 import lancedb as ldb
-import lancedb.fts
 
 
 @pytest.fixture

python/tests/test_query.py

@@ -17,7 +17,6 @@ import pandas as pd
 import pandas.testing as tm
 import pyarrow as pa
 import pytest
-
 from lancedb.query import LanceQueryBuilder
 
 

python/tests/test_table.py

@@ -16,7 +16,6 @@ from pathlib import Path
 import pandas as pd
 import pyarrow as pa
 import pytest
-
 from lancedb.table import LanceTable
 
 

rust/ffi/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb-node"
-version = "0.1.2"
+version = "0.1.0"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
 edition = "2018"

rust/vectordb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb"
-version = "0.1.2"
+version = "0.0.1"
 edition = "2021"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"