[python] Bump version: 0.3.0 → 0.3.1

#488

.bumpversion.cfg

@@ -0,0 +1,12 @@
+[bumpversion]
+current_version = 0.3.0
+commit = True
+message = Bump version: {current_version} → {new_version}
+tag = True
+tag_name = v{new_version}
+
+[bumpversion:file:node/package.json]
+
+[bumpversion:file:rust/ffi/node/Cargo.toml]
+
+[bumpversion:file:rust/vectordb/Cargo.toml]

.bumpversion.toml

@@ -1,80 +0,0 @@
-[tool.bumpversion]
-current_version = "0.23.1-beta.1"
-parse = """(?x)
-    (?P<major>0|[1-9]\\d*)\\.
-    (?P<minor>0|[1-9]\\d*)\\.
-    (?P<patch>0|[1-9]\\d*)
-    (?:-(?P<pre_l>[a-zA-Z-]+)\\.(?P<pre_n>0|[1-9]\\d*))?
-"""
-serialize = [
-    "{major}.{minor}.{patch}-{pre_l}.{pre_n}",
-    "{major}.{minor}.{patch}",
-]
-search = "{current_version}"
-replace = "{new_version}"
-regex = false
-ignore_missing_version = false
-ignore_missing_files = false
-tag = true
-sign_tags = false
-tag_name = "v{new_version}"
-tag_message = "Bump version: {current_version} → {new_version}"
-allow_dirty = true
-commit = true
-message = "Bump version: {current_version} → {new_version}"
-commit_args = ""
-
-# Java maven files
-pre_commit_hooks = [
-  """
-    NEW_VERSION="${BVHOOK_NEW_MAJOR}.${BVHOOK_NEW_MINOR}.${BVHOOK_NEW_PATCH}"
-    if [ ! -z "$BVHOOK_NEW_PRE_L" ] && [ ! -z "$BVHOOK_NEW_PRE_N" ]; then
-        NEW_VERSION="${NEW_VERSION}-${BVHOOK_NEW_PRE_L}.${BVHOOK_NEW_PRE_N}"
-    fi
-    echo "Constructed new version: $NEW_VERSION"
-    cd java && mvn versions:set -DnewVersion=$NEW_VERSION && mvn versions:commit
-
-    # Check for any modified but unstaged pom.xml files
-    MODIFIED_POMS=$(git ls-files -m | grep pom.xml)
-    if [ ! -z "$MODIFIED_POMS" ]; then
-        echo "The following pom.xml files were modified but not staged. Adding them now:"
-        echo "$MODIFIED_POMS" | while read -r file; do
-            git add "$file"
-            echo "Added: $file"
-        done
-    fi
-    """,
-]
-
-[tool.bumpversion.parts.pre_l]
-optional_value = "final"
-values = ["beta", "final"]
-
-[[tool.bumpversion.files]]
-filename = "nodejs/package.json"
-replace = "\"version\": \"{new_version}\","
-search = "\"version\": \"{current_version}\","
-
-# nodejs binary packages
-[[tool.bumpversion.files]]
-glob = "nodejs/npm/*/package.json"
-replace = "\"version\": \"{new_version}\","
-search = "\"version\": \"{current_version}\","
-
-# Cargo files
-# ------------
-[[tool.bumpversion.files]]
-filename = "rust/lancedb/Cargo.toml"
-replace = "\nversion = \"{new_version}\""
-search = "\nversion = \"{current_version}\""
-
-[[tool.bumpversion.files]]
-filename = "nodejs/Cargo.toml"
-replace = "\nversion = \"{new_version}\""
-search = "\nversion = \"{current_version}\""
-
-# Java documentation
-[[tool.bumpversion.files]]
-filename = "docs/src/java/java.md"
-replace = "<version>{new_version}</version>"
-search = "<version>{current_version}</version>"

.cargo/config.toml

@@ -1,51 +0,0 @@
-[profile.release]
-lto = "fat"
-codegen-units = 1
-
-[profile.release-with-debug]
-inherits = "release"
-debug = true
-# Prioritize compile time over runtime performance
-codegen-units = 16
-lto = "thin"
-
-[target.'cfg(all())']
-rustflags = [
-    "-Wclippy::all",
-    "-Wclippy::style",
-    "-Wclippy::fallible_impl_from",
-    "-Wclippy::manual_let_else",
-    "-Wclippy::redundant_pub_crate",
-    "-Wclippy::string_add_assign",
-    "-Wclippy::string_add",
-    "-Wclippy::string_lit_as_bytes",
-    "-Wclippy::implicit_clone",
-    "-Wclippy::use_self",
-    "-Dclippy::cargo",
-    "-Dclippy::dbg_macro",
-    # not too much we can do to avoid multiple crate versions
-    "-Aclippy::multiple-crate-versions",
-    "-Aclippy::wildcard_dependencies",
-]
-
-[target.x86_64-unknown-linux-gnu]
-rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"]
-
-[target.x86_64-unknown-linux-musl]
-rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=-crt-static,+avx2,+fma,+f16c"]
-
-[target.aarch64-unknown-linux-musl]
-linker = "aarch64-linux-musl-gcc"
-rustflags = ["-C", "target-feature=-crt-static"]
-
-[target.aarch64-apple-darwin]
-rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm,+dotprod"]
-
-# Not all Windows systems have the C runtime installed, so this avoids library
-# not found errors on systems that are missing it.
-[target.x86_64-pc-windows-msvc]
-rustflags = ["-Ctarget-feature=+crt-static"]
-
-# Experimental target for Arm64 Windows
-[target.aarch64-pc-windows-msvc]
-rustflags = ["-Ctarget-feature=+crt-static"]

.github/ISSUE_TEMPLATE/bug-node.yml

@@ -1,33 +0,0 @@
-name: Bug Report - Node / Typescript
-description: File a bug report
-title: "bug(node): "
-labels: [bug, typescript]
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Thanks for taking the time to fill out this bug report!
-  - type: input
-    id: version
-    attributes:
-      label: LanceDB version
-      description: What version of LanceDB are you using? `npm list | grep vectordb`.
-      placeholder: v0.3.2
-    validations:
-      required: false
-  - type: textarea
-    id: what-happened
-    attributes:
-      label: What happened?
-      description: Also tell us, what did you expect to happen?
-    validations:
-      required: true
-  - type: textarea
-    id: reproduction
-    attributes:
-      label: Are there known steps to reproduce?
-      description: |
-        Let us know how to reproduce the bug and we may be able to fix it more
-        quickly. This is not required, but it is helpful.
-    validations:
-      required: false

.github/ISSUE_TEMPLATE/bug-python.yml

@@ -1,33 +0,0 @@
-name: Bug Report - Python
-description: File a bug report
-title: "bug(python): "
-labels: [bug, python]
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Thanks for taking the time to fill out this bug report!
-  - type: input
-    id: version
-    attributes:
-      label: LanceDB version
-      description: What version of LanceDB are you using? `python -c "import lancedb; print(lancedb.__version__)"`.
-      placeholder: v0.3.2
-    validations:
-      required: false
-  - type: textarea
-    id: what-happened
-    attributes:
-      label: What happened?
-      description: Also tell us, what did you expect to happen?
-    validations:
-      required: true
-  - type: textarea
-    id: reproduction
-    attributes:
-      label: Are there known steps to reproduce?
-      description: |
-        Let us know how to reproduce the bug and we may be able to fix it more
-        quickly. This is not required, but it is helpful.
-    validations:
-      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +0,0 @@
-blank_issues_enabled: true
-contact_links:
-  - name: Discord Community Support
-    url: https://discord.com/invite/zMM32dvNtd
-    about: Please ask and answer questions here.

.github/ISSUE_TEMPLATE/documentation.yml

@@ -1,23 +0,0 @@
-name: 'Documentation improvement'
-description: Report an issue with the documentation.
-labels: [documentation]
-
-body:
-  - type: textarea
-    id: description
-    attributes:
-      label: Description
-      description: >
-        Describe the issue with the documentation and how it can be fixed or improved.
-    validations:
-      required: true
-
-  - type: input
-    id: link
-    attributes:
-      label: Link
-      description: >
-        Provide a link to the existing documentation, if applicable.
-      placeholder: ex. https://lancedb.com/docs/tables/...
-    validations:
-      required: false

.github/ISSUE_TEMPLATE/feature.yml

@@ -1,31 +0,0 @@
-name: Feature suggestion
-description: Suggestion a new feature for LanceDB
-title: "Feature: "
-labels: [enhancement]
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Share a new idea for a feature or improvement. Be sure to search existing
-        issues first to avoid duplicates.
-  - type: dropdown
-    id: sdk
-    attributes:
-      label: SDK
-      description: Which SDK are you using? This helps us prioritize.
-      options:
-        - Python
-        - Node
-        - Rust
-      default: 0
-    validations:
-      required: false
-  - type: textarea
-    id: description
-    attributes:
-      label: Description
-      description: |
-        Describe the feature and why it would be useful. If applicable, consider
-        providing a code example of what it might be like to use the feature.
-    validations:
-      required: true

.github/actions/create-failure-issue/action.yml

@@ -1,45 +0,0 @@
-name: Create Failure Issue
-description: Creates a GitHub issue if any jobs in the workflow failed
-
-inputs:
-  job-results:
-    description: 'JSON string of job results from needs context'
-    required: true
-  workflow-name:
-    description: 'Name of the workflow'
-    required: true
-
-runs:
-  using: composite
-  steps:
-    - name: Check for failures and create issue
-      shell: bash
-      env:
-        JOB_RESULTS: ${{ inputs.job-results }}
-        WORKFLOW_NAME: ${{ inputs.workflow-name }}
-        RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-        GH_TOKEN: ${{ github.token }}
-      run: |
-        # Check if any job failed
-        if echo "$JOB_RESULTS" | jq -e 'to_entries | any(.value.result == "failure")' > /dev/null; then
-          echo "Detected job failures, creating issue..."
-
-          # Extract failed job names
-          FAILED_JOBS=$(echo "$JOB_RESULTS" | jq -r 'to_entries | map(select(.value.result == "failure")) | map(.key) | join(", ")')
-
-          # Create issue with workflow name, failed jobs, and run URL
-          gh issue create \
-            --title "$WORKFLOW_NAME Failed ($FAILED_JOBS)" \
-            --body "The workflow **$WORKFLOW_NAME** failed during execution.
-
-        **Failed jobs:** $FAILED_JOBS
-
-        **Run URL:** $RUN_URL
-
-        Please investigate the failed jobs and address any issues." \
-            --label "ci"
-
-          echo "Issue created successfully"
-        else
-          echo "No job failures detected, skipping issue creation"
-        fi

.github/labeler.yml

@@ -1,33 +0,0 @@
-version: 1
-appendOnly: true
-# Labels are applied based on conventional commits standard
-# https://www.conventionalcommits.org/en/v1.0.0/
-# These labels are later used in release notes. See .github/release.yml
-labels:
-# If the PR title has an ! before the : it will be considered a breaking change
-# For example, `feat!: add new feature` will be considered a breaking change
-- label: breaking-change
-  title: "^[^:]+!:.*"
-- label: breaking-change
-  body: "BREAKING CHANGE"
-- label: enhancement
-  title: "^feat(\\(.+\\))?!?:.*"
-- label: bug
-  title: "^fix(\\(.+\\))?!?:.*"
-- label: documentation
-  title: "^docs(\\(.+\\))?!?:.*"
-- label: performance
-  title: "^perf(\\(.+\\))?!?:.*"
-- label: ci
-  title: "^ci(\\(.+\\))?!?:.*"
-- label: chore
-  title: "^(chore|test|build|style)(\\(.+\\))?!?:.*"
-- label: Python
-  files:
-    - "^python\\/.*"
-- label: Rust
-  files:
-    - "^rust\\/.*"
-- label: typescript
-  files:
-    - "^node\\/.*"

.github/release_notes.json

@@ -1,41 +0,0 @@
-{
-    "ignore_labels": ["chore"],
-    "pr_template": "- ${{TITLE}} by @${{AUTHOR}} in ${{URL}}",
-    "categories": [
-        {
-            "title": "## 🏆 Highlights",
-            "labels": ["highlight"]
-        },
-        {
-            "title": "## 🛠 Breaking Changes",
-            "labels": ["breaking-change"]
-        },
-        {
-            "title": "## ⚠️ Deprecations ",
-            "labels": ["deprecation"]
-        },
-        {
-            "title": "## 🎉 New Features",
-            "labels": ["enhancement"]
-        },
-        {
-            "title": "## 🐛 Bug Fixes",
-            "labels": ["bug"]
-        },
-        {
-            "title": "## 📚 Documentation",
-            "labels": ["documentation"]
-        },
-        {
-            "title": "## 🚀 Performance Improvements",
-            "labels": ["performance"]
-        },
-        {
-            "title": "## Other Changes"
-        },
-        {
-            "title": "## 🔧 Build and CI",
-            "labels": ["ci"]
-        }
-    ]
-}

.github/workflows/build_linux_wheel/action.yml

@@ -1,58 +0,0 @@
-# We create a composite action to be re-used both for testing and for releasing
-name: build-linux-wheel
-description: "Build a manylinux wheel for lance"
-inputs:
-  python-minor-version:
-    description: "8, 9, 10, 11, 12"
-    required: true
-  args:
-    description: "--release"
-    required: false
-    default: ""
-  arm-build:
-    description: "Build for arm64 instead of x86_64"
-    # Note: this does *not* mean the host is arm64, since we might be cross-compiling.
-    required: false
-    default: "false"
-  manylinux:
-    description: "The manylinux version to build for"
-    required: false
-    default: "2_17"
-runs:
-  using: "composite"
-  steps:
-    - name: CONFIRM ARM BUILD
-      shell: bash
-      run: |
-        echo "ARM BUILD: ${{ inputs.arm-build }}"
-    - name: Build x86_64 Manylinux wheel
-      if: ${{ inputs.arm-build == 'false' }}
-      uses: PyO3/maturin-action@v1
-      with:
-        command: build
-        working-directory: python
-        docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
-        target: x86_64-unknown-linux-gnu
-        manylinux: ${{ inputs.manylinux }}
-        args: ${{ inputs.args }}
-        before-script-linux: |
-          set -e
-          curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
-            && unzip /tmp/protoc.zip -d /usr/local \
-            && rm /tmp/protoc.zip
-    - name: Build Arm Manylinux Wheel
-      if: ${{ inputs.arm-build == 'true' }}
-      uses: PyO3/maturin-action@v1
-      with:
-        command: build
-        working-directory: python
-        docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
-        target: aarch64-unknown-linux-gnu
-        manylinux: ${{ inputs.manylinux }}
-        args: ${{ inputs.args }}
-        before-script-linux: |
-          set -e
-          yum install -y clang \
-            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip > /tmp/protoc.zip \
-            && unzip /tmp/protoc.zip -d /usr/local \
-            && rm /tmp/protoc.zip

.github/workflows/build_mac_wheel/action.yml

@@ -1,26 +0,0 @@
-# We create a composite action to be re-used both for testing and for releasing
-name: build_wheel
-description: "Build a lance wheel"
-inputs:
-  python-minor-version:
-    description: "8, 9, 10, 11"
-    required: true
-  args:
-    description: "--release"
-    required: false
-    default: ""
-runs:
-  using: "composite"
-  steps:
-    - name: Install macos dependency
-      shell: bash
-      run: |
-        brew install protobuf
-    - name: Build wheel
-      uses: PyO3/maturin-action@v1
-      with:
-        command: build
-        # TODO: pass through interpreter
-        args: ${{ inputs.args }}
-        docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
-        working-directory: python

.github/workflows/build_windows_wheel/action.yml

@@ -1,34 +0,0 @@
-# We create a composite action to be re-used both for testing and for releasing
-name: build_wheel
-description: "Build a lance wheel"
-inputs:
-  python-minor-version:
-    description: "8, 9, 10, 11"
-    required: true
-  args:
-    description: "--release"
-    required: false
-    default: ""
-runs:
-  using: "composite"
-  steps:
-    - name: Install Protoc v21.12
-      working-directory: C:\
-      run: |
-        New-Item -Path 'C:\protoc' -ItemType Directory
-        Set-Location C:\protoc
-        Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-        7z x protoc.zip
-        Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-      shell: powershell
-    - name: Build wheel
-      uses: PyO3/maturin-action@v1
-      with:
-        command: build
-        args: ${{ inputs.args }}
-        docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
-        working-directory: python
-    - uses: actions/upload-artifact@v4
-      with:
-        name: windows-wheels
-        path: python\target\wheels

.github/workflows/cargo-publish.yml

@@ -1,31 +1,22 @@
 name: Cargo Publish
 
 on:
-  push:
-    tags-ignore:
-      # We don't publish pre-releases for Rust. Crates.io is just a source
-      # distribution, so we don't need to publish pre-releases.
-      - "v*-beta*"
-      - "*-v*" # for example, python-vX.Y.Z
+  release:
+    types: [ published ]
 
 env:
   # This env var is used by Swatinem/rust-cache@v2 for the cache
   # key, so we set it to make sure it is always consistent.
   CARGO_TERM_COLOR: always
-  # Up-to-date compilers needed for fp16kernels.
-  CC: gcc-12
-  CXX: g++-12
 
 jobs:
   build:
     runs-on: ubuntu-22.04
-    permissions:
-      id-token: write
     timeout-minutes: 30
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: rust
@@ -33,22 +24,6 @@ jobs:
         run: |
           sudo apt update
           sudo apt install -y protobuf-compiler libssl-dev
-      - uses: rust-lang/crates-io-auth-action@v1
-        id: auth
       - name: Publish the package
         run: |
-          cargo publish -p lancedb --all-features --token ${{ steps.auth.outputs.token }}
-  report-failure:
-    name: Report Workflow Failure
-    runs-on: ubuntu-latest
-    needs: [build]
-    if: always() && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/create-failure-issue
-        with:
-          job-results: ${{ toJSON(needs) }}
-          workflow-name: ${{ github.workflow }}
+          cargo publish -p vectordb --all-features --token ${{ secrets.CARGO_REGISTRY_TOKEN }}

.github/workflows/codex-autofix-ci.yml

@@ -1,297 +0,0 @@
-name: Codex Autofix CI
-
-on:
-  check_suite:
-    types:
-      - completed
-
-permissions:
-  contents: write
-  pull-requests: write
-  actions: read
-
-concurrency:
-  group: codex-autofix-${{ github.event.check_suite.head_branch }}
-  cancel-in-progress: false
-
-jobs:
-  autofix:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Resolve PR and failing required checks
-        id: ctx
-        env:
-          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
-          REPO: ${{ github.repository }}
-          SHA: ${{ github.event.check_suite.head_sha }}
-          HEAD_BRANCH: ${{ github.event.check_suite.head_branch }}
-          MAX_ATTEMPTS: "3"
-        run: |
-          set -euo pipefail
-
-          echo "Repository: $REPO"
-          echo "head_branch: $HEAD_BRANCH"
-          echo "head_sha: $SHA"
-
-          if [[ "$HEAD_BRANCH" != codex/update-lance-* ]]; then
-            echo "Skip: branch '$HEAD_BRANCH' does not match codex/update-lance-*"
-            echo "needs_fix=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          prs_json="$(gh api -H "Accept: application/vnd.github+json" "repos/$REPO/commits/$SHA/pulls")"
-          pr_json="$(echo "$prs_json" | jq -c '[.[] | select(.state=="open")] | .[0]')"
-          if [[ -z "$pr_json" || "$pr_json" == "null" ]]; then
-            echo "Skip: no open PR found for sha $SHA"
-            echo "needs_fix=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          pr_number="$(echo "$pr_json" | jq -r '.number')"
-          head_ref="$(echo "$pr_json" | jq -r '.head.ref')"
-          head_repo="$(echo "$pr_json" | jq -r '.head.repo.full_name')"
-          pr_head_sha="$(echo "$pr_json" | jq -r '.head.sha')"
-
-          if [[ "$head_repo" != "$REPO" ]]; then
-            echo "Skip: cross-repo PR ($head_repo != $REPO)"
-            echo "needs_fix=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          if [[ "$pr_head_sha" != "$SHA" ]]; then
-            echo "Skip: stale check_suite event (pr head sha $pr_head_sha != event sha $SHA)"
-            echo "needs_fix=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          set +e
-          checks_json="$(gh pr checks "$pr_number" --required --repo "$REPO" --json name,state,bucket,link,workflow)"
-          checks_rc=$?
-          set -e
-
-          if [[ "$checks_rc" -eq 8 ]]; then
-            echo "Skip: required checks still pending"
-            echo "needs_fix=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          if [[ "$checks_rc" -ne 0 ]]; then
-            echo "Skip: failed to query required checks (exit=$checks_rc)"
-            echo "needs_fix=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          fail_count="$(echo "$checks_json" | jq '[.[] | select(.bucket=="fail")] | length')"
-          if [[ "$fail_count" -eq 0 ]]; then
-            echo "Skip: no failing required checks"
-            echo "needs_fix=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          comments_json="$(gh api "repos/$REPO/issues/$pr_number/comments" --paginate)"
-          stopped_count="$(echo "$comments_json" | jq '[.[].body | select(test("<!-- codex-autofix stopped -->"))] | length')"
-          if [[ "$stopped_count" -gt 0 ]]; then
-            echo "Skip: codex-autofix already stopped for this PR"
-            echo "needs_fix=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          prior_attempts="$(echo "$comments_json" | jq '[.[].body | select(test("<!-- codex-autofix attempt:"))] | length')"
-          attempt="$((prior_attempts + 1))"
-
-          if [[ "$attempt" -gt "$MAX_ATTEMPTS" ]]; then
-            run_url="${GITHUB_SERVER_URL}/${REPO}/actions/runs/${GITHUB_RUN_ID}"
-            comment_file="$(mktemp /tmp/codex-autofix-comment.XXXXXX.md)"
-            {
-              printf '%s\n' '<!-- codex-autofix stopped -->'
-              printf '<!-- codex-autofix attempt: %s -->\n' "$attempt"
-              printf 'Codex autofix stopped: reached max attempts (%s).\n\n' "$MAX_ATTEMPTS"
-              printf -- '- Run: %s\n' "$run_url"
-              printf -- '- head_sha: `%s`\n' "$SHA"
-              printf -- '- head_ref: `%s`\n' "$head_ref"
-            } >"$comment_file"
-            gh pr comment "$pr_number" --repo "$REPO" --body-file "$comment_file"
-            echo "needs_fix=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          evidence_file="$(mktemp /tmp/codex-autofix-evidence.XXXXXX.txt)"
-          run_url="${GITHUB_SERVER_URL}/${REPO}/actions/runs/${GITHUB_RUN_ID}"
-
-          {
-            echo "PR: #$pr_number"
-            echo "head_ref: $head_ref"
-            echo "head_sha: $SHA"
-            echo "Run: $run_url"
-            echo ""
-            echo "Failing required checks:"
-            echo "$checks_json" | jq -r '.[] | select(.bucket=="fail") | "- \(.name) (\(.workflow // "unknown")): \(.link // "n/a")"'
-            echo ""
-          } > "$evidence_file"
-
-          while IFS= read -r row; do
-            name="$(echo "$row" | jq -r '.name')"
-            link="$(echo "$row" | jq -r '.link // empty')"
-            workflow="$(echo "$row" | jq -r '.workflow // "unknown"')"
-
-            {
-              echo "================================================================================"
-              echo "CHECK: $name"
-              echo "WORKFLOW: $workflow"
-              echo "LINK: ${link:-n/a}"
-            } >> "$evidence_file"
-
-            run_id=""
-            if [[ -n "$link" ]]; then
-              run_id="$(echo "$link" | sed -n 's#.*actions/runs/\\([0-9][0-9]*\\).*#\\1#p' | head -n 1 || true)"
-            fi
-
-            if [[ -z "$run_id" ]]; then
-              echo "LOGS: unavailable (no run id found in link)" >> "$evidence_file"
-              echo "" >> "$evidence_file"
-              continue
-            fi
-
-            echo "LOGS: gh run view $run_id --log-failed (tail -c 20000)" >> "$evidence_file"
-            set +e
-            gh run view "$run_id" --repo "$REPO" --log-failed 2>/dev/null | tail -c 20000 >> "$evidence_file"
-            echo "" >> "$evidence_file"
-            set -e
-          done < <(echo "$checks_json" | jq -c '.[] | select(.bucket=="fail")')
-
-          comment_file="$(mktemp /tmp/codex-autofix-comment.XXXXXX.md)"
-          {
-            printf '<!-- codex-autofix attempt: %s -->\n' "$attempt"
-            printf 'Starting Codex autofix attempt %s.\n\n' "$attempt"
-            printf -- '- Run: %s\n' "$run_url"
-            printf -- '- head_sha: `%s`\n' "$SHA"
-            printf -- '- head_ref: `%s`\n' "$head_ref"
-            printf -- '- Failing required checks: %s\n' "$fail_count"
-          } >"$comment_file"
-          gh pr comment "$pr_number" --repo "$REPO" --body-file "$comment_file"
-
-          {
-            echo "needs_fix=true"
-            echo "pr_number=$pr_number"
-            echo "head_ref=$head_ref"
-            echo "attempt=$attempt"
-            echo "evidence_file=$evidence_file"
-          } >> "$GITHUB_OUTPUT"
-
-      - name: Checkout PR branch
-        if: steps.ctx.outputs.needs_fix == 'true'
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ steps.ctx.outputs.head_ref }}
-          token: ${{ secrets.ROBOT_TOKEN }}
-          persist-credentials: true
-
-      - name: Configure git
-        if: steps.ctx.outputs.needs_fix == 'true'
-        run: |
-          git config user.name "lancedb automation"
-          git config user.email "robot@lancedb.com"
-
-      - name: Set up Node.js
-        if: steps.ctx.outputs.needs_fix == 'true'
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-
-      - name: Install Codex CLI
-        if: steps.ctx.outputs.needs_fix == 'true'
-        run: npm install -g @openai/codex
-
-      - name: Install Rust toolchain
-        if: steps.ctx.outputs.needs_fix == 'true'
-        uses: dtolnay/rust-toolchain@stable
-        with:
-          toolchain: stable
-          components: clippy, rustfmt
-
-      - name: Install system dependencies
-        if: steps.ctx.outputs.needs_fix == 'true'
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y protobuf-compiler libssl-dev
-
-      - name: Run Codex to fix failing CI
-        if: steps.ctx.outputs.needs_fix == 'true'
-        env:
-          PR_NUMBER: ${{ steps.ctx.outputs.pr_number }}
-          HEAD_REF: ${{ steps.ctx.outputs.head_ref }}
-          ATTEMPT: ${{ steps.ctx.outputs.attempt }}
-          EVIDENCE_FILE: ${{ steps.ctx.outputs.evidence_file }}
-          GITHUB_TOKEN: ${{ secrets.ROBOT_TOKEN }}
-          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
-          OPENAI_API_KEY: ${{ secrets.CODEX_TOKEN }}
-        run: |
-          set -euo pipefail
-
-          prompt_file="/tmp/codex-prompt.txt"
-          {
-            printf 'You are running inside the lancedb repository on a GitHub Actions runner.\n'
-            printf 'Your task is to fix failing required CI checks for pull request #%s on branch %s.\n\n' "$PR_NUMBER" "$HEAD_REF"
-            printf 'Goal:\n'
-            printf -- '- Make the smallest change necessary so that all required checks pass.\n\n'
-            printf 'Evidence (generated from GitHub checks and logs):\n'
-            printf '---\n'
-            cat "${EVIDENCE_FILE}"
-            printf '\n---\n\n'
-            printf 'Follow these steps exactly:\n'
-            printf '1. Identify the root cause from the evidence and repository state.\n'
-            printf '2. Make changes to fix the failures.\n'
-            printf '3. Run the relevant local commands that correspond to the failing checks until they succeed.\n'
-            printf '   If unsure, start with:\n'
-            printf '   - cargo fmt --all -- --check\n'
-            printf '   - cargo clippy --profile ci --workspace --tests --all-features -- -D warnings\n'
-            printf '4. Ensure the repository is clean except for intentional changes (git status --short, git diff).\n'
-            printf '5. Create a commit with message "fix: codex autofix (attempt %s)".\n' "$ATTEMPT"
-            printf '6. Push to origin branch "%s" (use --force-with-lease only if required).\n' "$HEAD_REF"
-            printf '7. Print the commands you ran and their results, plus git status --short and git log -1 --oneline.\n\n'
-            printf 'Constraints:\n'
-            printf -- '- Do not create a new pull request.\n'
-            printf -- '- Do not merge.\n'
-            printf -- '- Avoid modifying GitHub workflow files unless strictly required to fix CI for this PR.\n'
-          } >"$prompt_file"
-
-          printenv OPENAI_API_KEY | codex login --with-api-key
-          codex --config shell_environment_policy.ignore_default_excludes=true exec --dangerously-bypass-approvals-and-sandbox "$(cat "$prompt_file")"
-
-      - name: Ensure branch is pushed
-        if: steps.ctx.outputs.needs_fix == 'true'
-        env:
-          HEAD_REF: ${{ steps.ctx.outputs.head_ref }}
-        run: |
-          set -euo pipefail
-          if git diff --quiet && git diff --cached --quiet; then
-            echo "Working tree clean."
-          else
-            git add -A
-            git commit -m "fix: codex autofix (post-run)" || true
-          fi
-          git push origin "HEAD:${HEAD_REF}" --force-with-lease
-
-      - name: Comment result
-        if: steps.ctx.outputs.needs_fix == 'true'
-        env:
-          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
-          REPO: ${{ github.repository }}
-          PR_NUMBER: ${{ steps.ctx.outputs.pr_number }}
-          ATTEMPT: ${{ steps.ctx.outputs.attempt }}
-        run: |
-          set -euo pipefail
-          run_url="${GITHUB_SERVER_URL}/${REPO}/actions/runs/${GITHUB_RUN_ID}"
-          sha="$(git rev-parse HEAD)"
-          summary="$(git log -1 --oneline || true)"
-          status="$(git status --short || true)"
-          comment_file="$(mktemp /tmp/codex-autofix-comment.XXXXXX.md)"
-          {
-            printf 'Codex autofix attempt %s finished.\n\n' "$ATTEMPT"
-            printf -- '- Run: %s\n' "$run_url"
-            printf -- '- head_sha: `%s`\n' "$sha"
-            printf -- '- Last commit: %s\n\n' "$summary"
-            printf '```\n%s\n```\n' "$status"
-          } >"$comment_file"
-          gh pr comment "$PR_NUMBER" --repo "$REPO" --body-file "$comment_file"

.github/workflows/codex-update-lance-dependency.yml

@@ -1,127 +0,0 @@
-name: Codex Update Lance Dependency
-
-on:
-  workflow_call:
-    inputs:
-      tag:
-        description: "Tag name from Lance"
-        required: true
-        type: string
-  workflow_dispatch:
-    inputs:
-      tag:
-        description: "Tag name from Lance"
-        required: true
-        type: string
-
-permissions:
-  contents: write
-  pull-requests: write
-  actions: read
-
-jobs:
-  update:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Show inputs
-        run: |
-          echo "tag = ${{ inputs.tag }}"
-
-      - name: Checkout Repo LanceDB
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: true
-
-      - name: Set up Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-
-      - name: Install Codex CLI
-        run: npm install -g @openai/codex
-
-      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
-        with:
-          toolchain: stable
-          components: clippy, rustfmt
-
-      - name: Install system dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y protobuf-compiler libssl-dev
-
-      - name: Install cargo-info
-        run: cargo install cargo-info
-
-      - name: Install Python dependencies
-        run: python3 -m pip install --upgrade pip packaging
-
-      - name: Configure git user
-        run: |
-          git config user.name "lancedb automation"
-          git config user.email "robot@lancedb.com"
-
-      - name: Run Codex to update Lance dependency
-        env:
-          TAG: ${{ inputs.tag }}
-          GITHUB_TOKEN: ${{ secrets.ROBOT_TOKEN }}
-          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
-          OPENAI_API_KEY: ${{ secrets.CODEX_TOKEN }}
-        run: |
-          set -euo pipefail
-          VERSION="${TAG#refs/tags/}"
-          VERSION="${VERSION#v}"
-          BRANCH_NAME="codex/update-lance-${VERSION//[^a-zA-Z0-9]/-}"
-
-          cat <<EOF >/tmp/codex-prompt.txt
-          You are running inside the lancedb repository on a GitHub Actions runner. Update the Lance dependency to version ${VERSION} and prepare a pull request for maintainers to review.
-
-          Follow these steps exactly:
-          1. Use script "ci/set_lance_version.py" to update Lance dependencies. The script already refreshes Cargo metadata, so allow it to finish even if it takes time.
-          2. Run "cargo clippy --workspace --tests --all-features -- -D warnings". If diagnostics appear, fix them yourself and rerun clippy until it exits cleanly. Do not skip any warnings.
-          3. After clippy succeeds, run "cargo fmt --all" to format the workspace.
-          4. Ensure the repository is clean except for intentional changes. Inspect "git status --short" and "git diff" to confirm the dependency update and any required fixes.
-          5. Create and switch to a new branch named "${BRANCH_NAME}" (replace any duplicated hyphens if necessary).
-          6. Stage all relevant files with "git add -A". Commit using the message "chore: update lance dependency to v${VERSION}".
-          7. Push the branch to origin. If the branch already exists, force-push your changes.
-          8. env "GH_TOKEN" is available, use "gh" tools for github related operations like creating pull request.
-          9. Create a pull request targeting "main" with title "chore: update lance dependency to v${VERSION}". In the body, summarize the dependency bump, clippy/fmt verification, and link the triggering tag (${TAG}).
-          10. After creating the PR, display the PR URL, "git status --short", and a concise summary of the commands run and their results.
-
-          Constraints:
-          - Use bash commands; avoid modifying GitHub workflow files other than through the scripted task above.
-          - Do not merge the PR.
-          - If any command fails, diagnose and fix the issue instead of aborting.
-          EOF
-
-          printenv OPENAI_API_KEY | codex login --with-api-key
-          codex --config shell_environment_policy.ignore_default_excludes=true exec --dangerously-bypass-approvals-and-sandbox "$(cat /tmp/codex-prompt.txt)"
-
-      - name: Trigger sophon dependency update
-        env:
-          TAG: ${{ inputs.tag }}
-          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
-        run: |
-          set -euo pipefail
-          VERSION="${TAG#refs/tags/}"
-          VERSION="${VERSION#v}"
-          LANCEDB_BRANCH="codex/update-lance-${VERSION//[^a-zA-Z0-9]/-}"
-
-          echo "Triggering sophon workflow with:"
-          echo "  lance_ref: ${TAG#refs/tags/}"
-          echo "  lancedb_ref: ${LANCEDB_BRANCH}"
-
-          gh workflow run codex-bump-lancedb-lance.yml \
-            --repo lancedb/sophon \
-            -f lance_ref="${TAG#refs/tags/}" \
-            -f lancedb_ref="${LANCEDB_BRANCH}"
-
-      - name: Show latest sophon workflow run
-        env:
-          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
-        run: |
-          set -euo pipefail
-          echo "Latest sophon workflow run:"
-          gh run list --repo lancedb/sophon --workflow codex-bump-lancedb-lance.yml --limit 1 --json databaseId,url,displayTitle

.github/workflows/dev.yml

@@ -1,81 +0,0 @@
-name: PR Checks
-
-on:
-  pull_request_target:
-    types: [opened, edited, synchronize, reopened]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  labeler:
-    permissions:
-      pull-requests: write
-    name: Label PR
-    runs-on: ubuntu-latest
-    steps:
-      - uses: srvaroa/labeler@master
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  commitlint:
-    permissions:
-      pull-requests: write
-    name: Verify PR title / description conforms to semantic-release
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/setup-node@v3
-        with:
-          node-version: "18"
-      # These rules are disabled because Github will always ensure there
-      # is a blank line between the title and the body and Github will
-      # word wrap the description field to ensure a reasonable max line
-      # length.
-      - run: npm install @commitlint/config-conventional
-      - run: >
-          echo 'module.exports = {
-            "rules": {
-              "body-max-line-length": [0, "always", Infinity],
-              "footer-max-line-length": [0, "always", Infinity],
-              "body-leading-blank": [0, "always"]
-            }
-          }' > .commitlintrc.js
-      - run: npx commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
-        env:
-          COMMIT_MSG: >
-            ${{ github.event.pull_request.title }}
-
-            ${{ github.event.pull_request.body }}
-      - if: failure()
-        uses: actions/github-script@v6
-        with:
-          script: |
-            const message = `**ACTION NEEDED**
-              
-              Lance follows the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) for release automation.
-
-              The PR title and description are used as the merge commit message.\
-              Please update your PR title and description to match the specification.
-
-              For details on the error please inspect the "PR Title Check" action.
-              `
-            // Get list of current comments
-            const comments = await github.paginate(github.rest.issues.listComments, {
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: context.issue.number
-            });
-            // Check if this job already commented
-            for (const comment of comments) {
-              if (comment.body === message) {
-                return // Already commented
-              }
-            }
-            // Post the comment about Conventional Commits
-            github.rest.issues.createComment({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: context.issue.number,
-              body: message
-            })
-            core.setFailed(message)

.github/workflows/docs.yml

@@ -18,61 +18,58 @@ concurrency:
   group: "pages"
   cancel-in-progress: true
 
-env:
-  # This reduces the disk space needed for the build
-  RUSTFLAGS: "-C debuginfo=0"
-  # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html
-  # CI builds are faster with incremental disabled.
-  CARGO_INCREMENTAL: "0"
-  PIP_EXTRA_INDEX_URL: "https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/"
-
 jobs:
   # Single deploy job since we're just deploying
   build:
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install dependencies needed for ubuntu
-        run: |
-          sudo apt install -y protobuf-compiler libssl-dev
-          rustup update && rustup default
+        uses: actions/checkout@v3
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v4
         with:
           python-version: "3.10"
           cache: "pip"
           cache-dependency-path: "docs/requirements.txt"
-      - uses: Swatinem/rust-cache@v2
       - name: Build Python
         working-directory: python
         run: |
-          python -m pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .
-          python -m pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -r ../docs/requirements.txt
+          python -m pip install -e .
+          python -m pip install -r ../docs/requirements.txt
       - name: Set up node
         uses: actions/setup-node@v3
         with:
-          node-version: 20
+          node-version: ${{ matrix.node-version }}
           cache: 'npm'
-          cache-dependency-path: docs/package-lock.json
+          cache-dependency-path: node/package-lock.json
+      - uses: Swatinem/rust-cache@v2
       - name: Install node dependencies
-        working-directory: nodejs
+        working-directory: node
         run: |
           sudo apt update
           sudo apt install -y protobuf-compiler libssl-dev
-      - name: Build docs
-        working-directory: docs
+      - name: Build node
+        working-directory: node
         run: |
-          PYTHONPATH=. mkdocs build
+          npm ci
+          npm run build
+          npm run tsc
+      - name: Create markdown files
+        working-directory: node
+        run: |
+          npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
+      - name: Build docs
+        run: |
+          PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
       - name: Setup Pages
         uses: actions/configure-pages@v2
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
+        uses: actions/upload-pages-artifact@v1
         with:
           path: "docs/site"
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v4
+        uses: actions/deploy-pages@v1

.github/workflows/docs_test.yml

@@ -0,0 +1,93 @@
+name: Documentation Code Testing
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - docs/**
+      - .github/workflows/docs_test.yml
+  pull_request:
+    paths:
+      - docs/**
+      - .github/workflows/docs_test.yml
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+env:
+  # Disable full debug symbol generation to speed up CI build and keep memory down
+  # "1" means line tables only, which is useful for panic tracebacks.
+  RUSTFLAGS: "-C debuginfo=1"
+  RUST_BACKTRACE: "1"
+
+jobs:
+  test-python:
+    name: Test doc python code
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-minor-version: [ "11" ]
+        os: ["ubuntu-22.04"]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.${{ matrix.python-minor-version }}
+        cache: "pip"
+        cache-dependency-path: "docs/test/requirements.txt"
+    - name: Build Python
+      working-directory: docs/test
+      run:
+        python -m pip install -r requirements.txt
+    - name: Create test files
+      run: |
+        cd docs/test
+        python md_testing.py
+    - name: Test
+      run: |
+        cd docs/test/python
+        for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
+  test-node:
+    name: Test doc nodejs code
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        node-version: [ "18" ]
+        os: ["ubuntu-22.04"]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - name: Set up Node
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+    - name: Install dependecies needed for ubuntu
+      if: ${{ matrix.os == 'ubuntu-22.04' }}
+      run: |
+        sudo apt install -y protobuf-compiler libssl-dev
+    - name: Install node dependencies
+      run: |
+        cd docs/test
+        npm install 
+    - name: Rust cache
+      uses: swatinem/rust-cache@v2
+    - name: Install LanceDB
+      run: |
+        cd docs/test/node_modules/vectordb 
+        npm ci
+        npm run build-release
+        npm run tsc
+    - name: Create test files
+      run: |
+        cd docs/test
+        node md_testing.js
+    - name: Test
+      run: |
+        cd docs/test/node
+        for d in *; do cd "$d"; echo "$d".js; node "$d".js; cd ..; done

.github/workflows/java-publish.yml

@@ -1,76 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: Build and publish Java packages
-on:
-  push:
-    tags:
-      - "v*"
-  pull_request:
-    paths:
-      - .github/workflows/java-publish.yml
-
-jobs:
-  publish:
-    name: Build and Publish
-    runs-on: ubuntu-24.04
-    timeout-minutes: 30
-    defaults:
-      run:
-        working-directory: ./java
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Set up Java 8
-        uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: 8
-          cache: "maven"
-          server-id: ossrh
-          server-username: SONATYPE_USER
-          server-password: SONATYPE_TOKEN
-          gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
-          gpg-passphrase: ${{ secrets.GPG_PASSPHRASE }}
-      - name: Set git config
-        run: |
-          git config --global user.email "dev+gha@lancedb.com"
-          git config --global user.name "LanceDB Github Runner"
-      - name: Dry run
-        if: github.event_name == 'pull_request'
-        run: |
-          ./mvnw --batch-mode -DskipTests package -pl lancedb-core -am
-      - name: Publish
-        if: startsWith(github.ref, 'refs/tags/v')
-        run: |
-          echo "use-agent" >> ~/.gnupg/gpg.conf
-          echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
-          export GPG_TTY=$(tty)
-          ./mvnw --batch-mode -DskipTests -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -pl lancedb-core -am -P deploy-to-ossrh
-        env:
-          SONATYPE_USER: ${{ secrets.SONATYPE_USER }}
-          SONATYPE_TOKEN: ${{ secrets.SONATYPE_TOKEN }}
-
-  report-failure:
-    name: Report Workflow Failure
-    runs-on: ubuntu-latest
-    needs: [publish]
-    if: always() && failure() && startsWith(github.ref, 'refs/tags/v')
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/create-failure-issue
-        with:
-          job-results: ${{ toJSON(needs) }}
-          workflow-name: ${{ github.workflow }}

.github/workflows/java.yml

@@ -1,46 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: Build Java LanceDB Core
-
-on:
-  push:
-    branches:
-      - main
-    paths:
-      - java/**
-      - .github/workflows/java.yml
-  pull_request:
-    paths:
-      - java/**
-      - .github/workflows/java.yml
-
-jobs:
-  build-java:
-    runs-on: ubuntu-24.04
-    name: Build
-    defaults:
-      run:
-        working-directory: ./java
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Set up Java 17
-        uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: 17
-          cache: "maven"
-      - name: Java Style Check
-        run: ./mvnw checkstyle:check
-      - name: Build and install
-        run: ./mvnw clean install

.github/workflows/lance-release-timer.yml

@@ -1,62 +0,0 @@
-name: Lance Release Timer
-
-on:
-  schedule:
-    - cron: "*/10 * * * *"
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  actions: write
-
-concurrency:
-  group: lance-release-timer
-  cancel-in-progress: false
-
-jobs:
-  trigger-update:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Check for new Lance tag
-        id: check
-        env:
-          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
-        run: |
-          python3 ci/check_lance_release.py --github-output "$GITHUB_OUTPUT"
-
-      - name: Look for existing PR
-        if: steps.check.outputs.needs_update == 'true'
-        id: pr
-        env:
-          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
-        run: |
-          set -euo pipefail
-          TITLE="chore: update lance dependency to v${{ steps.check.outputs.latest_version }}"
-          COUNT=$(gh pr list --search "\"$TITLE\" in:title" --state open --limit 1 --json number --jq 'length')
-          if [ "$COUNT" -gt 0 ]; then
-            echo "Open PR already exists for $TITLE"
-            echo "pr_exists=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "No existing PR for $TITLE"
-            echo "pr_exists=false" >> "$GITHUB_OUTPUT"
-          fi
-
-      - name: Trigger codex update workflow
-        if: steps.check.outputs.needs_update == 'true' && steps.pr.outputs.pr_exists != 'true'
-        env:
-          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
-        run: |
-          set -euo pipefail
-          TAG=${{ steps.check.outputs.latest_tag }}
-          gh workflow run codex-update-lance-dependency.yml -f tag=refs/tags/$TAG
-
-      - name: Show latest codex workflow run
-        if: steps.check.outputs.needs_update == 'true' && steps.pr.outputs.pr_exists != 'true'
-        env:
-          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
-        run: |
-          set -euo pipefail
-          gh run list --workflow codex-update-lance-dependency.yml --limit 1 --json databaseId,url,displayTitle

.github/workflows/license-header-check.yml

@@ -1,31 +0,0 @@
-name: Check license headers
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths:
-      - rust/**
-      - python/**
-      - nodejs/**
-      - java/**
-      - .github/workflows/license-header-check.yml
-jobs:
-  check-licenses:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v4
-      - name: Install license-header-checker
-        working-directory: /tmp
-        run: |
-            curl -s https://raw.githubusercontent.com/lluissm/license-header-checker/master/install.sh | bash
-            mv /tmp/bin/license-header-checker /usr/local/bin/
-      - name: Check license headers (rust)
-        run: license-header-checker -a -v ./rust/license_header.txt ./ rs && [[ -z `git status -s` ]]
-      - name: Check license headers (python)
-        run: license-header-checker -a -v ./python/license_header.txt python py && [[ -z `git status -s` ]]
-      - name: Check license headers (typescript)
-        run: license-header-checker -a -v ./nodejs/license_header.txt nodejs ts && [[ -z `git status -s` ]]
-      - name: Check license headers (java)
-        run: license-header-checker -a -v ./nodejs/license_header.txt java java && [[ -z `git status -s` ]]

.github/workflows/make-release-commit.yml

@@ -1,95 +1,59 @@
 name: Create release commit
 
-# This workflow increments versions, tags the version, and pushes it.
-# When a tag is pushed, another workflow is triggered that creates a GH release
-# and uploads the binaries. This workflow is only for creating the tag.
-
-# This script will enforce that a minor version is incremented if there are any
-# breaking changes since the last minor increment. However, it isn't able to
-# differentiate between breaking changes in Node versus Python. If you wish to
-# bypass this check, you can manually increment the version and push the tag.
 on:
   workflow_dispatch:
     inputs:
       dry_run:
         description: 'Dry run (create the local commit/tags but do not push it)'
         required: true
-        default: false
-        type: boolean
-      type:
-        description: 'What kind of release is this?'
-        required: true
-        default: 'preview'
+        default: "false"
         type: choice
         options:
-          - preview
-          - stable
-      python:
-        description: 'Make a Python release'
+          - "true"
+          - "false"
+      part:
+        description: 'What kind of release is this?'
         required: true
-        default: true
-        type: boolean
-      other:
-        description: 'Make a Node/Rust/Java release'
-        required: true
-        default: true
-        type: boolean
-      bump-minor:
-        description: 'Bump minor version'
-        required: true
-        default: false
-        type: boolean
+        default: 'patch'
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
 
 jobs:
-  make-release:
-    # Creates tag and GH release. The GH release will trigger the build and release jobs.
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: write
+  bump-version:
+    runs-on: ubuntu-latest
     steps:
-      - name: Output Inputs
-        run: echo "${{ toJSON(github.event.inputs) }}"
-      - uses: actions/checkout@v4
+      - name: Check out main
+        uses: actions/checkout@v3
         with:
+          ref: main
+          persist-credentials: false
           fetch-depth: 0
           lfs: true
-          # It's important we use our token here, as the default token will NOT
-          # trigger any workflows watching for new tags. See:
-          # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow
-          token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-      - name: Validate Lance dependency is at stable version
-        if: ${{ inputs.type == 'stable' }}
-        run: python ci/validate_stable_lance.py
       - name: Set git configs for bumpversion
         shell: bash
         run: |
           git config user.name 'Lance Release'
           git config user.email 'lance-dev@lancedb.com'
-      - name: Bump Python version
-        if: ${{ inputs.python }}
-        working-directory: python
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Bump version, create tag and commit
         run: |
-          # Need to get the commit before bumping the version, so we can
-          # determine if there are breaking changes in the next step as well.
-          echo "COMMIT_BEFORE_BUMP=$(git rev-parse HEAD)" >> $GITHUB_ENV
-
-          pip install bump-my-version PyGithub packaging
-          bash ../ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} python-v
-      - name: Bump Node/Rust version
-        if: ${{ inputs.other }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          pip install bump-my-version PyGithub packaging
-          bash ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} v $COMMIT_BEFORE_BUMP
-          bash ci/update_lockfiles.sh --amend
-      - name: Push new version tag
-        if: ${{ !inputs.dry_run }}
+          pip install bump2version
+          bumpversion --verbose ${{ inputs.part }}
+      - name: Push new version and tag
+        if: ${{ inputs.dry_run }} == "false"
         uses: ad-m/github-push-action@master
         with:
-          # Need to use PAT here too to trigger next workflow. See comment above.
           github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-          branch: ${{ github.ref }}
+          branch: main
           tags: true
+      - uses: ./.github/workflows/update_package_lock
+        if: ${{ inputs.dry_run }} == "false"
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+

.github/workflows/node.yml

@@ -0,0 +1,163 @@
+name: Node
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - node/**
+      - rust/ffi/node/**
+      - .github/workflows/node.yml
+      - docker-compose.yml
+
+env:
+  # Disable full debug symbol generation to speed up CI build and keep memory down
+  # "1" means line tables only, which is useful for panic tracebacks.
+  RUSTFLAGS: "-C debuginfo=1"
+  RUST_BACKTRACE: "1"
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 18
+        cache: 'npm'
+        cache-dependency-path: node/package-lock.json
+    - name: Lint
+      run: |
+        npm ci
+        npm run lint
+  linux:
+    name: Linux (Node ${{ matrix.node-version }})
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        node-version: [ "16", "18" ]
+    runs-on: "ubuntu-22.04"
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+        cache: 'npm'
+        cache-dependency-path: node/package-lock.json
+    - uses: Swatinem/rust-cache@v2
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y protobuf-compiler libssl-dev
+    - name: Build
+      run: |
+        npm ci
+        npm run tsc
+        npm run build
+        npm run pack-build
+        npm install --no-save ./dist/lancedb-vectordb-*.tgz
+        # Remove index.node to test with dependency installed
+        rm index.node
+    - name: Test
+      run: npm run test
+  macos:
+    timeout-minutes: 30
+    runs-on: "macos-13"
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 18
+        cache: 'npm'
+        cache-dependency-path: node/package-lock.json
+    - uses: Swatinem/rust-cache@v2
+    - name: Install dependencies
+      run: brew install protobuf
+    - name: Build
+      run: |
+        npm ci
+        npm run tsc
+        npm run build
+        npm run pack-build
+        npm install --no-save ./dist/lancedb-vectordb-*.tgz
+        # Remove index.node to test with dependency installed
+        rm index.node
+    - name: Test
+      run: |
+        npm run test
+  aws-integtest:
+    timeout-minutes: 45
+    runs-on: "ubuntu-22.04"
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
+    env:
+      AWS_ACCESS_KEY_ID: ACCESSKEY
+      AWS_SECRET_ACCESS_KEY: SECRETKEY
+      AWS_DEFAULT_REGION: us-west-2
+      # this one is for s3
+      AWS_ENDPOINT: http://localhost:4566
+      # this one is for dynamodb
+      DYNAMODB_ENDPOINT: http://localhost:4566
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 18
+        cache: 'npm'
+        cache-dependency-path: node/package-lock.json
+    - name: start local stack
+      run: docker compose -f ../docker-compose.yml up -d --wait
+    - name: create s3
+      run: aws s3 mb s3://lancedb-integtest --endpoint $AWS_ENDPOINT
+    - name: create ddb
+      run: |
+        aws dynamodb create-table \
+          --table-name lancedb-integtest \
+          --attribute-definitions '[{"AttributeName": "base_uri", "AttributeType": "S"}, {"AttributeName": "version", "AttributeType": "N"}]' \
+          --key-schema '[{"AttributeName": "base_uri", "KeyType": "HASH"}, {"AttributeName": "version", "KeyType": "RANGE"}]' \
+          --provisioned-throughput '{"ReadCapacityUnits": 10, "WriteCapacityUnits": 10}' \
+          --endpoint-url $DYNAMODB_ENDPOINT
+    - uses: Swatinem/rust-cache@v2
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y protobuf-compiler libssl-dev
+    - name: Build
+      run: |
+        npm ci
+        npm run tsc
+        npm run build
+        npm run pack-build
+        npm install --no-save ./dist/lancedb-vectordb-*.tgz
+        # Remove index.node to test with dependency installed
+        rm index.node
+    - name: Test
+      run: npm run integration-test

.github/workflows/nodejs.yml

@@ -1,154 +0,0 @@
-name: NodeJS (NAPI)
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths:
-      - Cargo.toml
-      - nodejs/**
-      - .github/workflows/nodejs.yml
-      - docker-compose.yml
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-env:
-  RUST_BACKTRACE: "1"
-
-jobs:
-  lint:
-    name: Lint
-    runs-on: ubuntu-22.04
-    defaults:
-      run:
-        shell: bash
-        working-directory: nodejs
-    env:
-      # Need up-to-date compilers for kernels
-      CC: gcc-12
-      CXX: g++-12
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 20
-        cache: 'npm'
-        cache-dependency-path: nodejs/package-lock.json
-    - uses: actions-rust-lang/setup-rust-toolchain@v1
-      with:
-        components: rustfmt, clippy
-    - name: Install dependencies
-      run: |
-        sudo apt update
-        sudo apt install -y protobuf-compiler libssl-dev
-    - uses: Swatinem/rust-cache@v2
-    - name: Format Rust
-      run: cargo fmt --all -- --check
-    - name: Lint Rust
-      run: cargo clippy --profile ci --all --all-features -- -D warnings
-    - name: Lint Typescript
-      run: |
-        npm ci
-        npm run lint-ci
-    - name: Lint examples
-      working-directory: nodejs/examples
-      run: npm ci && npm run lint-ci
-  linux:
-    name: Linux (NodeJS ${{ matrix.node-version }})
-    timeout-minutes: 30
-    strategy:
-      matrix:
-        node-version: [ "18", "20" ]
-    runs-on: "ubuntu-22.04"
-    defaults:
-      run:
-        shell: bash
-        working-directory: nodejs
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - uses: actions/setup-node@v3
-      with:
-        node-version: ${{ matrix.node-version }}
-        cache: 'npm'
-        cache-dependency-path: nodejs/package-lock.json
-    - uses: Swatinem/rust-cache@v2
-    - name: Install dependencies
-      run: |
-        sudo apt update
-        sudo apt install -y protobuf-compiler libssl-dev
-        npm install -g @napi-rs/cli
-    - name: Build
-      run: |
-        npm ci --include=optional
-        npm run build:debug -- --profile ci
-        npm run tsc
-    - name: Setup localstack
-      working-directory: .
-      run: docker compose up --detach --wait
-    - name: Test
-      env:
-        S3_TEST: "1"
-      run: npm run test
-    - name: Setup examples
-      working-directory: nodejs/examples
-      run: npm ci
-    - name: Test examples
-      working-directory: ./
-      env:
-        OPENAI_API_KEY: test
-        OPENAI_BASE_URL: http://0.0.0.0:8000
-      run: |
-        python ci/mock_openai.py &
-        cd nodejs/examples
-        npm test
-    - name: Check docs
-      run: |
-        # We run this as part of the job because the binary needs to be built
-        # first to export the types of the native code.
-        set -e
-        npm ci
-        npm run docs
-        if ! git diff --exit-code -- ../ ':(exclude)Cargo.lock'; then
-          echo "Docs need to be updated"
-          echo "Run 'npm run docs', fix any warnings, and commit the changes."
-          exit 1
-        fi
-  macos:
-    timeout-minutes: 30
-    runs-on: "macos-14"
-    defaults:
-      run:
-        shell: bash
-        working-directory: nodejs
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 20
-        cache: 'npm'
-        cache-dependency-path: nodejs/package-lock.json
-    - uses: Swatinem/rust-cache@v2
-    - name: Install dependencies
-      run: |
-        brew install protobuf
-        npm install -g @napi-rs/cli
-    - name: Build
-      run: |
-        npm ci --include=optional
-        npm run build:debug -- --profile ci
-        npm run tsc
-    - name: Test
-      run: |
-        npm run test

.github/workflows/npm-publish.yml

@@ -1,375 +1,163 @@
 name: NPM Publish
 
-env:
-  MACOSX_DEPLOYMENT_TARGET: '10.13'
-  CARGO_INCREMENTAL: '0'
-
-permissions:
-  contents: write
-  id-token: write
-
 on:
-  push:
-    branches:
-      - main
-    tags:
-      - "v*"
-  pull_request:
-    # This should trigger a dry run (we skip the final publish step)
-    paths:
-      - .github/workflows/npm-publish.yml
-      - Cargo.toml # Change in dependency frequently breaks builds
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  release:
+    types: [ published ]
 
 jobs:
-  gh-release:
-    if: startsWith(github.ref, 'refs/tags/v')
+  node:
     runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Extract version
-        id: extract_version
-        env:
-          GITHUB_REF: ${{ github.ref }}
-        run: |
-          set -e
-          echo "Extracting tag and version from $GITHUB_REF"
-          if [[ $GITHUB_REF =~ refs/tags/v(.*) ]]; then
-            VERSION=${BASH_REMATCH[1]}
-            TAG=v$VERSION
-            echo "tag=$TAG" >> $GITHUB_OUTPUT
-            echo "version=$VERSION" >> $GITHUB_OUTPUT
-          else
-            echo "Failed to extract version from $GITHUB_REF"
-            exit 1
-          fi
-          echo "Extracted version $VERSION from $GITHUB_REF"
-          if [[ $VERSION =~ beta ]]; then
-            echo "This is a beta release"
-
-            # Get last release (that is not this one)
-            FROM_TAG=$(git tag --sort='version:refname' \
-              | grep ^v \
-              | grep -vF "$TAG" \
-              | python ci/semver_sort.py v \
-              | tail -n 1)
-          else
-            echo "This is a stable release"
-            # Get last stable tag (ignore betas)
-            FROM_TAG=$(git tag --sort='version:refname' \
-              | grep ^v \
-              | grep -vF "$TAG" \
-              | grep -v beta \
-              | python ci/semver_sort.py v \
-              | tail -n 1)
-          fi
-          echo "Found from tag $FROM_TAG"
-          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
-      - name: Create Release Notes
-        id: release_notes
-        uses: mikepenz/release-changelog-builder-action@v4
-        with:
-          configuration: .github/release_notes.json
-          toTag: ${{ steps.extract_version.outputs.tag }}
-          fromTag: ${{ steps.extract_version.outputs.from_tag }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Create GH release
-        uses: softprops/action-gh-release@v2
-        with:
-          prerelease: ${{ contains('beta', github.ref) }}
-          tag_name: ${{ steps.extract_version.outputs.tag }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-          generate_release_notes: false
-          name: Node/Rust LanceDB v${{ steps.extract_version.outputs.version }}
-          body: ${{ steps.release_notes.outputs.changelog }}
-
-  build-lancedb:
-    strategy:
-      fail-fast: false
-      matrix:
-        settings:
-          - target: aarch64-apple-darwin
-            host: macos-latest
-            features: fp16kernels
-            pre_build: brew install protobuf
-          - target: x86_64-pc-windows-msvc
-            host: windows-latest
-            features: ","
-            pre_build: |-
-              choco install --no-progress protoc ninja nasm
-              tail -n 1000 /c/ProgramData/chocolatey/logs/chocolatey.log
-              # There is an issue where choco doesn't add nasm to the path
-              export PATH="$PATH:/c/Program Files/NASM"
-              nasm -v
-          - target: aarch64-pc-windows-msvc
-            host: windows-latest
-            features: ","
-            pre_build: |-
-                choco install --no-progress protoc
-                rustup target add aarch64-pc-windows-msvc
-          - target: x86_64-unknown-linux-gnu
-            host: ubuntu-latest
-            features: fp16kernels
-            # https://github.com/napi-rs/napi-rs/blob/main/debian.Dockerfile
-            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
-            pre_build: |-
-              set -e &&
-              apt-get update &&
-              apt-get install -y protobuf-compiler pkg-config
-          - target: x86_64-unknown-linux-musl
-            # This one seems to need some extra memory
-            host: ubuntu-2404-8x-x64
-            # https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile
-            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
-            features: fp16kernels
-            pre_build: |-
-              set -e &&
-              apk add protobuf-dev curl &&
-              ln -s /usr/lib/gcc/x86_64-alpine-linux-musl/14.2.0/crtbeginS.o /usr/lib/crtbeginS.o &&
-              ln -s /usr/lib/libgcc_s.so /usr/lib/libgcc.so &&
-              CC=gcc &&
-              CXX=g++
-          - target: aarch64-unknown-linux-gnu
-            host: ubuntu-2404-8x-x64
-            # https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
-            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
-            features: "fp16kernels"
-            pre_build: |-
-              set -e &&
-              apt-get update &&
-              apt-get install -y protobuf-compiler pkg-config &&
-              # https://github.com/aws/aws-lc-rs/issues/737#issuecomment-2725918627
-              ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc/aarch64-unknown-linux-gnu/4.8.5/crtbeginS.o /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/crtbeginS.o &&
-              ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/gcc &&
-              rustup target add aarch64-unknown-linux-gnu
-          - target: aarch64-unknown-linux-musl
-            host: ubuntu-2404-8x-x64
-            # https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile
-            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
-            features: ","
-            pre_build: |-
-              set -e &&
-              apk add protobuf-dev &&
-              rustup target add aarch64-unknown-linux-musl &&
-              export CC_aarch64_unknown_linux_musl=aarch64-linux-musl-gcc &&
-              export CXX_aarch64_unknown_linux_musl=aarch64-linux-musl-g++
-    name: build - ${{ matrix.settings.target }}
-    runs-on: ${{ matrix.settings.host }}
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
     defaults:
       run:
-        working-directory: nodejs
+        shell: bash
+        working-directory: node
     steps:
-      - uses: actions/checkout@v4
-      - name: Setup node
-        uses: actions/setup-node@v4
-        if: ${{ !matrix.settings.docker }}
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
         with:
           node-version: 20
-          cache: npm
-          cache-dependency-path: nodejs/package-lock.json
-      - name: Install
-        uses: dtolnay/rust-toolchain@stable
-        if: ${{ !matrix.settings.docker }}
-        with:
-          toolchain: stable
-          targets: ${{ matrix.settings.target }}
-      - name: Cache cargo
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            .cargo-cache
-            target/
-          key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}
-      - name: Setup toolchain
-        run: ${{ matrix.settings.setup }}
-        if: ${{ matrix.settings.setup }}
-        shell: bash
+          cache: 'npm'
+          cache-dependency-path: node/package-lock.json
       - name: Install dependencies
-        run: npm ci
-      - name: Build in docker
-        uses: addnab/docker-run-action@v3
-        if: ${{ matrix.settings.docker }}
-        with:
-          image: ${{ matrix.settings.docker }}
-          options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
-                    -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache \
-                    -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index \
-                    -v ${{ github.workspace }}:/build -w /build/nodejs"
-          run: |
-            set -e
-            ${{ matrix.settings.pre_build }}
-            npx napi build --platform  --release --no-const-enum \
-              --features ${{ matrix.settings.features }} \
-              --target ${{ matrix.settings.target }} \
-              --dts ../lancedb/native.d.ts \
-              --js ../lancedb/native.js \
-              --strip \
-              dist/
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
       - name: Build
         run: |
-          ${{ matrix.settings.pre_build }}
-          npx napi build --platform  --release --no-const-enum \
-              --features ${{ matrix.settings.features }} \
-              --target ${{ matrix.settings.target }} \
-              --dts ../lancedb/native.d.ts \
-              --js ../lancedb/native.js \
-              --strip \
-              $EXTRA_ARGS \
-              dist/
-        if: ${{ !matrix.settings.docker }}
-        shell: bash
-      - name: Upload artifact
-        uses: actions/upload-artifact@v4
+          npm ci
+          npm run tsc
+          npm pack
+      - name: Upload Linux Artifacts
+        uses: actions/upload-artifact@v3
         with:
-          name: lancedb-${{ matrix.settings.target }}
-          path: nodejs/dist/*.node
-          if-no-files-found: error
-      # The generic files are the same in all distros so we just pick
-      # one to do the upload.
-      - name: Make generic artifacts
-        if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
-        run: npm run tsc
-      - name: Upload Generic Artifacts
-        if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-dist
+          name: node-package
           path: |
-            nodejs/dist/*
-            !nodejs/dist/*.node
-  test-lancedb:
-    name: "Test: ${{ matrix.settings.target }} - node@${{ matrix.node }}"
-    needs:
-      - build-lancedb
+            node/vectordb-*.tgz
+
+  node-macos:
+    runs-on: macos-12
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
     strategy:
       fail-fast: false
       matrix:
-        settings:
-          # TODO: Get tests passing on Windows (failing from test tmpdir issue)
-          # - host: windows-latest
-          #   target: x86_64-pc-windows-msvc
-          - host: macos-latest
-            target: aarch64-apple-darwin
-          - target: x86_64-unknown-linux-gnu
-            host: ubuntu-latest
-          - target: aarch64-unknown-linux-gnu
-            host: buildjet-16vcpu-ubuntu-2204-arm
-        node:
-          - '20'
-    runs-on: ${{ matrix.settings.host }}
-    defaults:
-      run:
-        shell: bash
-        working-directory: nodejs
+        target: [x86_64-apple-darwin, aarch64-apple-darwin]
     steps:
-      - uses: actions/checkout@v4
-      - name: Setup node
-        uses: actions/setup-node@v4
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install system dependencies
+        run: brew install protobuf
+      - name: Install npm dependencies
+        run: |
+          cd node
+          npm ci
+      - name: Install rustup target
+        if: ${{ matrix.target == 'aarch64-apple-darwin' }}
+        run: rustup target add aarch64-apple-darwin
+      - name: Build MacOS native node modules
+        run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
+      - name: Upload Darwin Artifacts
+        uses: actions/upload-artifact@v3
         with:
-          node-version: ${{ matrix.node }}
-          cache: npm
-          cache-dependency-path: nodejs/package-lock.json
-      - name: Install dependencies
-        run: npm ci
-      - name: Download artifacts
-        uses: actions/download-artifact@v4
+          name: native-darwin
+          path: |
+            node/dist/lancedb-vectordb-darwin*.tgz
+
+  node-linux:
+    name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu
+    runs-on: ${{ matrix.config.runner }}
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - arch: x86_64
+            runner: ubuntu-latest
+          - arch: aarch64
+            runner: buildjet-4vcpu-ubuntu-2204-arm
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Build Linux Artifacts
+        run: |
+          bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}
+      - name: Upload Linux Artifacts
+        uses: actions/upload-artifact@v3
         with:
-          name: lancedb-${{ matrix.settings.target }}
-          path: nodejs/dist/
-          # For testing purposes:
-          # run-id: 13982782871
-          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
-      - uses: actions/download-artifact@v4
+          name: native-linux
+          path: |
+            node/dist/lancedb-vectordb-linux*.tgz
+
+  node-windows:
+    runs-on: windows-2022
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64-pc-windows-msvc]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install Protoc v21.12
+        working-directory: C:\
+        run: |
+          New-Item -Path 'C:\protoc' -ItemType Directory
+          Set-Location C:\protoc
+          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+          7z x protoc.zip
+          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+        shell: powershell
+      - name: Install npm dependencies
+        run: |
+          cd node
+          npm ci
+      - name: Build Windows native node modules
+        run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }}
+      - name: Upload Windows Artifacts
+        uses: actions/upload-artifact@v3
         with:
-          name: nodejs-dist
-          path: nodejs/dist
-          # For testing purposes:
-          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
-          # run-id: 13982782871
-      - name: List packages
-        run: ls -R dist
-      - name: Move built files
-        run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
-      - name: Test bindings
-        run: npm test
-  publish:
-    name: Publish
+          name: native-windows
+          path: |
+            node/dist/lancedb-vectordb-win32*.tgz
+
+  release:
+    needs: [node, node-macos, node-linux, node-windows]
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash
-        working-directory: nodejs
-    needs:
-      - test-lancedb
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
     steps:
-      - uses: actions/checkout@v4
-      - name: Setup node
-        uses: actions/setup-node@v4
+      - uses: actions/download-artifact@v3
+      - name: Display structure of downloaded files
+        run: ls -R
+      - uses: actions/setup-node@v3
         with:
           node-version: 20
-          cache: npm
-          cache-dependency-path: nodejs/package-lock.json
-          registry-url: "https://registry.npmjs.org"
-      - name: Install dependencies
-        run: npm ci
-      - uses: actions/download-artifact@v4
-        with:
-          name: nodejs-dist
-          path: nodejs/dist
-          # For testing purposes:
-          # run-id: 13982782871
-          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
-      - uses: actions/download-artifact@v4
-        name: Download arch-specific binaries
-        with:
-          pattern: lancedb-*
-          path: nodejs/nodejs-artifacts
-          merge-multiple: true
-          # For testing purposes:
-          # run-id: 13982782871
-          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
-      - name: Display structure of downloaded files
-        run: find dist && find nodejs-artifacts
-      - name: Move artifacts
-        run: npx napi artifacts -d nodejs-artifacts
-      - name: List packages
-        run: find npm
-      - name: Publish
+          registry-url: 'https://registry.npmjs.org'
+      - name: Publish to NPM
         env:
           NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
-          DRY_RUN: ${{ !startsWith(github.ref, 'refs/tags/v') }}
         run: |
-          ARGS="--access public"
-          if [[ $DRY_RUN == "true" ]]; then
-            ARGS="$ARGS --dry-run"
-          fi
-          if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
-            ARGS="$ARGS --tag preview"
-          fi
-          npm publish $ARGS
-  report-failure:
-    name: Report Workflow Failure
+          mv */*.tgz .
+          for filename in *.tgz; do
+            npm publish $filename
+          done
+
+  update-package-lock:
+    needs: [release]
     runs-on: ubuntu-latest
-    needs: [build-lancedb, test-lancedb, publish]
-    if: always() && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
-    permissions:
-      contents: read
-      issues: write
     steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/create-failure-issue
+      - name: Checkout
+        uses: actions/checkout@v3
         with:
-          job-results: ${{ toJSON(needs) }}
-          workflow-name: ${{ github.workflow }}
+          ref: main
+          persist-credentials: false
+          fetch-depth: 0
+          lfs: true
+      - uses: ./.github/workflows/update_package_lock
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.github/workflows/pypi-publish.yml

@@ -1,190 +1,31 @@
 name: PyPI Publish
 
 on:
-  push:
-    tags:
-      - 'python-v*'
-  pull_request:
-    # This should trigger a dry run (we skip the final publish step)
-    paths:
-      - .github/workflows/pypi-publish.yml
-      - Cargo.toml # Change in dependency frequently breaks builds
-
-env:
-  PIP_EXTRA_INDEX_URL: "https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/"
+  release:
+    types: [ published ]
 
 jobs:
-  linux:
-    name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
-    timeout-minutes: 60
-    strategy:
-      matrix:
-        config:
-          - platform: x86_64
-            manylinux: "2_17"
-            extra_args: ""
-            runner: ubuntu-22.04
-          - platform: x86_64
-            manylinux: "2_28"
-            extra_args: "--features fp16kernels"
-            runner: ubuntu-22.04
-          - platform: aarch64
-            manylinux: "2_17"
-            extra_args: ""
-            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
-            runner: ubuntu-2404-8x-arm64
-          - platform: aarch64
-            manylinux: "2_28"
-            extra_args: "--features fp16kernels"
-            runner: ubuntu-2404-8x-arm64
-    runs-on: ${{ matrix.config.runner }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-      - uses: ./.github/workflows/build_linux_wheel
-        with:
-          python-minor-version: 8
-          args: "--release --strip ${{ matrix.config.extra_args }}"
-          arm-build: ${{ matrix.config.platform == 'aarch64' }}
-          manylinux: ${{ matrix.config.manylinux }}
-      - uses: ./.github/workflows/upload_wheel
-        if: startsWith(github.ref, 'refs/tags/python-v')
-        with:
-          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
-          fury_token: ${{ secrets.FURY_TOKEN }}
-  mac:
-    timeout-minutes: 90
-    runs-on: ${{ matrix.config.runner }}
-    strategy:
-      matrix:
-        config:
-          - target: aarch64-apple-darwin
-            runner: warp-macos-14-arm64-6x
-    env:
-      MACOSX_DEPLOYMENT_TARGET: 10.15
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.12
-      - uses: ./.github/workflows/build_mac_wheel
-        with:
-          python-minor-version: 8
-          args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
-      - uses: ./.github/workflows/upload_wheel
-        if: startsWith(github.ref, 'refs/tags/python-v')
-        with:
-          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
-          fury_token: ${{ secrets.FURY_TOKEN }}
-  windows:
-    timeout-minutes: 60
-    runs-on: windows-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.12
-      - uses: ./.github/workflows/build_windows_wheel
-        with:
-          python-minor-version: 8
-          args: "--release --strip"
-          vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }}
-      - uses: ./.github/workflows/upload_wheel
-        if: startsWith(github.ref, 'refs/tags/python-v')
-        with:
-          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
-          fury_token: ${{ secrets.FURY_TOKEN }}
-  gh-release:
+  publish:
+    runs-on: ubuntu-latest
+    # Only runs on tags that matches the python-make-release action
     if: startsWith(github.ref, 'refs/tags/python-v')
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
-          fetch-depth: 0
-          lfs: true
-      - name: Extract version
-        id: extract_version
-        env:
-          GITHUB_REF: ${{ github.ref }}
+          python-version: "3.8"
+      - name: Build distribution
         run: |
-          set -e
-          echo "Extracting tag and version from $GITHUB_REF"
-          if [[ $GITHUB_REF =~ refs/tags/python-v(.*) ]]; then
-            VERSION=${BASH_REMATCH[1]}
-            TAG=python-v$VERSION
-            echo "tag=$TAG" >> $GITHUB_OUTPUT
-            echo "version=$VERSION" >> $GITHUB_OUTPUT
-          else
-            echo "Failed to extract version from $GITHUB_REF"
-            exit 1
-          fi
-          echo "Extracted version $VERSION from $GITHUB_REF"
-          if [[ $VERSION =~ beta ]]; then
-            echo "This is a beta release"
-
-            # Get last release (that is not this one)
-            FROM_TAG=$(git tag --sort='version:refname' \
-              | grep ^python-v \
-              | grep -vF "$TAG" \
-              | python ci/semver_sort.py python-v \
-              | tail -n 1)
-          else
-            echo "This is a stable release"
-            # Get last stable tag (ignore betas)
-            FROM_TAG=$(git tag --sort='version:refname' \
-              | grep ^python-v \
-              | grep -vF "$TAG" \
-              | grep -v beta \
-              | python ci/semver_sort.py python-v \
-              | tail -n 1)
-          fi
-          echo "Found from tag $FROM_TAG"
-          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
-      - name: Create Python Release Notes
-        id: python_release_notes
-        uses: mikepenz/release-changelog-builder-action@v4
+          ls -la
+          pip install wheel setuptools --upgrade
+          python setup.py sdist bdist_wheel
+      - name: Publish
+        uses: pypa/gh-action-pypi-publish@v1.8.5
         with:
-          configuration: .github/release_notes.json
-          toTag: ${{ steps.extract_version.outputs.tag }}
-          fromTag: ${{ steps.extract_version.outputs.from_tag }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Create Python GH release
-        uses: softprops/action-gh-release@v2
-        with:
-          prerelease: ${{ contains('beta', github.ref) }}
-          tag_name: ${{ steps.extract_version.outputs.tag }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-          generate_release_notes: false
-          name: Python LanceDB v${{ steps.extract_version.outputs.version }}
-          body: ${{ steps.python_release_notes.outputs.changelog }}
-  report-failure:
-    name: Report Workflow Failure
-    runs-on: ubuntu-latest
-    needs: [linux, mac, windows]
-    permissions:
-      contents: read
-      issues: write
-    if: always() && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/create-failure-issue
-        with:
-          job-results: ${{ toJSON(needs) }}
-          workflow-name: ${{ github.workflow }}
+          password: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
+          packages-dir: python/dist

.github/workflows/python-make-release-commit.yml

@@ -0,0 +1,56 @@
+name: Python - Create release commit
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: 'Dry run (create the local commit/tags but do not push it)'
+        required: true
+        default: "false"
+        type: choice
+        options:
+          - "true"
+          - "false"
+      part:
+        description: 'What kind of release is this?'
+        required: true
+        default: 'patch'
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  bump-version:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out main
+      uses: actions/checkout@v3
+      with:
+        ref: main
+        persist-credentials: false
+        fetch-depth: 0
+        lfs: true
+    - name: Set git configs for bumpversion
+      shell: bash
+      run: |
+        git config user.name 'Lance Release'
+        git config user.email 'lance-dev@lancedb.com'
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+    - name: Bump version, create tag and commit
+      working-directory: python
+      run: |
+        pip install bump2version
+        bumpversion --verbose ${{ inputs.part }}
+    - name: Push new version and tag
+      if: ${{ inputs.dry_run }} == "false"
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+        branch: main
+        tags: true
+

.github/workflows/python.yml

@@ -6,24 +6,67 @@ on:
       - main
   pull_request:
     paths:
-      - Cargo.toml
       - python/**
       - .github/workflows/python.yml
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-env:
-  # Color output for pytest is off by default.
-  PYTEST_ADDOPTS: "--color=yes"
-  FORCE_COLOR: "1"
-  PIP_EXTRA_INDEX_URL: "https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/"
-  RUST_BACKTRACE: "1"
-
 jobs:
-  lint:
-    name: "Lint"
+  linux:
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        python-minor-version: [ "8", "9", "10", "11" ]
+    runs-on: "ubuntu-22.04"
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.${{ matrix.python-minor-version }}
+    - name: Install lancedb
+      run: |
+        pip install -e .[tests]
+        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+        pip install pytest pytest-mock black isort
+    - name: Black
+      run: black --check --diff --no-color --quiet .
+    - name: isort
+      run: isort --check --diff --quiet .
+    - name: Run tests
+      run: pytest -m "not slow" -x -v --durations=30 tests
+    - name: doctest
+      run: pytest --doctest-modules lancedb
+  mac:
+    timeout-minutes: 30
+    runs-on: "macos-12"
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.11"
+    - name: Install lancedb
+      run: |
+        pip install -e .[tests]
+        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+        pip install pytest pytest-mock black
+    - name: Black
+      run: black --check --diff --no-color --quiet .
+    - name: Run tests
+      run: pytest -m "not slow" -x -v --durations=30 tests
+  pydantic1x:
     timeout-minutes: 30
     runs-on: "ubuntu-22.04"
     defaults:
@@ -31,195 +74,25 @@ jobs:
         shell: bash
         working-directory: python
     steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
-      - name: Install ruff
-        run: |
-          pip install ruff==0.9.9
-      - name: Format check
-        run: ruff format --check .
-      - name: Lint
-        run: ruff check .
-
-  type-check:
-    name: "Type Check"
-    timeout-minutes: 60
-    runs-on: ubuntu-2404-8x-x64
-    defaults:
-      run:
-        shell: bash
-        working-directory: python
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
-      - name: Install protobuf compiler
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler
-          pip install toml
-      - name: Install dependencies
-        run: |
-          python ../ci/parse_requirements.py pyproject.toml --extras dev,tests,embeddings > requirements.txt
-          pip install -r requirements.txt
-      - name: Run pyright
-        run: pyright
-
-  doctest:
-    name: "Doctest"
-    timeout-minutes: 60
-    runs-on: ubuntu-2404-8x-x64
-    defaults:
-      run:
-        shell: bash
-        working-directory: python
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
-          cache: "pip"
-      - name: Install protobuf
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler
-      - name: Install
-        run: |
-          pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings]
-          pip install tantivy
-          pip install mlx
-      - name: Doctest
-        run: pytest --doctest-modules python/lancedb
-  linux:
-    name: "Linux: python-3.${{ matrix.python-minor-version }}"
-    timeout-minutes: 30
-    strategy:
-      matrix:
-        python-minor-version: ["9", "12"]
-    runs-on: "ubuntu-24.04"
-    defaults:
-      run:
-        shell: bash
-        working-directory: python
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Install protobuf
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.${{ matrix.python-minor-version }}
-      - uses: ./.github/workflows/build_linux_wheel
-        with:
-          args: --profile ci
-      - uses: ./.github/workflows/run_tests
-        with:
-          integration: true
-      - name: Test without pylance or pandas
-        run: |
-          pip uninstall -y pylance pandas
-          pytest -vv python/tests/test_table.py
-      # Make sure wheels are not included in the Rust cache
-      - name: Delete wheels
-        run: rm -rf target/wheels
-  platform:
-    name: "Mac"
-    timeout-minutes: 30
-    runs-on: macos-14
-    defaults:
-      run:
-        shell: bash
-        working-directory: python
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
-      - uses: ./.github/workflows/build_mac_wheel
-        with:
-          args: --profile ci
-      - uses: ./.github/workflows/run_tests
-      # Make sure wheels are not included in the Rust cache
-      - name: Delete wheels
-        run: rm -rf target/wheels
-  windows:
-    name: "Windows: ${{ matrix.config.name }}"
-    timeout-minutes: 60
-    strategy:
-      matrix:
-        config:
-          - name: x86
-            runner: windows-latest
-    runs-on: "${{ matrix.config.runner }}"
-    defaults:
-      run:
-        shell: bash
-        working-directory: python
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
-      - uses: ./.github/workflows/build_windows_wheel
-        with:
-          args: --profile ci
-      - uses: ./.github/workflows/run_tests
-      # Make sure wheels are not included in the Rust cache
-      - name: Delete wheels
-        run: rm -rf target/wheels
-  pydantic1x:
-    timeout-minutes: 30
-    runs-on: "ubuntu-24.04"
-    defaults:
-      run:
-        shell: bash
-        working-directory: python
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.9
-      - name: Install lancedb
-        run: |
-          pip install "pydantic<2"
-          pip install pyarrow==16
-          pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
-          pip install tantivy
-      - name: Run tests
-        run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+    - name: Install lancedb
+      run: |
+        pip install "pydantic<2"
+        pip install -e .[tests]
+        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+        pip install pytest pytest-mock black isort
+    - name: Black
+      run: black --check --diff --no-color --quiet .
+    - name: isort
+      run: isort --check --diff --quiet .
+    - name: Run tests
+      run: pytest -m "not slow" -x -v --durations=30 tests
+    - name: doctest
+      run: pytest --doctest-modules lancedb

.github/workflows/run_tests/action.yml

@@ -1,31 +0,0 @@
-name: run-tests
-
-description: "Install lance wheel and run unit tests"
-inputs:
-  python-minor-version:
-    required: true
-    description: "8 9 10 11 12"
-  integration:
-    required: false
-    description: "Run integration tests"
-    default: "false"
-runs:
-  using: "composite"
-  steps:
-    - name: Install lancedb
-      shell: bash
-      run: |
-        pip3 install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ $(ls target/wheels/lancedb-*.whl)[tests,dev]
-    - name: Setup localstack for integration tests
-      if: ${{ inputs.integration == 'true' }}
-      shell: bash
-      working-directory: .
-      run: docker compose up --detach --wait
-    - name: pytest (with integration)
-      shell: bash
-      if: ${{ inputs.integration == 'true' }}
-      run: pytest -m "not slow" -vv --durations=30 python/python/tests
-    - name: pytest (no integration tests)
-      shell: bash
-      if: ${{ inputs.integration != 'true' }}
-      run: pytest -m "not slow and not s3_test" -vv --durations=30 python/python/tests

.github/workflows/rust.yml

@@ -10,206 +10,80 @@ on:
       - rust/**
       - .github/workflows/rust.yml
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
 env:
   # This env var is used by Swatinem/rust-cache@v2 for the cache
   # key, so we set it to make sure it is always consistent.
   CARGO_TERM_COLOR: always
+  # Disable full debug symbol generation to speed up CI build and keep memory down
+  # "1" means line tables only, which is useful for panic tracebacks.
+  RUSTFLAGS: "-C debuginfo=1"
   RUST_BACKTRACE: "1"
 
 jobs:
-  lint:
-    timeout-minutes: 30
-    runs-on: ubuntu-24.04
-    defaults:
-      run:
-        shell: bash
-    env:
-      # Need up-to-date compilers for kernels
-      CC: clang-18
-      CXX: clang++-18
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
-        with:
-          components: rustfmt, clippy
-      - uses: Swatinem/rust-cache@v2
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Run format
-        run: cargo fmt --all -- --check
-      - name: Run clippy
-        run: cargo clippy --profile ci --workspace --tests --all-features -- -D warnings
-
-  build-no-lock:
-    runs-on: ubuntu-24.04
-    timeout-minutes: 30
-    env:
-      # Need up-to-date compilers for kernels
-      CC: clang
-      CXX: clang++
-    steps:
-      - uses: actions/checkout@v4
-      # Building without a lock file often requires the latest Rust version since downstream
-      # dependencies may have updated their minimum Rust version.
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
-        with:
-          toolchain: "stable"
-      # Remove cargo.lock to force a fresh build
-      - name: Remove Cargo.lock
-        run: rm -f Cargo.lock
-      - uses: rui314/setup-mold@v1
-      - uses: Swatinem/rust-cache@v2
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Build all
-        run: |
-          cargo build --profile ci --benches --all-features --tests
-
   linux:
     timeout-minutes: 30
-    # To build all features, we need more disk space than is available
-    # on the free OSS github runner. This is mostly due to the the
-    # sentence-transformers feature.
-    runs-on: ubuntu-2404-4x-x64
+    runs-on: ubuntu-22.04
     defaults:
       run:
         shell: bash
         working-directory: rust
-    env:
-      # Need up-to-date compilers for kernels
-      CC: clang-18
-      CXX: clang++-18
-      GH_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
     steps:
-      - uses: actions/checkout@v4
-        with:
+    - uses: actions/checkout@v3
+      with:
           fetch-depth: 0
           lfs: true
-      - uses: Swatinem/rust-cache@v2
-      - name: Install dependencies
-        run: sudo apt install -y protobuf-compiler libssl-dev
-      - uses: rui314/setup-mold@v1
-      - name: Make Swap
-        run: |
-          sudo fallocate -l 16G /swapfile
-          sudo chmod 600 /swapfile
-          sudo mkswap /swapfile
-          sudo swapon /swapfile
-      - name: Build
-        run: cargo build --profile ci --all-features --tests --locked --examples
-      - name: Run feature tests
-        run: CARGO_ARGS="--profile ci" make -C ./lancedb feature-tests
-      - name: Run examples
-        run: cargo run  --profile ci --example simple --locked
-      - name: Run remote tests
-        # Running this requires access to secrets, so skip if this is
-        # a PR from a fork.
-        if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
-        run: CARGO_ARGS="--profile ci" make -C ./lancedb remote-tests
-
+    - uses: Swatinem/rust-cache@v2
+      with:
+          workspaces: rust
+    - name: Install dependencies
+      run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+    - name: Build
+      run: cargo build --all-features
+    - name: Run tests
+      run: cargo test --all-features
   macos:
+    runs-on: macos-12
     timeout-minutes: 30
-    strategy:
-      matrix:
-        mac-runner: ["macos-14", "macos-15"]
-    runs-on: "${{ matrix.mac-runner }}"
     defaults:
       run:
         shell: bash
         working-directory: rust
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
         with:
-          fetch-depth: 0
-          lfs: true
+            fetch-depth: 0
+            lfs: true
       - name: CPU features
         run: sysctl -a | grep cpu
       - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust
       - name: Install dependencies
         run: brew install protobuf
+      - name: Build
+        run: cargo build --all-features
       - name: Run tests
-        run: |
-          # Don't run the s3 integration tests since docker isn't available
-          # on this image.
-          ALL_FEATURES=`cargo metadata --format-version=1 --no-deps \
-            | jq -r '.packages[] | .features | keys | .[]' \
-            | grep -v s3-test | sort | uniq | paste -s -d "," -`
-          cargo test --profile ci --features $ALL_FEATURES --locked
-
+        run: cargo test --all-features
   windows:
     runs-on: windows-2022
-    strategy:
-      matrix:
-        target:
-          - x86_64-pc-windows-msvc
-          - aarch64-pc-windows-msvc
-    defaults:
-      run:
-        working-directory: rust/lancedb
     steps:
-      - uses: actions/checkout@v4
-      - name: Set target
-        run: rustup target add ${{ matrix.target }}
+      - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust
       - name: Install Protoc v21.12
-        run: choco install --no-progress protoc
-      - name: Build
+        working-directory: C:\
         run: |
-          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
-          cargo build --profile ci --features remote --tests --locked --target ${{ matrix.target }}
+          New-Item -Path 'C:\protoc' -ItemType Directory
+          Set-Location C:\protoc
+          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+          7z x protoc.zip
+          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+        shell: powershell
       - name: Run tests
-        # Can only run tests when target matches host
-        if: ${{ matrix.target == 'x86_64-pc-windows-msvc' }}
         run: |
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
-          cargo test --profile ci --features remote --locked
-
-  msrv:
-    # Check the minimum supported Rust version
-    name: MSRV Check - Rust v${{ matrix.msrv }}
-    runs-on: ubuntu-24.04
-    strategy:
-      matrix:
-        msrv: ["1.78.0"] # This should match up with rust-version in Cargo.toml
-    env:
-      # Need up-to-date compilers for kernels
-      CC: clang-18
-      CXX: clang++-18
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Install ${{ matrix.msrv }}
-        uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ matrix.msrv }}
-      - uses: Swatinem/rust-cache@v2
-      - name: Downgrade  dependencies
-        # These packages have newer requirements for MSRV
-        run: |
-          cargo update -p aws-sdk-bedrockruntime --precise 1.64.0
-          cargo update -p aws-sdk-dynamodb --precise 1.55.0
-          cargo update -p aws-config --precise 1.5.10
-          cargo update -p aws-sdk-kms --precise 1.51.0
-          cargo update -p aws-sdk-s3 --precise 1.65.0
-          cargo update -p aws-sdk-sso --precise 1.50.0
-          cargo update -p aws-sdk-ssooidc --precise 1.51.0
-          cargo update -p aws-sdk-sts --precise 1.51.0
-          cargo update -p home --precise 0.5.9
-      - name: cargo +${{ matrix.msrv }} check
-        run: cargo check --profile ci --workspace --tests --benches --all-features
+          cargo build
+          cargo test

.github/workflows/trigger-vectordb-recipes.yml

@@ -0,0 +1,26 @@
+name: Trigger vectordb-recipers workflow
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    paths:
+      - .github/workflows/trigger-vectordb-recipes.yml
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Trigger vectordb-recipes workflow
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.VECTORDB_RECIPES_ACTION_TOKEN }}
+          script: |
+            const result = await github.rest.actions.createWorkflowDispatch({
+                owner: 'lancedb',
+                repo: 'vectordb-recipes',
+                workflow_id: 'examples-test.yml',
+                ref: 'main'
+            });
+            console.log(result);

.github/workflows/update_package_lock/action.yml

@@ -0,0 +1,33 @@
+name: update_package_lock
+description: "Update node's package.lock"
+
+inputs:
+  github_token:
+    required: true
+    description: "github token for the repo"
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 20
+    - name: Set git configs
+      shell: bash
+      run: |
+        git config user.name 'Lance Release'
+        git config user.email 'lance-dev@lancedb.com'
+    - name: Update package-lock.json file
+      working-directory: ./node
+      run: |
+        npm install
+        git add package-lock.json
+        git commit -m "Updating package-lock.json"
+      shell: bash
+    - name: Push changes
+      if: ${{ inputs.dry_run }} == "false"
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ inputs.github_token }}
+        branch: main
+        tags: true

.github/workflows/update_package_lock_run.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           ref: main
           persist-credentials: false

.github/workflows/update_package_lock_run_nodejs.yml

@@ -1,19 +0,0 @@
-name: Update NodeJs package-lock.json
-
-on:
-  workflow_dispatch:
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: main
-          persist-credentials: false
-          fetch-depth: 0
-          lfs: true
-      - uses: ./.github/workflows/update_package_lock_nodejs
-        with:
-          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.github/workflows/upload_wheel/action.yml

@@ -1,45 +0,0 @@
-name: upload-wheel
-
-description: "Upload wheels to Pypi"
-inputs:
-  pypi_token:
-    required: true
-    description: "release token for the repo"
-  fury_token:
-    required: true
-    description: "release token for the fury repo"
-
-runs:
-  using: "composite"
-  steps:
-  - name: Install dependencies
-    shell: bash
-    run: |
-      python -m pip install --upgrade pip
-      pip install twine
-      python3 -m pip install --upgrade pkginfo
-  - name: Choose repo
-    shell: bash
-    id: choose_repo
-    run: |
-      if [[ ${{ github.ref }} == *beta* ]]; then
-        echo "repo=fury" >> $GITHUB_OUTPUT
-      else
-        echo "repo=pypi" >> $GITHUB_OUTPUT
-      fi
-  - name: Publish to PyPI
-    shell: bash
-    env:
-      FURY_TOKEN: ${{ inputs.fury_token }}
-      PYPI_TOKEN: ${{ inputs.pypi_token }}
-    run: |
-      if [[ ${{ steps.choose_repo.outputs.repo }} == fury ]]; then
-        WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1)
-        echo "Uploading $WHEEL to Fury"
-        curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/
-      else
-        twine upload --repository ${{ steps.choose_repo.outputs.repo }} \
-          --username __token__ \
-          --password $PYPI_TOKEN \
-          target/wheels/lancedb-*.whl
-      fi

.gitignore

@@ -1,15 +1,14 @@
 .idea
-*.swp
 **/*.whl
 *.egg-info
 **/__pycache__
 .DS_Store
 venv
-.venv
 
 .vscode
-.zed
+
 rust/target
+rust/Cargo.lock
 
 site
 
@@ -23,19 +22,15 @@ python/dist
 
 **/.hypothesis
 
-# Compiled Dynamic libraries
-*.so
-*.dylib
-*.dll
-
 ## Javascript
 *.node
 **/node_modules
 **/.DS_Store
-nodejs/lancedb/native*
-dist
+node/dist
+node/examples/**/package-lock.json
+node/examples/**/dist
 
 ## Rust
 target
 
-**/sccache.log
+Cargo.lock

.pre-commit-config.yaml

@@ -1,27 +1,21 @@
 repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
+-   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v3.2.0
     hooks:
-      - id: check-yaml
-      - id: end-of-file-fixer
-      - id: trailing-whitespace
-  - repo: https://github.com/astral-sh/ruff-pre-commit
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+-   repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+    -   id: black
+-   repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.9.9
+    rev: v0.0.277
     hooks:
-      - id: ruff
-  # - repo: https://github.com/RobertCraigie/pyright-python
-  #   rev: v1.1.395
-  #   hooks:
-  #     - id: pyright
-  #       args: ["--project", "python"]
-  #       additional_dependencies: [pyarrow-stubs]
-  - repo: local
+    - id: ruff
+-   repo: https://github.com/pycqa/isort
+    rev: 5.12.0
     hooks:
-      - id: local-biome-check
-        name: biome check
-        entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
-        language: system
-        types: [text]
-        files: "nodejs/.*"
-        exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*|nodejs/examples/.*
+    - id: isort
+      name: isort (python)

AGENTS.md

@@ -1,101 +0,0 @@
-LanceDB is a database designed for retrieval, including vector, full-text, and hybrid search.
-It is a wrapper around Lance. There are two backends: local (in-process like SQLite) and
-remote (against LanceDB Cloud).
-
-The core of LanceDB is written in Rust. There are bindings in Python, Typescript, and Java.
-
-Project layout:
-
-* `rust/lancedb`: The LanceDB core Rust implementation.
-* `python`: The Python bindings, using PyO3.
-* `nodejs`: The Typescript bindings, using napi-rs
-* `java`: The Java bindings
-
-Common commands:
-
-* Check for compiler errors: `cargo check --quiet --features remote --tests --examples`
-* Run tests: `cargo test --quiet --features remote --tests`
-* Run specific test: `cargo test --quiet --features remote -p <package_name> --test <test_name>`
-* Lint: `cargo clippy --quiet --features remote --tests --examples`
-* Format: `cargo fmt --all`
-
-Before committing changes, run formatting.
-
-## Coding tips
-
-* When writing Rust doctests for things that require a connection or table reference,
-  write them as a function instead of a fully executable test. This allows type checking
-  to run but avoids needing a full test environment. For example:
-    ```rust
-    /// ```
-    /// use lance_index::scalar::FullTextSearchQuery;
-    /// use lancedb::query::{QueryBase, ExecutableQuery};
-    ///
-    /// # use lancedb::Table;
-    /// # async fn query(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
-    /// let results = table.query()
-    ///     .full_text_search(FullTextSearchQuery::new("hello world".into()))
-    ///     .execute()
-    ///     .await?;
-    /// # Ok(())
-    /// # }
-    /// ```
-    ```
-
-## Example plan: adding a new method on Table
-
-Adding a new method involves first adding it to the Rust core, then exposing it
-in the Python and TypeScript bindings. There are both local and remote tables.
-Remote tables are implemented via a HTTP API and require the `remote` cargo
-feature flag to be enabled. Python has both sync and async methods.
-
-Rust core changes:
-
-1. Add method on `Table` struct in `rust/lancedb/src/table.rs` (calls `BaseTable` trait).
-2. Add method to `BaseTable` trait in `rust/lancedb/src/table.rs`.
-3. Implement new trait method on `NativeTable` in `rust/lancedb/src/table.rs`.
-    * Test with unit test in `rust/lancedb/src/table.rs`.
-4. Implement new trait method on `RemoteTable` in `rust/lancedb/src/remote/table.rs`.
-    * Test with unit test in `rust/lancedb/src/remote/table.rs` against mocked endpoint.
-
-Python bindings changes:
-
-1. Add PyO3 method binding in `python/src/table.rs`. Run `make develop` to compile bindings.
-2. Add types for PyO3 method in `python/python/lancedb/_lancedb.pyi`.
-3. Add method to `AsyncTable` class in `python/python/lancedb/table.py`.
-4. Add abstract method to `Table` abstract base class in `python/python/lancedb/table.py`.
-5. Add concrete sync method to `LanceTable` class in `python/python/lancedb/table.py`.
-    * Should use `LOOP.run()` to call the corresponding `AsyncTable` method.
-6. Add concrete sync method to `RemoteTable` class in `python/python/lancedb/remote/table.py`.
-7. Add unit test in `python/tests/test_table.py`.
-
-TypeScript bindings changes:
-
-1. Add napi-rs method binding on `Table` in `nodejs/src/table.rs`.
-2. Run `npm run build` to generate TypeScript definitions.
-3. Add typescript method on abstract class `Table` in `nodejs/src/table.ts`.
-4. Add concrete method on `LocalTable` class in `nodejs/src/native_table.ts`.
-    * Note: despite the name, this class is also used for remote tables.
-5. Add test in `nodejs/__test__/table.test.ts`.
-6. Run `npm run docs` to generate TypeScript documentation.
-
-## Review Guidelines
-
-Please consider the following when reviewing code contributions.
-
-### Rust API design
-* Design public APIs so they can be evolved easily in the future without breaking
-  changes. Often this means using builder patterns or options structs instead of
-  long argument lists.
-* For public APIs, prefer inputs that use `Into<T>` or `AsRef<T>` traits to allow
-  more flexible inputs. For example, use `name: Into<String>` instead of `name: String`,
-  so we don't have to write `func("my_string".to_string())`.
-
-### Testing
-* Ensure all new public APIs have documentation and examples.
-* Ensure that all bugfixes and features have corresponding tests. **We do not merge
-  code without tests.**
-
-### Documentation
-* New features must include updates to the rust documentation comments. Link to
-  relevant structs and methods to increase the value of documentation.

CLAUDE.md

@@ -1 +0,0 @@
-AGENTS.md

CONTRIBUTING.md

@@ -1,78 +0,0 @@
-# Contributing to LanceDB
-
-LanceDB is an open-source project and we welcome contributions from the community.
-This document outlines the process for contributing to LanceDB.
-
-## Reporting Issues
-
-If you encounter a bug or have a feature request, please open an issue on the
-[GitHub issue tracker](https://github.com/lancedb/lancedb).
-
-## Picking an issue
-
-We track issues on the GitHub issue tracker. If you are looking for something to
-work on, check the [good first issue](https://github.com/lancedb/lancedb/contribute) label. These issues are typically the best described and have the smallest scope.
-
-If there's an issue you are interested in working on, please leave a comment on the issue. This will help us avoid duplicate work. Additionally, if you have questions about the issue, please ask them in the issue comments. We are happy to provide guidance on how to approach the issue.
-
-## Configuring Git
-
-First, fork the repository on GitHub, then clone your fork:
-
-```bash
-git clone https://github.com/<username>/lancedb.git
-cd lancedb
-```
-
-Then add the main repository as a remote:
-
-```bash
-git remote add upstream https://github.com/lancedb/lancedb.git
-git fetch upstream
-```
-
-## Setting up your development environment
-
-We have development environments for Python, Typescript, and Java. Each environment has its own setup instructions.
-
-* [Python](python/CONTRIBUTING.md)
-* [Typescript](nodejs/CONTRIBUTING.md)
-<!-- TODO: add Java contributing guide -->
-* [Documentation](docs/README.md)
-
-
-## Best practices for pull requests
-
-For the best chance of having your pull request accepted, please follow these guidelines:
-
-1. Unit test all bug fixes and new features. Your code will not be merged if it
-   doesn't have tests.
-1. If you change the public API, update the documentation in the `docs` directory.
-1. Aim to minimize the number of changes in each pull request. Keep to solving
-   one problem at a time, when possible.
-1. Before marking a pull request ready-for-review, do a self review of your code.
-   Is it clear why you are making the changes? Are the changes easy to understand?
-1. Use [conventional commit messages](https://www.conventionalcommits.org/en/) as pull request titles. Examples:
-    * New feature: `feat: adding foo API`
-    * Bug fix: `fix: issue with foo API`
-    * Documentation change: `docs: adding foo API documentation`
-1. If your pull request is a work in progress, leave the pull request as a draft.
-   We will assume the pull request is ready for review when it is opened.
-1. When writing tests, test the error cases. Make sure they have understandable
-   error messages.
-
-## Project structure
-
-The core library is written in Rust. The Python, Typescript, and Java libraries
-are wrappers around the Rust library.
-
-* `src/lancedb`: Rust library source code
-* `python`: Python package source code
-* `nodejs`: Typescript package source code
-* `node`: **Deprecated** Typescript package source code
-* `java`: Java package source code
-* `docs`: Documentation source code
-
-## Release process
-
-For information on the release process, see: [release_process.md](release_process.md)

Cargo.toml

@@ -1,79 +1,27 @@
 [workspace]
-members = ["rust/lancedb", "nodejs", "python"]
+members = ["rust/ffi/node", "rust/vectordb"]
 # Python package needs to be built by maturin.
 exclude = ["python"]
 resolver = "2"
 
-[workspace.package]
-edition = "2021"
-authors = ["LanceDB Devs <dev@lancedb.com>"]
-license = "Apache-2.0"
-repository = "https://github.com/lancedb/lancedb"
-description = "Serverless, low-latency vector database for AI applications"
-keywords = ["lancedb", "lance", "database", "vector", "search"]
-categories = ["database-implementations"]
-rust-version = "1.78.0"
-
 [workspace.dependencies]
-lance = { "version" = "=1.0.1-beta.1", default-features = false, "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=1.0.1-beta.1", default-features = false, "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=1.0.1-beta.1", default-features = false, "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-ahash = "0.8"
+lance = { "version" = "=0.8.3", "features" = ["dynamodb"] }
+lance-linalg = { "version" = "=0.8.3" }
+lance-testing = { "version" = "=0.8.3" }
 # Note that this one does not include pyarrow
-arrow = { version = "56.2", optional = false }
-arrow-array = "56.2"
-arrow-data = "56.2"
-arrow-ipc = "56.2"
-arrow-ord = "56.2"
-arrow-schema = "56.2"
-arrow-select = "56.2"
-arrow-cast = "56.2"
-async-trait = "0"
-datafusion = { version = "50.1", default-features = false }
-datafusion-catalog = "50.1"
-datafusion-common = { version = "50.1", default-features = false }
-datafusion-execution = "50.1"
-datafusion-expr = "50.1"
-datafusion-physical-plan = "50.1"
-env_logger = "0.11"
-half = { "version" = "2.6.0", default-features = false, features = [
-    "num-traits",
+arrow = { version = "43.0.0", optional = false }
+arrow-array = "43.0"
+arrow-data = "43.0"
+arrow-ipc = "43.0"
+arrow-ord = "43.0"
+arrow-schema = "43.0"
+arrow-arith = "43.0"
+arrow-cast = "43.0"
+chrono = "0.4.23"
+half = { "version" = "=2.2.1", default-features = false, features = [
+    "num-traits"
 ] }
-futures = "0"
 log = "0.4"
-moka = { version = "0.12", features = ["future"] }
-object_store = "0.12.0"
-pin-project = "1.0.7"
-rand = "0.9"
-snafu = "0.8"
+object_store = "0.6.1"
+snafu = "0.7.4"
 url = "2"
-num-traits = "0.2"
-regex = "1.10"
-lazy_static = "1"
-semver = "1.0.25"
-chrono = "0.4"
-
-[profile.ci]
-debug = "line-tables-only"
-inherits = "dev"
-incremental = false
-
-# This rule applies to every package except workspace members (dependencies
-# such as `arrow` and `tokio`). It disables debug info and related features on
-# dependencies so their binaries stay smaller, improving cache reuse.
-[profile.ci.package."*"]
-debug = false
-debug-assertions = false
-strip = "debuginfo"
-incremental = false

README.md

@@ -1,97 +1,79 @@
-<a href="https://cloud.lancedb.com" target="_blank">
-  <img src="https://github.com/user-attachments/assets/92dad0a2-2a37-4ce1-b783-0d1b4f30a00c" alt="LanceDB Cloud Public Beta" width="100%" style="max-width: 100%;">
-</a>
 <div align="center">
+<p align="center">
 
-[![LanceDB](docs/src/assets/hero-header.png)](https://lancedb.com)
-[![Website](https://img.shields.io/badge/-Website-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://lancedb.com/)
-[![Blog](https://img.shields.io/badge/Blog-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://blog.lancedb.com/)
-[![Discord](https://img.shields.io/badge/-Discord-100000?style=for-the-badge&logo=discord&logoColor=white&labelColor=645cfb&color=645cfb)](https://discord.gg/zMM32dvNtd)
-[![Twitter](https://img.shields.io/badge/-Twitter-100000?style=for-the-badge&logo=x&logoColor=white&labelColor=645cfb&color=645cfb)](https://twitter.com/lancedb)
-[![LinkedIn](https://img.shields.io/badge/-LinkedIn-100000?style=for-the-badge&logo=linkedin&logoColor=white&labelColor=645cfb&color=645cfb)](https://www.linkedin.com/company/lancedb/)
+<img width="275" alt="LanceDB Logo" src="https://user-images.githubusercontent.com/917119/226205734-6063d87a-1ecc-45fe-85be-1dea6383a3d8.png">
 
+**Developer-friendly, serverless vector database for AI applications**
 
-<img src="docs/src/assets/lancedb.png" alt="LanceDB" width="50%">
+<a href="https://lancedb.github.io/lancedb/">Documentation</a> •
+<a href="https://blog.lancedb.com/">Blog</a> •
+<a href="https://discord.gg/zMM32dvNtd">Discord</a> •
+<a href="https://twitter.com/lancedb">Twitter</a>
 
-# **The Multimodal AI Lakehouse**
+</p>
 
-[**How to Install** ](#how-to-install) ✦ [**Detailed Documentation**](https://lancedb.com/docs) ✦ [**Tutorials and Recipes**](https://github.com/lancedb/vectordb-recipes/tree/main) ✦  [**Contributors**](#contributors) 
-
-**The ultimate multimodal data platform for AI/ML applications.** 
-
-LanceDB is designed for fast, scalable, and production-ready vector search. It is built on top of the Lance columnar format. You can store, index, and search over petabytes of multimodal data and vectors with ease. 
-LanceDB is a central location where developers can build, train and analyze their AI workloads.
-
-</div>
-
-<br>
-
-## **Demo: Multimodal Search by Keyword, Vector or with SQL**
 <img max-width="750px" alt="LanceDB Multimodal Search" src="https://github.com/lancedb/lancedb/assets/917119/09c5afc5-7816-4687-bae4-f2ca194426ec">
 
-## **Star LanceDB to get updates!**
-
-<details>
-<summary>⭐ Click here ⭐  to see how fast we're growing!</summary>
-<picture>
-  <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=lancedb/lancedb&theme=dark&type=Date">
-  <img width="100%" src="https://api.star-history.com/svg?repos=lancedb/lancedb&theme=dark&type=Date">
-</picture>
-</details>
-
-## **Key Features**:
-
-- **Fast Vector Search**: Search billions of vectors in milliseconds with state-of-the-art indexing.
-- **Comprehensive Search**: Support for vector similarity search, full-text search and SQL.
-- **Multimodal Support**: Store, query and filter vectors, metadata and multimodal data (text, images, videos, point clouds, and more).
-- **Advanced Features**: Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure. GPU support in building vector index.
-
-### **Products**:
-- **Open Source & Local**: 100% open source, runs locally or in your cloud. No vendor lock-in.
-- **Cloud and Enterprise**: Production-scale vector search with no servers to manage. Complete data sovereignty and security.
-
-### **Ecosystem**:
-- **Columnar Storage**: Built on the Lance columnar format for efficient storage and analytics.
-- **Seamless Integration**: Python, Node.js, Rust, and REST APIs for easy integration. Native Python and Javascript/Typescript support.
-- **Rich Ecosystem**: Integrations with [**LangChain** 🦜️🔗](https://python.langchain.com/docs/integrations/vectorstores/lancedb/), [**LlamaIndex** 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
-
-## **How to Install**:
-
-Follow the [Quickstart](https://lancedb.com/docs/quickstart/) doc to set up LanceDB locally. 
-
-**API & SDK:** We also support Python, Typescript and Rust SDKs
-
-| Interface | Documentation |
-|-----------|---------------|
-| Python SDK | https://lancedb.github.io/lancedb/python/python/ |
-| Typescript SDK | https://lancedb.github.io/lancedb/js/globals/ |
-| Rust SDK | https://docs.rs/lancedb/latest/lancedb/index.html |
-| REST API | https://docs.lancedb.com/api-reference/introduction |
-
-## **Join Us and Contribute**
-
-We welcome contributions from everyone! Whether you're a developer, researcher, or just someone who wants to help out. 
-
-If you have any suggestions or feature requests, please feel free to open an issue on GitHub or discuss it on our [**Discord**](https://discord.gg/G5DcmnZWKB) server.
-
-[**Check out the GitHub Issues**](https://github.com/lancedb/lancedb/issues) if you would like to work on the features that are planned for the future. If you have any suggestions or feature requests, please feel free to open an issue on GitHub. 
-
-## **Contributors**
-
-<a href="https://github.com/lancedb/lancedb/graphs/contributors">
-  <img src="https://contrib.rocks/image?repo=lancedb/lancedb" />
-</a>
-
-
-## **Stay in Touch With Us**
-<div align="center">
-
-</br>
-
-[![Website](https://img.shields.io/badge/-Website-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://lancedb.com/)
-[![Blog](https://img.shields.io/badge/Blog-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://blog.lancedb.com/)
-[![Discord](https://img.shields.io/badge/-Discord-100000?style=for-the-badge&logo=discord&logoColor=white&labelColor=645cfb&color=645cfb)](https://discord.gg/zMM32dvNtd)
-[![Twitter](https://img.shields.io/badge/-Twitter-100000?style=for-the-badge&logo=x&logoColor=white&labelColor=645cfb&color=645cfb)](https://twitter.com/lancedb)
-[![LinkedIn](https://img.shields.io/badge/-LinkedIn-100000?style=for-the-badge&logo=linkedin&logoColor=white&labelColor=645cfb&color=645cfb)](https://www.linkedin.com/company/lancedb/)
-
+</p>
 </div>
+
+<hr />
+
+LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrevial, filtering and management of embeddings.
+
+The key features of LanceDB include:
+
+* Production-scale vector search with no servers to manage.
+
+* Store, query and filter vectors, metadata and multi-modal data (text, images, videos, point clouds, and more).
+
+* Support for vector similarity search, full-text search and SQL.
+
+* Native Python and Javascript/Typescript support.
+
+* Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure.
+
+* GPU support in building vector index(*).
+
+* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lanecdb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
+
+LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
+
+## Quick Start
+
+**Javascript**
+```shell
+npm install vectordb
+```
+
+```javascript
+const lancedb = require('vectordb');
+const db = await lancedb.connect('data/sample-lancedb');
+
+const table = await db.createTable('vectors',
+      [{ id: 1, vector: [0.1, 0.2], item: "foo", price: 10 },
+       { id: 2, vector: [1.1, 1.2], item: "bar", price: 50 }])
+
+const query = table.search([0.1, 0.3]).limit(2);
+const results = await query.execute();
+```
+
+**Python**
+```shell
+pip install lancedb
+```
+
+```python
+import lancedb
+
+uri = "data/sample-lancedb"
+db = lancedb.connect(uri)
+table = db.create_table("my_table",
+                         data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                               {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+result = table.search([100, 100]).limit(2).to_pandas()
+```
+
+## Blogs, Tutorials & Videos
+* 📈 <a href="https://blog.eto.ai/benchmarking-random-access-in-lance-ed690757a826">2000x better performance with Lance over Parquet</a>
+* 🤖 <a href="https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb">Build a question and answer bot with LanceDB</a>

ci/build_linux_artifacts.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+ARCH=${1:-x86_64}
+
+# We pass down the current user so that when we later mount the local files 
+# into the container, the files are accessible by the current user.
+pushd ci/manylinux_node
+docker build \
+    -t lancedb-node-manylinux \
+    --build-arg="ARCH=$ARCH" \
+    --build-arg="DOCKER_USER=$(id -u)" \
+    --progress=plain \
+    .
+popd
+
+docker run \
+    -v $(pwd):/io -w /io \
+    lancedb-node-manylinux \
+    bash ci/manylinux_node/build.sh $ARCH

ci/build_macos_artifacts.sh

@@ -0,0 +1,33 @@
+# Builds the macOS artifacts (node binaries).
+# Usage: ./ci/build_macos_artifacts.sh [target]
+# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
+
+prebuild_rust() {
+    # Building here for the sake of easier debugging.
+    pushd rust/ffi/node
+    echo "Building rust library for $1"
+    export RUST_BACKTRACE=1
+    cargo build --release --target $1
+    popd
+}
+
+build_node_binaries() {
+    pushd node
+    echo "Building node library for $1"
+    npm run build-release -- --target $1
+    npm run pack-build -- --target $1
+    popd
+}
+
+if [ -n "$1" ]; then
+    targets=$1
+else
+    targets="x86_64-apple-darwin aarch64-apple-darwin"
+fi
+
+echo "Building artifacts for targets: $targets"
+for target in $targets
+    do
+    prebuild_rust $target
+    build_node_binaries $target
+done

ci/build_windows_artifacts.ps1

@@ -0,0 +1,41 @@
+# Builds the Windows artifacts (node binaries).
+# Usage:  .\ci\build_windows_artifacts.ps1 [target]
+# Targets supported:
+# - x86_64-pc-windows-msvc
+# - i686-pc-windows-msvc
+
+function Prebuild-Rust {
+    param (
+        [string]$target
+    )
+
+    # Building here for the sake of easier debugging.
+    Push-Location -Path "rust/ffi/node"
+    Write-Host "Building rust library for $target"
+    $env:RUST_BACKTRACE=1
+    cargo build --release --target $target
+    Pop-Location
+}
+
+function Build-NodeBinaries {
+    param (
+        [string]$target
+    )
+
+    Push-Location -Path "node"
+    Write-Host "Building node library for $target"
+    npm run build-release -- --target $target
+    npm run pack-build -- --target $target
+    Pop-Location
+}
+
+$targets = $args[0]
+if (-not $targets) {
+    $targets = "x86_64-pc-windows-msvc"
+}
+
+Write-Host "Building artifacts for targets: $targets"
+foreach ($target in $targets) {
+    Prebuild-Rust $target
+    Build-NodeBinaries $target
+}

ci/bump_version.sh

@@ -1,51 +0,0 @@
-set -e
-
-RELEASE_TYPE=${1:-"stable"}
-BUMP_MINOR=${2:-false}
-TAG_PREFIX=${3:-"v"} # Such as "python-v"
-HEAD_SHA=${4:-$(git rev-parse HEAD)}
-
-readonly SELF_DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-
-PREV_TAG=$(git tag --sort='version:refname' | grep ^$TAG_PREFIX | python $SELF_DIR/semver_sort.py $TAG_PREFIX | tail -n 1)
-echo "Found previous tag $PREV_TAG"
-
-# Initially, we don't want to tag if we are doing stable, because we will bump
-# again later. See comment at end for why.
-if [[ "$RELEASE_TYPE" == 'stable' ]]; then 
-  BUMP_ARGS="--no-tag"
-fi
-
-# If last is stable and not bumping minor
-if [[ $PREV_TAG != *beta* ]]; then
-    if [[ "$BUMP_MINOR" != "false" ]]; then
-      # X.Y.Z -> X.(Y+1).0-beta.0
-      bump-my-version bump -vv $BUMP_ARGS minor
-    else
-      # X.Y.Z -> X.Y.(Z+1)-beta.0
-      bump-my-version bump -vv $BUMP_ARGS patch
-    fi
-else
-  if [[ "$BUMP_MINOR" != "false" ]]; then
-    # X.Y.Z-beta.N -> X.(Y+1).0-beta.0
-    bump-my-version bump -vv $BUMP_ARGS minor
-  else
-    # X.Y.Z-beta.N -> X.Y.Z-beta.(N+1)
-    bump-my-version bump -vv $BUMP_ARGS pre_n
-  fi
-fi
-
-# The above bump will always bump to a pre-release version. If we are releasing
-# a stable version, bump the pre-release level ("pre_l") to make it stable.
-if [[ $RELEASE_TYPE == 'stable' ]]; then
-  # X.Y.Z-beta.N -> X.Y.Z
-  bump-my-version bump -vv pre_l
-fi
-
-# Validate that we have incremented version appropriately for breaking changes
-NEW_TAG=$(git describe --tags --exact-match HEAD)
-NEW_VERSION=$(echo $NEW_TAG | sed "s/^$TAG_PREFIX//")
-LAST_STABLE_RELEASE=$(git tag --sort='version:refname' | grep ^$TAG_PREFIX | grep -v beta | grep -vF "$NEW_TAG" | python $SELF_DIR/semver_sort.py $TAG_PREFIX | tail -n 1)
-LAST_STABLE_VERSION=$(echo $LAST_STABLE_RELEASE | sed "s/^$TAG_PREFIX//")
-
-python $SELF_DIR/check_breaking_changes.py $LAST_STABLE_RELEASE $HEAD_SHA $LAST_STABLE_VERSION $NEW_VERSION

ci/check_breaking_changes.py

@@ -1,35 +0,0 @@
-"""
-Check whether there are any breaking changes in the PRs between the base and head commits.
-If there are, assert that we have incremented the minor version.
-"""
-import argparse
-import os
-from packaging.version import parse
-
-from github import Github
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("base")
-    parser.add_argument("head")
-    parser.add_argument("last_stable_version")
-    parser.add_argument("current_version")
-    args = parser.parse_args()
-
-    repo = Github(os.environ["GITHUB_TOKEN"]).get_repo(os.environ["GITHUB_REPOSITORY"])
-    commits = repo.compare(args.base, args.head).commits
-    prs = (pr for commit in commits for pr in commit.get_pulls())
-
-    for pr in prs:
-        if any(label.name == "breaking-change" for label in pr.labels):
-            print(f"Breaking change in PR: {pr.html_url}")
-            break
-    else:
-        print("No breaking changes found.")
-        exit(0)
-    
-    last_stable_version = parse(args.last_stable_version)
-    current_version = parse(args.current_version)
-    if current_version.minor <= last_stable_version.minor:
-        print("Minor version is not greater than the last stable version.")
-        exit(1)

ci/check_lance_release.py

@@ -1,208 +0,0 @@
-#!/usr/bin/env python3
-"""Determine whether a newer Lance tag exists and expose results for CI."""
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import re
-import subprocess
-import sys
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Iterable, List, Sequence, Tuple, Union
-
-try:  # Python >=3.11
-    import tomllib  # type: ignore
-except ModuleNotFoundError:  # pragma: no cover - fallback for older Python
-    import tomli as tomllib  # type: ignore
-
-LANCE_REPO = "lance-format/lance"
-
-SEMVER_RE = re.compile(
-    r"^\s*(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)"
-    r"(?:-(?P<prerelease>[0-9A-Za-z.-]+))?"
-    r"(?:\+[0-9A-Za-z.-]+)?\s*$"
-)
-
-
-@dataclass(frozen=True)
-class SemVer:
-    major: int
-    minor: int
-    patch: int
-    prerelease: Tuple[Union[int, str], ...]
-
-    def __lt__(self, other: "SemVer") -> bool:  # pragma: no cover - simple comparison
-        if (self.major, self.minor, self.patch) != (other.major, other.minor, other.patch):
-            return (self.major, self.minor, self.patch) < (other.major, other.minor, other.patch)
-        if self.prerelease == other.prerelease:
-            return False
-        if not self.prerelease:
-            return False  # release > anything else
-        if not other.prerelease:
-            return True
-        for left, right in zip(self.prerelease, other.prerelease):
-            if left == right:
-                continue
-            if isinstance(left, int) and isinstance(right, int):
-                return left < right
-            if isinstance(left, int):
-                return True
-            if isinstance(right, int):
-                return False
-            return str(left) < str(right)
-        return len(self.prerelease) < len(other.prerelease)
-
-    def __eq__(self, other: object) -> bool:  # pragma: no cover - trivial
-        if not isinstance(other, SemVer):
-            return NotImplemented
-        return (
-            self.major == other.major
-            and self.minor == other.minor
-            and self.patch == other.patch
-            and self.prerelease == other.prerelease
-        )
-
-
-def parse_semver(raw: str) -> SemVer:
-    match = SEMVER_RE.match(raw)
-    if not match:
-        raise ValueError(f"Unsupported version format: {raw}")
-    prerelease = match.group("prerelease")
-    parts: Tuple[Union[int, str], ...] = ()
-    if prerelease:
-        parsed: List[Union[int, str]] = []
-        for piece in prerelease.split("."):
-            if piece.isdigit():
-                parsed.append(int(piece))
-            else:
-                parsed.append(piece)
-        parts = tuple(parsed)
-    return SemVer(
-        major=int(match.group("major")),
-        minor=int(match.group("minor")),
-        patch=int(match.group("patch")),
-        prerelease=parts,
-    )
-
-
-@dataclass
-class TagInfo:
-    tag: str  # e.g. v1.0.0-beta.2
-    version: str  # e.g. 1.0.0-beta.2
-    semver: SemVer
-
-
-def run_command(cmd: Sequence[str]) -> str:
-    result = subprocess.run(cmd, capture_output=True, text=True, check=False)
-    if result.returncode != 0:
-        raise RuntimeError(
-            f"Command {' '.join(cmd)} failed with {result.returncode}: {result.stderr.strip()}"
-        )
-    return result.stdout.strip()
-
-
-def fetch_remote_tags() -> List[TagInfo]:
-    output = run_command(
-        [
-            "gh",
-            "api",
-            "-X",
-            "GET",
-            f"repos/{LANCE_REPO}/git/refs/tags",
-            "--paginate",
-            "--jq",
-            ".[].ref",
-        ]
-    )
-    tags: List[TagInfo] = []
-    for line in output.splitlines():
-        ref = line.strip()
-        if not ref.startswith("refs/tags/v"):
-            continue
-        tag = ref.split("refs/tags/")[-1]
-        version = tag.lstrip("v")
-        try:
-            tags.append(TagInfo(tag=tag, version=version, semver=parse_semver(version)))
-        except ValueError:
-            continue
-    if not tags:
-        raise RuntimeError("No Lance tags could be parsed from GitHub API output")
-    return tags
-
-
-def read_current_version(repo_root: Path) -> str:
-    cargo_path = repo_root / "Cargo.toml"
-    with cargo_path.open("rb") as fh:
-        data = tomllib.load(fh)
-    try:
-        deps = data["workspace"]["dependencies"]
-        entry = deps["lance"]
-    except KeyError as exc:  # pragma: no cover - configuration guard
-        raise RuntimeError("Failed to locate workspace.dependencies.lance in Cargo.toml") from exc
-
-    if isinstance(entry, str):
-        raw_version = entry
-    elif isinstance(entry, dict):
-        raw_version = entry.get("version", "")
-    else:  # pragma: no cover - defensive
-        raise RuntimeError("Unexpected lance dependency format")
-
-    raw_version = raw_version.strip()
-    if not raw_version:
-        raise RuntimeError("lance dependency does not declare a version")
-    return raw_version.lstrip("=")
-
-
-def determine_latest_tag(tags: Iterable[TagInfo]) -> TagInfo:
-    return max(tags, key=lambda tag: tag.semver)
-
-
-def write_outputs(args: argparse.Namespace, payload: dict) -> None:
-    target = getattr(args, "github_output", None)
-    if not target:
-        return
-    with open(target, "a", encoding="utf-8") as handle:
-        for key, value in payload.items():
-            handle.write(f"{key}={value}\n")
-
-
-def main(argv: Sequence[str] | None = None) -> int:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument(
-        "--repo-root",
-        default=Path(__file__).resolve().parents[1],
-        type=Path,
-        help="Path to the lancedb repository root",
-    )
-    parser.add_argument(
-        "--github-output",
-        default=os.environ.get("GITHUB_OUTPUT"),
-        help="Optional file path for writing GitHub Action outputs",
-    )
-    args = parser.parse_args(argv)
-
-    repo_root = Path(args.repo_root)
-    current_version = read_current_version(repo_root)
-    current_semver = parse_semver(current_version)
-
-    tags = fetch_remote_tags()
-    latest = determine_latest_tag(tags)
-    needs_update = latest.semver > current_semver
-
-    payload = {
-        "current_version": current_version,
-        "current_tag": f"v{current_version}",
-        "latest_version": latest.version,
-        "latest_tag": latest.tag,
-        "needs_update": "true" if needs_update else "false",
-    }
-
-    print(json.dumps(payload))
-    write_outputs(args, payload)
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

ci/create_lancedb_test_connection.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-export RUST_LOG=info
-exec ./lancedb server --port 0 --sql-port 0  --data-dir "${1}"

ci/manylinux_node/Dockerfile

@@ -0,0 +1,31 @@
+# Many linux dockerfile with Rust, Node, and Lance dependencies installed.
+# This container allows building the node modules native libraries in an 
+# environment with a very old glibc, so that we are compatible with a wide
+# range of linux distributions.
+ARG ARCH=x86_64
+
+FROM quay.io/pypa/manylinux2014_${ARCH}
+
+ARG ARCH=x86_64
+ARG DOCKER_USER=default_user
+
+# Install static openssl
+COPY install_openssl.sh install_openssl.sh
+RUN ./install_openssl.sh ${ARCH} > /dev/null
+
+# Protobuf is also installed as root.
+COPY install_protobuf.sh install_protobuf.sh
+RUN ./install_protobuf.sh ${ARCH}
+
+ENV DOCKER_USER=${DOCKER_USER}
+# Create a group and user
+RUN echo ${ARCH} && adduser --user-group --create-home --uid ${DOCKER_USER} build_user
+
+# We switch to the user to install Rust and Node, since those like to be 
+# installed at the user level.
+USER ${DOCKER_USER}
+
+COPY prepare_manylinux_node.sh prepare_manylinux_node.sh
+RUN cp /prepare_manylinux_node.sh $HOME/ && \
+    cd $HOME && \
+    ./prepare_manylinux_node.sh ${ARCH}

ci/manylinux_node/build.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Builds the node module for manylinux. Invoked by ci/build_linux_artifacts.sh.
+set -e
+ARCH=${1:-x86_64}
+
+if [ "$ARCH" = "x86_64" ]; then
+    export OPENSSL_LIB_DIR=/usr/local/lib64/
+else 
+    export OPENSSL_LIB_DIR=/usr/local/lib/
+fi
+export OPENSSL_STATIC=1
+export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
+
+source $HOME/.bashrc
+
+cd node
+npm ci
+npm run build-release
+npm run pack-build

ci/manylinux_node/install_openssl.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Builds openssl from source so we can statically link to it
+
+# this is to avoid the error we get with the system installation:
+# /usr/bin/ld: <library>: version node not found for symbol SSLeay@@OPENSSL_1.0.1
+# /usr/bin/ld: failed to set dynamic section sizes: Bad value
+set -e
+
+git clone -b OpenSSL_1_1_1u \
+    --single-branch \
+    https://github.com/openssl/openssl.git
+
+pushd openssl
+
+if [[ $1 == x86_64* ]]; then
+    ARCH=linux-x86_64
+else
+    # gnu target
+    ARCH=linux-aarch64
+fi
+
+./Configure no-shared $ARCH
+
+make
+
+make install

ci/manylinux_node/install_protobuf.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Installs protobuf compiler. Should be run as root.
+set -e
+
+if [[ $1 == x86_64* ]]; then
+    ARCH=x86_64
+else
+    # gnu target
+    ARCH=aarch_64
+fi
+
+PB_REL=https://github.com/protocolbuffers/protobuf/releases
+PB_VERSION=23.1
+curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
+unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local

ci/manylinux_node/prepare_manylinux_node.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+
+install_node() {
+    echo "Installing node..."
+
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+
+    source "$HOME"/.bashrc
+
+    nvm install --no-progress 16
+}
+
+install_rust() {
+    echo "Installing rust..."
+    curl https://sh.rustup.rs -sSf | bash -s -- -y
+    export PATH="$PATH:/root/.cargo/bin"
+}
+
+install_node
+install_rust

ci/mock_openai.py

@@ -1,57 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-"""A zero-dependency mock OpenAI embeddings API endpoint for testing purposes."""
-import argparse
-import json
-import http.server
-
-
-class MockOpenAIRequestHandler(http.server.BaseHTTPRequestHandler):
-    def do_POST(self):
-        content_length = int(self.headers["Content-Length"])
-        post_data = self.rfile.read(content_length)
-        post_data = json.loads(post_data.decode("utf-8"))
-        # See: https://platform.openai.com/docs/api-reference/embeddings/create
-
-        if isinstance(post_data["input"], str):
-            num_inputs = 1
-        else:
-            num_inputs = len(post_data["input"])
-
-        model = post_data.get("model", "text-embedding-ada-002")
-
-        data = []
-        for i in range(num_inputs):
-            data.append({
-                "object": "embedding",
-                "embedding": [0.1] * 1536,
-                "index": i,
-            })
-
-        response = {
-            "object": "list",
-            "data": data,
-            "model": model,
-            "usage": {
-                "prompt_tokens": 0,
-                "total_tokens": 0,
-            }
-        }
-
-        self.send_response(200)
-        self.send_header("Content-type", "application/json")
-        self.end_headers()
-        self.wfile.write(json.dumps(response).encode("utf-8"))
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Mock OpenAI embeddings API endpoint")
-    parser.add_argument("--port", type=int, default=8000, help="Port to listen on")
-    args = parser.parse_args()
-    port = args.port
-
-    print(f"server started on port {port}. Press Ctrl-C to stop.")
-    print(f"To use, set OPENAI_BASE_URL=http://localhost:{port} in your environment.")
-
-    with http.server.HTTPServer(("0.0.0.0", port), MockOpenAIRequestHandler) as server:
-        server.serve_forever()

ci/parse_requirements.py

@@ -1,41 +0,0 @@
-import argparse
-import toml
-
-
-def parse_dependencies(pyproject_path, extras=None):
-    with open(pyproject_path, "r") as file:
-        pyproject = toml.load(file)
-
-    dependencies = pyproject.get("project", {}).get("dependencies", [])
-    for dependency in dependencies:
-        print(dependency)
-
-    optional_dependencies = pyproject.get("project", {}).get(
-        "optional-dependencies", {}
-    )
-
-    if extras:
-        for extra in extras.split(","):
-            for dep in optional_dependencies.get(extra, []):
-                print(dep)
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Generate requirements.txt from pyproject.toml"
-    )
-    parser.add_argument("path", type=str, help="Path to pyproject.toml")
-    parser.add_argument(
-        "--extras",
-        type=str,
-        help="Comma-separated list of extras to include",
-        default="",
-    )
-
-    args = parser.parse_args()
-
-    parse_dependencies(args.path, args.extras)
-
-
-if __name__ == "__main__":
-    main()

ci/run_with_docker_compose.sh

@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# A script for running the given command together with a docker compose environment.
-#
-
-# Bring down the docker setup once the command is done running.
-tear_down() {
-    docker compose -p fixture down
-}
-trap tear_down EXIT
-
-set +xe
-
-# Clean up any existing docker setup and bring up a new one.
-docker compose -p fixture up --detach --wait || exit 1
-
-"${@}"

ci/run_with_test_connection.sh

@@ -1,68 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# A script for running the given command together with the lancedb cli.
-#
-
-die() {
-    echo $?
-    exit 1
-}
-
-check_command_exists() {
-    command="${1}"
-    which ${command} &> /dev/null || \
-        die "Unable to locate command: ${command}. Did you install it?"
-}
-
-if [[ ! -e ./lancedb ]]; then
-    if [[ -v SOPHON_READ_TOKEN ]]; then
-        INPUT="lancedb-linux-x64"
-        gh release \
-            --repo lancedb/lancedb \
-            download ci-support-binaries \
-            --pattern "${INPUT}" \
-            || die "failed to fetch cli."
-        check_command_exists openssl
-        openssl enc -aes-256-cbc \
-            -d -pbkdf2 \
-            -pass "env:SOPHON_READ_TOKEN" \
-            -in "${INPUT}" \
-            -out ./lancedb-linux-x64.tar.gz \
-            || die "openssl failed"
-        TARGET="${INPUT}.tar.gz"
-    else
-        ARCH="x64"
-        if [[ $OSTYPE == 'darwin'* ]]; then
-            UNAME=$(uname -m)
-            if [[ $UNAME == 'arm64' ]]; then
-                ARCH='arm64'
-            fi
-            OSTYPE="macos"
-        elif [[ $OSTYPE == 'linux'* ]]; then
-            if [[ $UNAME == 'aarch64' ]]; then
-                ARCH='arm64'
-            fi
-            OSTYPE="linux"
-        else
-            die "unknown OSTYPE: $OSTYPE"
-        fi
-
-        check_command_exists gh
-        TARGET="lancedb-${OSTYPE}-${ARCH}.tar.gz"
-        gh release \
-            --repo lancedb/sophon \
-            download lancedb-cli-v0.0.3 \
-            --pattern "${TARGET}" \
-            || die "failed to fetch cli."
-    fi
-
-    check_command_exists tar
-    tar xvf "${TARGET}" || die "tar failed."
-    [[ -e ./lancedb ]] || die "failed to extract lancedb."
-fi
-
-SCRIPT_DIR=$(dirname "$(readlink -f "$0")")
-export CREATE_LANCEDB_TEST_CONNECTION_SCRIPT="${SCRIPT_DIR}/create_lancedb_test_connection.sh"
-
-"${@}"

ci/semver_sort.py

@@ -1,35 +0,0 @@
-"""
-Takes a list of semver strings and sorts them in ascending order.
-"""
-
-import sys
-from packaging.version import parse, InvalidVersion
-
-if __name__ == "__main__":
-    import argparse
-    parser = argparse.ArgumentParser()
-    parser.add_argument("prefix", default="v")
-    args = parser.parse_args()
-
-    # Read the input from stdin
-    lines = sys.stdin.readlines()
-
-    # Parse the versions
-    versions = []
-    for line in lines:
-        line = line.strip()
-        try:
-            version_str = line.removeprefix(args.prefix)
-            version = parse(version_str)
-        except InvalidVersion:
-            # There are old tags that don't follow the semver format
-            print(f"Invalid version: {line}", file=sys.stderr)
-            continue
-        versions.append((line, version))
-
-    # Sort the versions
-    versions.sort(key=lambda x: x[1])
-
-    # Print the sorted versions as original strings
-    for line, _ in versions:
-        print(line)

ci/set_lance_version.py

@@ -1,294 +0,0 @@
-import argparse
-import re
-import sys
-import json
-
-LANCE_GIT_URL = "https://github.com/lance-format/lance.git"
-
-
-def run_command(command: str) -> str:
-    """
-    Run a shell command and return stdout as a string.
-    If exit code is not 0, raise an exception with the stderr output.
-    """
-    import subprocess
-
-    result = subprocess.run(command, shell=True, capture_output=True, text=True)
-    if result.returncode != 0:
-        raise Exception(f"Command failed with error: {result.stderr.strip()}")
-    return result.stdout.strip()
-
-
-def get_latest_stable_version() -> str:
-    version_line = run_command("cargo info lance | grep '^version:'")
-    # Example output: "version: 0.35.0 (latest 0.37.0)"
-    match = re.search(r'\(latest ([0-9.]+)\)', version_line)
-    if match:
-        return match.group(1)
-    # Fallback: use the first version after 'version:'
-    return version_line.split("version:")[1].split()[0].strip()
-
-
-def get_latest_preview_version() -> str:
-    lance_tags = run_command(
-        f"git ls-remote --tags {LANCE_GIT_URL} | grep 'refs/tags/v[0-9beta.-]\\+$'"
-    ).splitlines()
-    lance_tags = (
-        tag.split("refs/tags/")[1]
-        for tag in lance_tags
-        if "refs/tags/" in tag and "beta" in tag
-    )
-    from packaging.version import Version
-
-    latest = max(
-        (tag[1:] for tag in lance_tags if tag.startswith("v")), key=lambda t: Version(t)
-    )
-    return str(latest)
-
-
-def extract_features(line: str) -> list:
-    """
-    Extracts the features from a line in Cargo.toml.
-    Example: 'lance = { "version" = "=0.29.0", "features" = ["dynamodb"] }'
-    Returns: ['dynamodb']
-    """
-    import re
-
-    match = re.search(r'"features"\s*=\s*\[\s*(.*?)\s*\]', line, re.DOTALL)
-    if match:
-        features_str = match.group(1)
-        return [f.strip().strip('"') for f in features_str.split(",") if f.strip()]
-    return []
-
-
-def extract_default_features(line: str) -> bool:
-    """
-    Checks if default-features = false is present in a line in Cargo.toml.
-    Example: 'lance = { "version" = "=0.29.0", default-features = false, "features" = ["dynamodb"] }'
-    Returns: True if default-features = false is present, False otherwise
-    """
-    import re
-
-    match = re.search(r'default-features\s*=\s*false', line)
-    return match is not None
-
-
-def dict_to_toml_line(package_name: str, config: dict) -> str:
-    """
-    Converts a configuration dictionary to a TOML dependency line.
-    Dictionary insertion order is preserved (Python 3.7+), so the caller
-    controls the order of fields in the output.
-
-    Args:
-        package_name: The name of the package (e.g., "lance", "lance-io")
-        config: Dictionary with keys like "version", "path", "git", "tag", "features", "default-features"
-                The order of keys in this dict determines the order in the output.
-
-    Returns:
-        A properly formatted TOML line with a trailing newline
-    """
-    # If only version is specified, use simple format
-    if len(config) == 1 and "version" in config:
-        return f'{package_name} = "{config["version"]}"\n'
-
-    # Otherwise, use inline table format
-    parts = []
-    for key, value in config.items():
-        if key == "default-features" and not value:
-            parts.append("default-features = false")
-        elif key == "features":
-            parts.append(f'"features" = {json.dumps(value)}')
-        elif isinstance(value, str):
-            parts.append(f'"{key}" = "{value}"')
-        else:
-            # This shouldn't happen with our current usage
-            parts.append(f'"{key}" = {json.dumps(value)}')
-
-    return f'{package_name} = {{ {", ".join(parts)} }}\n'
-
-
-def update_cargo_toml(line_updater):
-    """
-    Updates the Cargo.toml file by applying the line_updater function to each line.
-    The line_updater function should take a line as input and return the updated line.
-    """
-    with open("Cargo.toml", "r") as f:
-        lines = f.readlines()
-
-    new_lines = []
-    lance_line = ""
-    is_parsing_lance_line = False
-    for line in lines:
-        if line.startswith("lance"):
-            # Check if this is a single-line or multi-line entry
-            # Single-line entries either:
-            # 1. End with } (complete inline table)
-            # 2. End with " (simple version string)
-            # Multi-line entries start with { but don't end with }
-            if line.strip().endswith("}") or line.strip().endswith('"'):
-                # Single-line entry - process immediately
-                new_lines.append(line_updater(line))
-            elif "{" in line and not line.strip().endswith("}"):
-                # Multi-line entry - start accumulating
-                lance_line = line
-                is_parsing_lance_line = True
-            else:
-                # Single-line entry without quotes or braces (shouldn't happen but handle it)
-                new_lines.append(line_updater(line))
-        elif is_parsing_lance_line:
-            lance_line += line
-            if line.strip().endswith("}"):
-                new_lines.append(line_updater(lance_line))
-                lance_line = ""
-                is_parsing_lance_line = False
-        else:
-            # Keep the line unchanged
-            new_lines.append(line)
-
-    with open("Cargo.toml", "w") as f:
-        f.writelines(new_lines)
-
-
-def set_stable_version(version: str):
-    """
-    Sets lines to
-    lance = { "version" = "=0.29.0", default-features = false, "features" = ["dynamodb"] }
-    lance-io = { "version" = "=0.29.0", default-features = false }
-    ...
-    """
-
-    def line_updater(line: str) -> str:
-        package_name = line.split("=", maxsplit=1)[0].strip()
-
-        # Build config in desired order: version, default-features, features
-        config = {"version": f"={version}"}
-
-        if extract_default_features(line):
-            config["default-features"] = False
-
-        features = extract_features(line)
-        if features:
-            config["features"] = features
-
-        return dict_to_toml_line(package_name, config)
-
-    update_cargo_toml(line_updater)
-
-
-def set_preview_version(version: str):
-    """
-    Sets lines to
-    lance = { "version" = "=0.29.0", default-features = false, "features" = ["dynamodb"], "tag" = "v0.29.0-beta.2", "git" = LANCE_GIT_URL }
-    lance-io = { "version" = "=0.29.0", default-features = false, "tag" = "v0.29.0-beta.2", "git" = LANCE_GIT_URL }
-    ...
-    """
-
-    def line_updater(line: str) -> str:
-        package_name = line.split("=", maxsplit=1)[0].strip()
-        # Build config in desired order: version, default-features, features, tag, git
-        config = {"version": f"={version}"}
-
-        if extract_default_features(line):
-            config["default-features"] = False
-
-        features = extract_features(line)
-        if features:
-            config["features"] = features
-
-        config["tag"] = f"v{version}"
-        config["git"] = LANCE_GIT_URL
-
-        return dict_to_toml_line(package_name, config)
-
-    update_cargo_toml(line_updater)
-
-
-def set_local_version():
-    """
-    Sets lines to
-    lance = { "path" = "../lance/rust/lance", default-features = false, "features" = ["dynamodb"] }
-    lance-io = { "path" = "../lance/rust/lance-io", default-features = false }
-    ...
-    """
-
-    def line_updater(line: str) -> str:
-        package_name = line.split("=", maxsplit=1)[0].strip()
-
-        # Build config in desired order: path, default-features, features
-        config = {"path": f"../lance/rust/{package_name}"}
-
-        if extract_default_features(line):
-            config["default-features"] = False
-
-        features = extract_features(line)
-        if features:
-            config["features"] = features
-
-        return dict_to_toml_line(package_name, config)
-
-    update_cargo_toml(line_updater)
-
-
-def update_lockfiles(version: str, fallback_to_git: bool = False):
-    """
-    Update Cargo metadata and optionally fall back to using the git tag if the
-    requested crates.io version is unavailable.
-    """
-    try:
-        print("Updating lockfiles...", file=sys.stderr, end="")
-        run_command("cargo metadata > /dev/null")
-        print(" done.", file=sys.stderr)
-    except Exception as e:
-        if fallback_to_git and "failed to select a version" in str(e):
-            print(
-                f" failed for crates.io v{version}, retrying with git tag...",
-                file=sys.stderr,
-            )
-            set_preview_version(version)
-            print("Updating lockfiles...", file=sys.stderr, end="")
-            run_command("cargo metadata > /dev/null")
-            print(" done.", file=sys.stderr)
-        else:
-            raise
-
-
-parser = argparse.ArgumentParser(description="Set the version of the Lance package.")
-parser.add_argument(
-    "version",
-    type=str,
-    help="The version to set for the Lance package. Use 'stable' for the latest stable version, 'preview' for latest preview version, or a specific version number (e.g., '0.1.0'). You can also specify 'local' to use a local path.",
-)
-args = parser.parse_args()
-
-if args.version == "stable":
-    latest_stable_version = get_latest_stable_version()
-    print(
-        f"Found latest stable version: \033[1mv{latest_stable_version}\033[0m",
-        file=sys.stderr,
-    )
-    set_stable_version(latest_stable_version)
-    update_lockfiles(latest_stable_version)
-elif args.version == "preview":
-    latest_preview_version = get_latest_preview_version()
-    print(
-        f"Found latest preview version: \033[1mv{latest_preview_version}\033[0m",
-        file=sys.stderr,
-    )
-    set_preview_version(latest_preview_version)
-    update_lockfiles(latest_preview_version)
-elif args.version == "local":
-    set_local_version()
-    update_lockfiles("local")
-else:
-    # Parse the version number.
-    version = args.version
-    # Ignore initial v if present.
-    if version.startswith("v"):
-        version = version[1:]
-
-    if "beta" in version:
-        set_preview_version(version)
-        update_lockfiles(version)
-    else:
-        set_stable_version(version)
-        update_lockfiles(version, fallback_to_git=True)

ci/sysroot-aarch64-pc-windows-msvc.sh

@@ -1,105 +0,0 @@
-#!/bin/sh
-
-# https://github.com/mstorsjo/msvc-wine/blob/master/vsdownload.py
-# https://github.com/mozilla/gecko-dev/blob/6027d1d91f2d3204a3992633b3ef730ff005fc64/build/vs/vs2022-car.yaml
-
-# function dl() {
-# 	curl -O https://download.visualstudio.microsoft.com/download/pr/$1
-# }
-
-# [[.h]]
-
-# "id": "Win11SDK_10.0.26100"
-# "version": "10.0.26100.7"
-
-# libucrt.lib
-
-# example: <assert.h>
-# dir: ucrt/
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ee3a5fc6e9fc832af7295b138e93839/universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b1aa09b90fe314aceb090f6ec7626624/16ab2ea2187acffa6435e334796c8c89.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/400609bb0ff5804e36dbe6dcd42a7f01/6ee7bbee8435130a869cf971694fd9e2.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ac327317abb865a0e3f56b2faefa918/78fa3c824c2c48bd4a49ab5969adaaf7.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/f034bc0b2680f67dccd4bfeea3d0f932/7afc7b670accd8e3cc94cfffd516f5cb.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7ed5e12f9d50f80825a8b27838cf4c7f/96076045170fe5db6d5dcf14b6f6688e.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/764edc185a696bda9e07df8891dddbbb/a1e2a83aa8a71c48c742eeaff6e71928.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/66854bedc6dbd5ccb5dd82c8e2412231/b2f03f34ff83ec013b9e45c7cd8e8a73.cab
-
-# example: <windows.h>
-# dir: um/
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b286efac4d83a54fc49190bddef1edc9/windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/e0dc3811d92ab96fcb72bf63d6c08d71/766c0ffd568bbb31bf7fb6793383e24a.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/613503da4b5628768497822826aed39f/8125ee239710f33ea485965f76fae646.cab
-
-# example: <winapifamily.h>
-# dir: /shared
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/122979f0348d3a2a36b6aa1a111d5d0c/windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/766e04beecdfccff39e91dd9eb32834a/e89e3dcbb016928c7e426238337d69eb.cab
-
-
-# "id": "Microsoft.VisualC.14.16.CRT.Headers"
-# "version": "14.16.27045"
-
-# example: <vcruntime.h>
-# dir: MSVC/
-curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/87bbe41e09a2f83711e72696f49681429327eb7a4b90618c35667a6ba2e2880e/Microsoft.VisualC.14.16.CRT.Headers.vsix
-
-# [[.lib]]
-
-# advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib runtimeobject.lib
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/944c4153b849a1f7d0c0404a4f1c05ea/windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5306aed3e1a38d1e8bef5934edeb2a9b/05047a45609f311645eebcac2739fc4c.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/13c8a73a0f5a6474040b26d016a26fab/13d68b8a7b6678a368e2d13ff4027521.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/149578fb3b621cdb61ee1813b9b3e791/463ad1b0783ebda908fd6c16a4abfe93.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5c986c4f393c6b09d5aec3b539e9fb4a/5a22e5cde814b041749fb271547f4dd5.cab
-
-# dbghelp.lib fwpuclnt.lib arm64rt.lib
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7a332420d812f7c1d41da865ae5a7c52/windows%20sdk%20desktop%20libs%20arm64-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/19de98ed4a79938d0045d19c047936b3/3e2f7be479e3679d700ce0782e4cc318.cab
-
-# libcmt.lib libvcruntime.lib
-curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/227f40682a88dc5fa0ccb9cadc9ad30af99ad1f1a75db63407587d079f60d035/Microsoft.VisualC.14.16.CRT.ARM64.Desktop.vsix
-
-
-msiextract universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
-msiextract windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
-msiextract windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
-msiextract windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
-msiextract windows%20sdk%20desktop%20libs%20arm64-x86_en-us.msi
-unzip -o Microsoft.VisualC.14.16.CRT.Headers.vsix
-unzip -o Microsoft.VisualC.14.16.CRT.ARM64.Desktop.vsix
-
-mkdir -p /usr/aarch64-pc-windows-msvc/usr/include
-mkdir -p /usr/aarch64-pc-windows-msvc/usr/lib
-
-# lowercase folder/file names
-echo "$(find . -regex ".*/[^/]*[A-Z][^/]*")" | xargs -I{} sh -c 'mv "$(echo "{}" | sed -E '"'"'s/(.*\/)/\L\1/'"'"')" "$(echo "{}" | tr [A-Z] [a-z])"'
-
-# .h
-(cd 'program files/windows kits/10/include/10.0.26100.0' && cp -r ucrt/* um/* shared/* -t /usr/aarch64-pc-windows-msvc/usr/include)
-
-cp -r contents/vc/tools/msvc/14.16.27023/include/* /usr/aarch64-pc-windows-msvc/usr/include
-
-# lowercase #include "" and #include <>
-find /usr/aarch64-pc-windows-msvc/usr/include -type f -exec sed -i -E 's/(#include <[^<>]*?[A-Z][^<>]*?>)|(#include "[^"]*?[A-Z][^"]*?")/\L\1\2/' "{}" ';'
-
-# ARM intrinsics
-# original dir: MSVC/
-
-# '__n128x4' redefined in arm_neon.h
-# "arm64_neon.h" included from intrin.h
-
-(cd /usr/lib/llvm19/lib/clang/19/include && cp arm_neon.h intrin.h -t /usr/aarch64-pc-windows-msvc/usr/include)
-
-# .lib
-
-# _Interlocked intrinsics
-# must always link with arm64rt.lib
-# reason: https://developercommunity.visualstudio.com/t/libucrtlibstreamobj-error-lnk2001-unresolved-exter/1544787#T-ND1599818
-# I don't understand the 'correct' fix for this, arm64rt.lib is supposed to be the workaround
-
-(cd 'program files/windows kits/10/lib/10.0.26100.0/um/arm64' && cp advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib runtimeobject.lib dbghelp.lib fwpuclnt.lib arm64rt.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
-
-(cd 'contents/vc/tools/msvc/14.16.27023/lib/arm64' && cp libcmt.lib libvcruntime.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
-
-cp 'program files/windows kits/10/lib/10.0.26100.0/ucrt/arm64/libucrt.lib' /usr/aarch64-pc-windows-msvc/usr/lib

ci/sysroot-x86_64-pc-windows-msvc.sh

@@ -1,105 +0,0 @@
-#!/bin/sh
-
-# https://github.com/mstorsjo/msvc-wine/blob/master/vsdownload.py
-# https://github.com/mozilla/gecko-dev/blob/6027d1d91f2d3204a3992633b3ef730ff005fc64/build/vs/vs2022-car.yaml
-
-# function dl() {
-# 	curl -O https://download.visualstudio.microsoft.com/download/pr/$1
-# }
-
-# [[.h]]
-
-# "id": "Win11SDK_10.0.26100"
-# "version": "10.0.26100.7"
-
-# libucrt.lib
-
-# example: <assert.h>
-# dir: ucrt/
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ee3a5fc6e9fc832af7295b138e93839/universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b1aa09b90fe314aceb090f6ec7626624/16ab2ea2187acffa6435e334796c8c89.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/400609bb0ff5804e36dbe6dcd42a7f01/6ee7bbee8435130a869cf971694fd9e2.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ac327317abb865a0e3f56b2faefa918/78fa3c824c2c48bd4a49ab5969adaaf7.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/f034bc0b2680f67dccd4bfeea3d0f932/7afc7b670accd8e3cc94cfffd516f5cb.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7ed5e12f9d50f80825a8b27838cf4c7f/96076045170fe5db6d5dcf14b6f6688e.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/764edc185a696bda9e07df8891dddbbb/a1e2a83aa8a71c48c742eeaff6e71928.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/66854bedc6dbd5ccb5dd82c8e2412231/b2f03f34ff83ec013b9e45c7cd8e8a73.cab
-
-# example: <windows.h>
-# dir: um/
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b286efac4d83a54fc49190bddef1edc9/windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/e0dc3811d92ab96fcb72bf63d6c08d71/766c0ffd568bbb31bf7fb6793383e24a.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/613503da4b5628768497822826aed39f/8125ee239710f33ea485965f76fae646.cab
-
-# example: <winapifamily.h>
-# dir: /shared
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/122979f0348d3a2a36b6aa1a111d5d0c/windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/766e04beecdfccff39e91dd9eb32834a/e89e3dcbb016928c7e426238337d69eb.cab
-
-
-# "id": "Microsoft.VisualC.14.16.CRT.Headers"
-# "version": "14.16.27045"
-
-# example: <vcruntime.h>
-# dir: MSVC/
-curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/87bbe41e09a2f83711e72696f49681429327eb7a4b90618c35667a6ba2e2880e/Microsoft.VisualC.14.16.CRT.Headers.vsix
-
-# [[.lib]]
-
-# advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/944c4153b849a1f7d0c0404a4f1c05ea/windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5306aed3e1a38d1e8bef5934edeb2a9b/05047a45609f311645eebcac2739fc4c.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/13c8a73a0f5a6474040b26d016a26fab/13d68b8a7b6678a368e2d13ff4027521.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/149578fb3b621cdb61ee1813b9b3e791/463ad1b0783ebda908fd6c16a4abfe93.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5c986c4f393c6b09d5aec3b539e9fb4a/5a22e5cde814b041749fb271547f4dd5.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/bfc3904a0195453419ae4dfea7abd6fb/e10768bb6e9d0ea730280336b697da66.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/637f9f3be880c71f9e3ca07b4d67345c/f9b24c8280986c0683fbceca5326d806.cab
-
-# dbghelp.lib fwpuclnt.lib
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/9f51690d5aa804b1340ce12d1ec80f89/windows%20sdk%20desktop%20libs%20x64-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/d3a7df4ca3303a698640a29e558a5e5b/58314d0646d7e1a25e97c902166c3155.cab
-
-# libcmt.lib libvcruntime.lib
-curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/8728f21ae09940f1f4b4ee47b4a596be2509e2a47d2f0c83bbec0ea37d69644b/Microsoft.VisualC.14.16.CRT.x64.Desktop.vsix
-
-
-msiextract universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
-msiextract windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
-msiextract windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
-msiextract windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
-msiextract windows%20sdk%20desktop%20libs%20x64-x86_en-us.msi
-unzip -o Microsoft.VisualC.14.16.CRT.Headers.vsix
-unzip -o Microsoft.VisualC.14.16.CRT.x64.Desktop.vsix
-
-mkdir -p /usr/x86_64-pc-windows-msvc/usr/include
-mkdir -p /usr/x86_64-pc-windows-msvc/usr/lib
-
-# lowercase folder/file names
-echo "$(find . -regex ".*/[^/]*[A-Z][^/]*")" | xargs -I{} sh -c 'mv "$(echo "{}" | sed -E '"'"'s/(.*\/)/\L\1/'"'"')" "$(echo "{}" | tr [A-Z] [a-z])"'
-
-# .h
-(cd 'program files/windows kits/10/include/10.0.26100.0' && cp -r ucrt/* um/* shared/* -t /usr/x86_64-pc-windows-msvc/usr/include)
-
-cp -r contents/vc/tools/msvc/14.16.27023/include/* /usr/x86_64-pc-windows-msvc/usr/include
-
-# lowercase #include "" and #include <>
-find /usr/x86_64-pc-windows-msvc/usr/include -type f -exec sed -i -E 's/(#include <[^<>]*?[A-Z][^<>]*?>)|(#include "[^"]*?[A-Z][^"]*?")/\L\1\2/' "{}" ';'
-
-# x86 intrinsics
-# original dir: MSVC/
-
-# '_mm_movemask_epi8' defined in emmintrin.h
-# '__v4sf' defined in xmmintrin.h
-# '__v2si' defined in mmintrin.h
-# '__m128d' redefined in immintrin.h
-# '__m128i' redefined in intrin.h
-# '_mm_comlt_epu8' defined in ammintrin.h
-
-(cd /usr/lib/llvm19/lib/clang/19/include && cp emmintrin.h xmmintrin.h mmintrin.h immintrin.h intrin.h ammintrin.h -t /usr/x86_64-pc-windows-msvc/usr/include)
-
-# .lib
-(cd 'program files/windows kits/10/lib/10.0.26100.0/um/x64' && cp advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib dbghelp.lib fwpuclnt.lib -t /usr/x86_64-pc-windows-msvc/usr/lib)
-
-(cd 'contents/vc/tools/msvc/14.16.27023/lib/x64' && cp libcmt.lib libvcruntime.lib -t /usr/x86_64-pc-windows-msvc/usr/lib)
-
-cp 'program files/windows kits/10/lib/10.0.26100.0/ucrt/x64/libucrt.lib' /usr/x86_64-pc-windows-msvc/usr/lib

ci/update_lockfiles.sh

@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-AMEND=false
-
-for arg in "$@"; do
-  if [[ "$arg" == "--amend" ]]; then
-    AMEND=true
-  fi
-done
-
-# This updates the lockfile without building
-cargo metadata --quiet > /dev/null
-
-pushd nodejs || exit 1
-npm install --package-lock-only --silent
-popd
-
-if git diff --quiet --exit-code; then
-  echo "No lockfile changes to commit; skipping amend."
-elif $AMEND; then
-  git add Cargo.lock nodejs/package-lock.json
-  git commit --amend --no-edit
-else
-  git add Cargo.lock nodejs/package-lock.json
-  git commit -m "Update lockfiles"
-fi

ci/validate_stable_lance.py

@@ -1,34 +0,0 @@
-import tomllib
-
-found_preview_lance = False
-
-with open("Cargo.toml", "rb") as f:
-    cargo_data = tomllib.load(f)
-
-    for name, dep in cargo_data["workspace"]["dependencies"].items():
-        if name == "lance" or name.startswith("lance-"):
-            if isinstance(dep, str):
-                version = dep
-            elif isinstance(dep, dict):
-                # Version doesn't have the beta tag in it, so we instead look
-                # at the git tag.
-                version = dep.get('tag', dep.get('version'))
-            else:
-                raise ValueError("Unexpected type for dependency: " + str(dep))
-
-            if "beta" in version:
-                found_preview_lance = True
-                print(f"Dependency '{name}' is a preview version: {version}")
-
-with open("python/pyproject.toml", "rb") as f:
-    py_proj_data = tomllib.load(f)
-
-    for dep in py_proj_data["project"]["dependencies"]:
-        if dep.startswith("pylance"):
-            if "b" in dep:
-                found_preview_lance = True
-                print(f"Dependency '{dep}' is a preview version")
-            break  # Only one pylance dependency
-
-if found_preview_lance:
-    raise ValueError("Found preview version of Lance in dependencies")

docker-compose.yml

@@ -1,18 +1,18 @@
 version: "3.9"
 services:
   localstack:
-    image: localstack/localstack:3.3
+    image: localstack/localstack:0.14
     ports:
       - 4566:4566
     environment:
-      - SERVICES=s3,dynamodb,kms
+      - SERVICES=s3,dynamodb
       - DEBUG=1
       - LS_LOG=trace
       - DOCKER_HOST=unix:///var/run/docker.sock
       - AWS_ACCESS_KEY_ID=ACCESSKEY
       - AWS_SECRET_ACCESS_KEY=SECRETKEY
     healthcheck:
-      test: [ "CMD", "curl", "-s", "http://localhost:4566/_localstack/health" ]
+      test: [ "CMD", "curl", "-f", "http://localhost:4566/health" ]
       interval: 5s
       retries: 3
       start_period: 10s

dockerfiles/Dockerfile

@@ -1,27 +0,0 @@
-#Simple base dockerfile that supports basic dependencies required to run lance with FTS and Hybrid Search
-#Usage docker build -t lancedb:latest -f Dockerfile .
-FROM python:3.10-slim-buster
-
-# Install Rust
-RUN apt-get update && apt-get install -y curl build-essential && \
-  curl https://sh.rustup.rs -sSf | sh -s -- -y
-
-# Set the environment variable for Rust
-ENV PATH="/root/.cargo/bin:${PATH}"
-
-# Install protobuf compiler
-RUN apt-get install -y protobuf-compiler && \
-  apt-get clean && \
-  rm -rf /var/lib/apt/lists/*
-
-RUN apt-get -y update &&\
-  apt-get -y upgrade && \
-  apt-get -y install git
-
-
-# Verify installations
-RUN python --version && \
-  rustc --version && \
-  protoc --version
-
-RUN pip install tantivy lancedb

docs/README.md

@@ -1,89 +0,0 @@
-# LanceDB Documentation
-
-LanceDB docs are available at [lancedb.com/docs](https://lancedb.com/docs).
-
-The SDK docs are built and deployed automatically by [Github Actions](../.github/workflows/docs.yml)
-whenever a commit is pushed to the `main` branch. So it is possible for the docs to show
-unreleased features.
-
-## Building the docs
-
-### Setup
-1. Install LanceDB Python. See setup in [Python contributing guide](../python/CONTRIBUTING.md).
-   Run `make develop` to install the Python package.
-2. Install documentation dependencies. From LanceDB repo root: `pip install -r docs/requirements.txt`
-
-### Preview the docs
-
-```shell
-cd docs
-mkdocs serve
-```
-
-If you want to just generate the HTML files:
-
-```shell
-PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
-```
-
-If successful, you should see a `docs/site` directory that you can verify locally.
-
-## Adding examples
-
-To make sure examples are correct, we put examples in test files so they can be
-run as part of our test suites.
-
-You can see the tests are at:
-
-* Python: `python/python/tests/docs`
-* Typescript: `nodejs/examples/`
-
-### Checking python examples
-
-```shell
-cd python
-pytest -vv python/tests/docs
-```
-
-### Checking typescript examples
-
-The `@lancedb/lancedb` package must be built before running the tests:
-
-```shell
-pushd nodejs
-npm ci
-npm run build
-popd
-```
-
-Then you can run the examples by going to the `nodejs/examples` directory and
-running the tests like a normal npm package:
-
-```shell
-pushd nodejs/examples
-npm ci
-npm test
-popd
-```
-
-## API documentation
-
-### Python
-
-The Python API documentation is organized based on the file `docs/src/python/python.md`.
-We manually add entries there so we can control the organization of the reference page.
-**However, this means any new types must be manually added to the file.** No additional
-steps are needed to generate the API documentation.
-
-### Typescript
-
-The typescript API documentation is generated from the typescript source code using [typedoc](https://typedoc.org/).
-
-When new APIs are added, you must manually re-run the typedoc command to update the API documentation.
-The new files should be checked into the repository.
-
-```shell
-pushd nodejs
-npm run docs
-popd
-```

docs/mkdocs.yml

@@ -1,146 +1,143 @@
-site_name: LanceDB
-site_url: https://lancedb.github.io/lancedb/
+site_name: LanceDB Docs
 repo_url: https://github.com/lancedb/lancedb
 edit_uri: https://github.com/lancedb/lancedb/tree/main/docs/src
 repo_name: lancedb/lancedb
 docs_dir: src
-watch:
-  - src
-  - ../python/python
 
 theme:
   name: "material"
   logo: assets/logo.png
   favicon: assets/logo.png
-  palette:
-    # Palette toggle for light mode
-    - scheme: lancedb
-      primary: custom
-      toggle:
-        icon: material/weather-night
-        name: Switch to dark mode
-    # Palette toggle for dark mode
-    - scheme: slate
-      primary: custom
-      toggle:
-        icon: material/weather-sunny
-        name: Switch to light mode
   features:
     - content.code.copy
     - content.tabs.link
     - content.action.edit
-    - content.tooltips
     - toc.follow
+    - toc.integrate
     - navigation.top
     - navigation.tabs
     - navigation.tabs.sticky
     - navigation.footer
     - navigation.tracking
     - navigation.instant
-    - content.footnote.tooltips
+    - navigation.indexes
+    - navigation.expand
   icon:
     repo: fontawesome/brands/github
-    annotation: material/arrow-right-circle
+  custom_dir: overrides
 
 plugins:
-  - search
-  - autorefs
-  - mkdocstrings:
-      handlers:
-        python:
-          # Ensure the handler points to the real package root
-          # so it reads local sources at python/python/lancedb
-          paths: [../python/python]
-          options:
-            docstring_style: numpy
-            heading_level: 3
-            show_source: true
-            show_symbol_type_in_heading: true
-            show_signature_annotations: true
-            show_root_heading: true
-            members_order: source
-            docstring_section_style: list
-            signature_crossrefs: true
-            separate_signature: true
-          import:
-            # for cross references
-            - https://arrow.apache.org/docs/objects.inv
-            - https://pandas.pydata.org/docs/objects.inv
-            - https://docs.pydantic.dev/latest/objects.inv
-  - render_swagger:
-      allow_arbitrary_locations: true
-#   - redirects:
-#       redirect_maps:
-#         # Redirect the home page and other top-level markdown files. This enables maximum SEO benefit
-#         # other sub-pages are handled by the ingected js in overrides/partials/header.html
-#         'index.md': 'https://lancedb.com/docs/'
-#         'guides/tables.md': 'https://lancedb.com/docs/tables/'
-#         'ann_indexes.md': 'https://lancedb.com/docs/indexing/'
-#         'basic.md': 'https://lancedb.com/docs/quickstart/'
-#         'faq.md': 'https://lancedb.com/docs/faq/'
-#         'embeddings/understanding_embeddings.md': 'https://lancedb.com/docs/embedding/'
-#         'integrations.md': 'https://lancedb.com/docs/integrations/'
-#         'examples.md': 'https://lancedb.com/docs/tutorials/'
-#         'concepts/vector_search.md': 'https://lancedb.com/docs/search/vector-search/'
-#         'troubleshooting.md': 'https://lancedb.com/docs/troubleshooting/'
-#         'guides/storage.md': 'https://lancedb.com/docs/storage/integrations'
-
-
+- search
+- autorefs
+- mkdocstrings:
+    handlers:
+      python:
+        paths: [../python]
+        selection:
+          docstring_style: numpy
+        rendering:
+          heading_level: 4
+          show_source: false
+          show_symbol_type_in_heading: true
+          show_signature_annotations: true
+          show_root_heading: true
+          members_order: source
+        import:
+          # for cross references
+          - https://arrow.apache.org/docs/objects.inv
+          - https://pandas.pydata.org/docs/objects.inv
+- mkdocs-jupyter
 
 markdown_extensions:
-  - admonition
-  - footnotes
-  - pymdownx.critic
-  - pymdownx.caret
-  - pymdownx.keys
-  - pymdownx.mark
-  - pymdownx.tilde
-  - pymdownx.details
-  - pymdownx.highlight:
-      anchor_linenums: true
-      line_spans: __span
-      pygments_lang_class: true
-  - pymdownx.inlinehilite
-  - pymdownx.snippets:
-      base_path: ..
-      dedent_subsections: true
-  - pymdownx.superfences
-  - pymdownx.tabbed:
-      alternate_style: true
-  - md_in_html
-  - abbr
-  - attr_list
-  - pymdownx.snippets
-  - pymdownx.emoji:
-      emoji_index: !!python/name:material.extensions.emoji.twemoji
-      emoji_generator: !!python/name:material.extensions.emoji.to_svg
-  - markdown.extensions.toc:
-      baselevel: 1
-      permalink: ""
+- admonition
+- footnotes
+- pymdownx.superfences
+- pymdownx.details
+- pymdownx.highlight:
+    anchor_linenums: true
+    line_spans: __span
+    pygments_lang_class: true
+- pymdownx.inlinehilite
+- pymdownx.snippets
+- pymdownx.superfences
+- pymdownx.tabbed:
+    alternate_style: true
+- md_in_html
 
 nav:
-  - API reference:
-      - Overview: index.md
-      - Python: python/python.md
-      - Javascript/TypeScript: js/globals.md
-      - Java: java/java.md
-      - Rust: https://docs.rs/lancedb/latest/lancedb/index.html
+- Home:
+  - 🏢 Home: index.md
+  - 💡 Basics: basic.md
+  - 📚 Guides:
+    - Create Ingest Update Delete: guides/tables.md
+    - Vector Search: search.md
+    - SQL filters: sql.md
+    - Indexing: ann_indexes.md
+  - 🧬 Embeddings: embedding.md
+  - 🔍 Python full-text search: fts.md
+  - 🔌 Integrations:
+    - integrations/index.md
+    - Pandas and PyArrow: python/arrow.md
+    - DuckDB: python/duckdb.md
+    - LangChain 🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
+    - LangChain JS/TS 🔗: https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb
+    - LlamaIndex 🦙: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
+    - Pydantic: python/pydantic.md
+    - Voxel51: integrations/voxel51.md
+    - PromptTools: integrations/prompttools.md
+  - 🐍 Python examples:
+    - examples/index.md
+    - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
+    - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
+    - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
+    - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
+    - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
+  - 🌐 Javascript examples:
+    - Examples: examples/index_js.md
+    - Serverless Website Chatbot: examples/serverless_website_chatbot.md
+    - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
+    - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
+  - ⚙️ CLI & Config: cli_config.md
+
+- Basics: basic.md
+- Guides:
+  - Create Ingest Update Delete: guides/tables.md
+  - Vector Search: search.md
+  - SQL filters: sql.md
+  - Indexing: ann_indexes.md
+- Embeddings: embedding.md
+- Python full-text search: fts.md
+- Integrations:
+  - integrations/index.md
+  - Pandas and PyArrow: python/arrow.md
+  - DuckDB: python/duckdb.md
+  - LangChain 🦜️🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
+  - LangChain JS/TS 🦜️🔗: https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb
+  - LlamaIndex 🦙: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
+  - Pydantic: python/pydantic.md
+  - Voxel51: integrations/voxel51.md
+  - PromptTools: integrations/prompttools.md
+- Python examples:
+  - examples/index.md
+  - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
+  - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
+  - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
+  - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
+  - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
+- Javascript examples:
+  - examples/index_js.md
+  - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
+  - Serverless Chatbot from any website: examples/serverless_website_chatbot.md
+  - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
+- API references:
+  - Python API: python/python.md
+  - Javascript API: javascript/modules.md
+- LanceDB Cloud↗: https://noteforms.com/forms/lancedb-mailing-list-cloud-kty1o5?notionforms=1&utm_source=notionforms
 
 extra_css:
   - styles/global.css
-  - styles/extra.css
-
-extra_javascript:
-  - "extra_js/reo.js"
 
 extra:
   analytics:
     provider: google
     property: G-B7NFM40W74
-  social:
-    - icon: fontawesome/brands/github
-      link: https://github.com/lancedb/lancedb
-    - icon: fontawesome/brands/x-twitter
-      link: https://twitter.com/lancedb
-    - icon: fontawesome/brands/linkedin
-      link: https://www.linkedin.com/company/lancedb

docs/openapi.yml

@@ -1,513 +0,0 @@
-openapi: 3.1.0
-info:
-  version: 1.0.0
-  title: LanceDB Cloud API
-  description: |
-    LanceDB Cloud API is a RESTful API that allows users to access and modify data stored in LanceDB Cloud.
-    Table actions are considered temporary resource creations and all use POST method.
-  contact:
-    name: LanceDB support
-    url: https://lancedb.com
-    email: contact@lancedb.com
-
-servers:
-  - url: https://{db}.{region}.api.lancedb.com
-    description: LanceDB Cloud REST endpoint.
-    variables:
-      db:
-        default: ""
-        description: the name of DB
-      region:
-        default: "us-east-1"
-        description: the service region of the DB
-
-security:
-  - key_auth: []
-
-components:
-  securitySchemes:
-    key_auth:
-      name: x-api-key
-      type: apiKey
-      in: header
-  parameters:
-    table_name:
-      name: name
-      in: path
-      description: name of the table
-      required: true
-      schema:
-        type: string
-    index_name:
-      name: index_name
-      in: path
-      description: name of the index
-      required: true
-      schema:
-        type: string
-  responses:
-    invalid_request:
-      description: Invalid request
-      content:
-        text/plain:
-          schema:
-            type: string
-    not_found:
-      description: Not found
-      content:
-        text/plain:
-          schema:
-            type: string
-    unauthorized:
-      description: Unauthorized
-      content:
-        text/plain:
-          schema:
-            type: string
-  requestBodies:
-    arrow_stream_buffer:
-      description: Arrow IPC stream buffer
-      required: true
-      content:
-        application/vnd.apache.arrow.stream:
-          schema:
-            type: string
-            format: binary
-
-paths:
-  /v1/table/:
-    get:
-      description: List tables, optionally, with pagination.
-      tags:
-        - Tables
-      summary: List Tables
-      operationId: listTables
-      parameters:
-        - name: limit
-          in: query
-          description: Limits the number of items to return.
-          schema:
-            type: integer
-        - name: page_token
-          in: query
-          description: Specifies the starting position of the next query
-          schema:
-            type: string
-      responses:
-        "200":
-          description: Successfully returned a list of tables in the DB
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  tables:
-                    type: array
-                    items:
-                      type: string
-                  page_token:
-                    type: string
-
-        "400":
-          $ref: "#/components/responses/invalid_request"
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"
-
-  /v1/table/{name}/create/:
-    post:
-      description: Create a new table
-      summary: Create a new table
-      operationId: createTable
-      tags:
-        - Tables
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-      requestBody:
-        $ref: "#/components/requestBodies/arrow_stream_buffer"
-      responses:
-        "200":
-          description: Table successfully created
-        "400":
-          $ref: "#/components/responses/invalid_request"
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"
-
-  /v1/table/{name}/query/:
-    post:
-      description: Vector Query
-      url: https://{db-uri}.{aws-region}.api.lancedb.com/v1/table/{name}/query/
-      tags:
-        - Data
-      summary: Vector Query
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                vector:
-                  type: FixedSizeList
-                  description: |
-                    The targetted vector to search for. Required.
-                vector_column:
-                  type: string
-                  description: |
-                    The column to query, it can be inferred from the schema if there is only one vector column.
-                prefilter:
-                  type: boolean
-                  description: |
-                    Whether to prefilter the data. Optional.
-                k:
-                  type: integer
-                  description: |
-                    The number of search results to return. Default is 10.
-                distance_type:
-                  type: string
-                  description: |
-                    The distance metric to use for search. l2, Cosine, Dot and Hamming are supported. Default is l2.
-                bypass_vector_index:
-                  type: boolean
-                  description: |
-                    Whether to bypass vector index. Optional.
-                filter:
-                  type: string
-                  description: |
-                    A filter expression that specifies the rows to query. Optional.
-                columns:
-                  type: array
-                  items:
-                    type: string
-                  description: |
-                    The columns to return. Optional.
-                nprobe:
-                  type: integer
-                  description: |
-                    The number of probes to use for search. Optional.
-                refine_factor:
-                  type: integer
-                  description: |
-                    The refine factor to use for search. Optional.
-                  default: null
-                fast_search:
-                  type: boolean
-                  description: |
-                    Whether to use fast search. Optional.
-                  default: false
-              required:
-                - vector
-
-      responses:
-        "200":
-          description: top k results if query is successfully executed
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  results:
-                    type: array
-                    items:
-                      type: object
-                      properties:
-                        id:
-                          type: integer
-                        selected_col_1_to_return:
-                          type: col_1_type
-                        selected_col_n_to_return:
-                          type: col_n_type
-                        _distance:
-                          type: float
-
-        "400":
-          $ref: "#/components/responses/invalid_request"
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"
-
-  /v1/table/{name}/insert/:
-    post:
-      description: Insert new data to the Table.
-      tags:
-        - Data
-      operationId: insertData
-      summary: Insert new data.
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-      requestBody:
-        $ref: "#/components/requestBodies/arrow_stream_buffer"
-      responses:
-        "200":
-          description: Insert successful
-        "400":
-          $ref: "#/components/responses/invalid_request"
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"
-  /v1/table/{name}/merge_insert/:
-    post:
-      description: Create a "merge insert" operation
-        This operation can add rows, update rows, and remove rows all in a single
-        transaction. See python method `lancedb.table.Table.merge_insert` for examples.
-      tags:
-        - Data
-      summary: Merge Insert
-      operationId: mergeInsert
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-        - name: on
-          in: query
-          description: |
-            The column to use as the primary key for the merge operation.
-          required: true
-          schema:
-            type: string
-        - name: when_matched_update_all
-          in: query
-          description: |
-            Rows that exist in both the source table (new data) and
-            the target table (old data) will be updated, replacing
-            the old row with the corresponding matching row.
-          required: false
-          schema:
-            type: boolean
-        - name: when_matched_update_all_filt
-          in: query
-          description: |
-            If present then only rows that satisfy the filter expression will
-            be updated
-          required: false
-          schema:
-            type: string
-        - name: when_not_matched_insert_all
-          in: query
-          description: |
-            Rows that exist only in the source table (new data) will be
-            inserted into the target table (old data).
-          required: false
-          schema:
-            type: boolean
-        - name: when_not_matched_by_source_delete
-          in: query
-          description: |
-            Rows that exist only in the target table (old data) will be
-            deleted. An optional condition (`when_not_matched_by_source_delete_filt`)
-            can be provided to limit what data is deleted.
-          required: false
-          schema:
-            type: boolean
-        - name: when_not_matched_by_source_delete_filt
-          in: query
-          description: |
-            The filter expression that specifies the rows to delete.
-          required: false
-          schema:
-            type: string
-      requestBody:
-        $ref: "#/components/requestBodies/arrow_stream_buffer"
-      responses:
-        "200":
-          description: Merge Insert successful
-        "400":
-          $ref: "#/components/responses/invalid_request"
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"
-  /v1/table/{name}/delete/:
-    post:
-      description: Delete rows from a table.
-      tags:
-        - Data
-      summary: Delete rows from a table
-      operationId: deleteData
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                predicate:
-                  type: string
-                  description: |
-                    A filter expression that specifies the rows to delete.
-      responses:
-        "200":
-          description: Delete successful
-        "401":
-          $ref: "#/components/responses/unauthorized"
-  /v1/table/{name}/drop/:
-    post:
-      description: Drop a table
-      tags:
-        - Tables
-      summary: Drop a table
-      operationId: dropTable
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-      requestBody:
-        $ref: "#/components/requestBodies/arrow_stream_buffer"
-      responses:
-        "200":
-          description: Drop successful
-        "401":
-          $ref: "#/components/responses/unauthorized"
-
-  /v1/table/{name}/describe/:
-    post:
-      description: Describe a table and return Table Information.
-      tags:
-        - Tables
-      summary: Describe a table
-      operationId: describeTable
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-      responses:
-        "200":
-          description: Table information
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  table:
-                    type: string
-                  version:
-                    type: integer
-                  schema:
-                    type: string
-                  stats:
-                    type: object
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"
-
-  /v1/table/{name}/index/list/:
-    post:
-      description: List indexes of a table
-      tags:
-        - Tables
-      summary: List indexes of a table
-      operationId: listIndexes
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-      responses:
-        "200":
-          description: Available list of indexes on the table.
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  indexes:
-                    type: array
-                    items:
-                      type: object
-                      properties:
-                        columns:
-                          type: array
-                          items:
-                            type: string
-                        index_name:
-                          type: string
-                        index_uuid:
-                          type: string
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"
-  /v1/table/{name}/create_index/:
-    post:
-      description: Create vector index on a Table
-      tags:
-        - Tables
-      summary: Create vector index on a Table
-      operationId: createIndex
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                column:
-                  type: string
-                metric_type:
-                  type: string
-                  nullable: false
-                  description: |
-                    The metric type to use for the index. l2, Cosine, Dot are supported.
-                index_type:
-                  type: string
-      responses:
-        "200":
-          description: Index successfully created
-        "400":
-          $ref: "#/components/responses/invalid_request"
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"
-  /v1/table/{name}/create_scalar_index/:
-    post:
-      description: Create a scalar index on a table
-      tags:
-        - Tables
-      summary: Create a scalar index on a table
-      operationId: createScalarIndex
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                column:
-                  type: string
-                index_type:
-                  type: string
-                  required: false
-      responses:
-        "200":
-          description: Scalar Index successfully created
-        "400":
-          $ref: "#/components/responses/invalid_request"
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"
-  /v1/table/{name}/index/{index_name}/drop/:
-    post:
-      description: Drop an index from the table
-      tags:
-        - Tables
-      summary: Drop an index from the table
-      operationId: dropIndex
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-        - $ref: "#/components/parameters/index_name"
-      responses:
-        "200":
-          description: Index successfully dropped
-        "400":
-          $ref: "#/components/responses/invalid_request"
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"

docs/overrides/partials/header.html

@@ -0,0 +1,176 @@
+<!--
+  Copyright (c) 2016-2023 Martin Donath <martin.donath@squidfunk.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to
+  deal in the Software without restriction, including without limitation the
+  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+  sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+  IN THE SOFTWARE.
+-->
+
+{% set class = "md-header" %}
+{% if "navigation.tabs.sticky" in features %}
+  {% set class = class ~ " md-header--shadow md-header--lifted" %}
+{% elif "navigation.tabs" not in features %}
+  {% set class = class ~ " md-header--shadow" %}
+{% endif %}
+
+<!-- Header -->
+<header class="{{ class }}" data-md-component="header">
+  <nav
+    class="md-header__inner md-grid"
+    aria-label="{{ lang.t('header') }}"
+  >
+
+    <!-- Link to home -->
+    <a
+      href="{{ config.extra.homepage | d(nav.homepage.url, true) | url }}"
+      title="{{ config.site_name | e }}"
+      class="md-header__button md-logo"
+      aria-label="{{ config.site_name }}"
+      data-md-component="logo"
+    >
+      {% include "partials/logo.html" %}
+    </a>
+
+    <!-- Button to open drawer -->
+    <label class="md-header__button md-icon" for="__drawer">
+      {% include ".icons/material/menu" ~ ".svg" %}
+    </label>
+
+    <!-- Header title -->
+    <div class="md-header__title" style="width: auto !important;" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            {{ config.site_name }}
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+            {% if page.meta and page.meta.title %}
+              {{ page.meta.title }}
+            {% else %}
+              {{ page.title }}
+            {% endif %}
+          </span>
+        </div>
+      </div>
+    </div>
+
+    <!-- Color palette -->
+    {% if config.theme.palette %}
+      {% if not config.theme.palette is mapping %}
+        <form class="md-header__option" data-md-component="palette">
+          {% for option in config.theme.palette %}
+            {% set scheme  = option.scheme  | d("default", true) %}
+            {% set primary = option.primary | d("indigo", true) %}
+            {% set accent  = option.accent  | d("indigo", true) %}
+            <input
+              class="md-option"
+              data-md-color-media="{{ option.media }}"
+              data-md-color-scheme="{{ scheme | replace(' ', '-') }}"
+              data-md-color-primary="{{ primary | replace(' ', '-') }}"
+              data-md-color-accent="{{ accent | replace(' ', '-') }}"
+              {% if option.toggle %}
+                aria-label="{{ option.toggle.name }}"
+              {% else  %}
+                aria-hidden="true"
+              {% endif %}
+              type="radio"
+              name="__palette"
+              id="__palette_{{ loop.index }}"
+            />
+            {% if option.toggle %}
+              <label
+                class="md-header__button md-icon"
+                title="{{ option.toggle.name }}"
+                for="__palette_{{ loop.index0 or loop.length }}"
+                hidden
+              >
+                {% include ".icons/" ~ option.toggle.icon ~ ".svg" %}
+              </label>
+            {% endif %}
+          {% endfor %}
+        </form>
+      {% endif %}
+    {% endif %}
+
+    <!-- Site language selector -->
+    {% if config.extra.alternate %}
+      <div class="md-header__option">
+        <div class="md-select">
+          {% set icon = config.theme.icon.alternate or "material/translate" %}
+          <button
+            class="md-header__button md-icon"
+            aria-label="{{ lang.t('select.language') }}"
+          >
+            {% include ".icons/" ~ icon ~ ".svg" %}
+          </button>
+          <div class="md-select__inner">
+            <ul class="md-select__list">
+              {% for alt in config.extra.alternate %}
+                <li class="md-select__item">
+                  <a
+                    href="{{ alt.link | url }}"
+                    hreflang="{{ alt.lang }}"
+                    class="md-select__link"
+                  >
+                    {{ alt.name }}
+                  </a>
+                </li>
+              {% endfor %}
+            </ul>
+          </div>
+        </div>
+      </div>
+    {% endif %}
+
+    <!-- Button to open search modal -->
+    {% if "material/search" in config.plugins %}
+      <label class="md-header__button md-icon" for="__search">
+        {% include ".icons/material/magnify.svg" %}
+      </label>
+
+      <!-- Search interface -->
+      {% include "partials/search.html" %}
+    {% endif %}
+
+    <div style="margin-left: 10px; margin-right: 5px;">
+        <a href="https://discord.com/invite/zMM32dvNtd" target="_blank" rel="noopener noreferrer">
+            <svg fill="#FFFFFF" xmlns="http://www.w3.org/2000/svg"  viewBox="0 0 50 50" width="25px" height="25px"><path d="M 41.625 10.769531 C 37.644531 7.566406 31.347656 7.023438 31.078125 7.003906 C 30.660156 6.96875 30.261719 7.203125 30.089844 7.589844 C 30.074219 7.613281 29.9375 7.929688 29.785156 8.421875 C 32.417969 8.867188 35.652344 9.761719 38.578125 11.578125 C 39.046875 11.867188 39.191406 12.484375 38.902344 12.953125 C 38.710938 13.261719 38.386719 13.429688 38.050781 13.429688 C 37.871094 13.429688 37.6875 13.378906 37.523438 13.277344 C 32.492188 10.15625 26.210938 10 25 10 C 23.789063 10 17.503906 10.15625 12.476563 13.277344 C 12.007813 13.570313 11.390625 13.425781 11.101563 12.957031 C 10.808594 12.484375 10.953125 11.871094 11.421875 11.578125 C 14.347656 9.765625 17.582031 8.867188 20.214844 8.425781 C 20.0625 7.929688 19.925781 7.617188 19.914063 7.589844 C 19.738281 7.203125 19.34375 6.960938 18.921875 7.003906 C 18.652344 7.023438 12.355469 7.566406 8.320313 10.8125 C 6.214844 12.761719 2 24.152344 2 34 C 2 34.175781 2.046875 34.34375 2.132813 34.496094 C 5.039063 39.605469 12.972656 40.941406 14.78125 41 C 14.789063 41 14.800781 41 14.8125 41 C 15.132813 41 15.433594 40.847656 15.621094 40.589844 L 17.449219 38.074219 C 12.515625 36.800781 9.996094 34.636719 9.851563 34.507813 C 9.4375 34.144531 9.398438 33.511719 9.765625 33.097656 C 10.128906 32.683594 10.761719 32.644531 11.175781 33.007813 C 11.234375 33.0625 15.875 37 25 37 C 34.140625 37 38.78125 33.046875 38.828125 33.007813 C 39.242188 32.648438 39.871094 32.683594 40.238281 33.101563 C 40.601563 33.515625 40.5625 34.144531 40.148438 34.507813 C 40.003906 34.636719 37.484375 36.800781 32.550781 38.074219 L 34.378906 40.589844 C 34.566406 40.847656 34.867188 41 35.1875 41 C 35.199219 41 35.210938 41 35.21875 41 C 37.027344 40.941406 44.960938 39.605469 47.867188 34.496094 C 47.953125 34.34375 48 34.175781 48 34 C 48 24.152344 43.785156 12.761719 41.625 10.769531 Z M 18.5 30 C 16.566406 30 15 28.210938 15 26 C 15 23.789063 16.566406 22 18.5 22 C 20.433594 22 22 23.789063 22 26 C 22 28.210938 20.433594 30 18.5 30 Z M 31.5 30 C 29.566406 30 28 28.210938 28 26 C 28 23.789063 29.566406 22 31.5 22 C 33.433594 22 35 23.789063 35 26 C 35 28.210938 33.433594 30 31.5 30 Z"/></svg>
+        </a>
+    </div>
+    <div style="margin-left: 5px; margin-right: 5px;">
+        <a href="https://twitter.com/lancedb" target="_blank" rel="noopener noreferrer">
+            <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0,0,256,256" width="25px" height="25px" fill-rule="nonzero"><g fill-opacity="0" fill="#ffffff" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><path d="M0,256v-256h256v256z" id="bgRectangle"></path></g><g fill="#ffffff" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(4,4)"><path d="M57,17.114c-1.32,1.973 -2.991,3.707 -4.916,5.097c0.018,0.423 0.028,0.847 0.028,1.274c0,13.013 -9.902,28.018 -28.016,28.018c-5.562,0 -12.81,-1.948 -15.095,-4.423c0.772,0.092 1.556,0.138 2.35,0.138c4.615,0 8.861,-1.575 12.23,-4.216c-4.309,-0.079 -7.946,-2.928 -9.199,-6.84c1.96,0.308 4.447,-0.17 4.447,-0.17c0,0 -7.7,-1.322 -7.899,-9.779c2.226,1.291 4.46,1.231 4.46,1.231c0,0 -4.441,-2.734 -4.379,-8.195c0.037,-3.221 1.331,-4.953 1.331,-4.953c8.414,10.361 20.298,10.29 20.298,10.29c0,0 -0.255,-1.471 -0.255,-2.243c0,-5.437 4.408,-9.847 9.847,-9.847c2.832,0 5.391,1.196 7.187,3.111c2.245,-0.443 4.353,-1.263 6.255,-2.391c-0.859,3.44 -4.329,5.448 -4.329,5.448c0,0 2.969,-0.329 5.655,-1.55z"></path></g></g></svg>
+        </a>
+    </div>
+
+    <!-- Repository information -->
+    {% if config.repo_url %}
+      <div class="md-header__source" style="margin-left: -5px !important;">
+        {% include "partials/source.html" %}
+      </div>
+    {% endif %}
+  </nav>
+
+  <!-- Navigation tabs (sticky) -->
+  {% if "navigation.tabs.sticky" in features %}
+    {% if "navigation.tabs" in features %}
+      {% include "partials/tabs.html" %}
+    {% endif %}
+  {% endif %}
+</header>

docs/package-lock.json

@@ -1,135 +0,0 @@
-{
-  "name": "lancedb-docs-test",
-  "version": "1.0.0",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {
-    "": {
-      "name": "lancedb-docs-test",
-      "version": "1.0.0",
-      "license": "Apache 2",
-      "dependencies": {
-        "apache-arrow": "file:../node/node_modules/apache-arrow",
-        "vectordb": "file:../node"
-      },
-      "devDependencies": {
-        "@types/node": "^20.11.8",
-        "typescript": "^5.3.3"
-      }
-    },
-    "../node": {
-      "name": "vectordb",
-      "version": "0.21.2-beta.0",
-      "cpu": [
-        "x64",
-        "arm64"
-      ],
-      "license": "Apache-2.0",
-      "os": [
-        "darwin",
-        "linux",
-        "win32"
-      ],
-      "dependencies": {
-        "@neon-rs/load": "^0.0.74",
-        "axios": "^1.4.0"
-      },
-      "devDependencies": {
-        "@neon-rs/cli": "^0.0.160",
-        "@types/chai": "^4.3.4",
-        "@types/chai-as-promised": "^7.1.5",
-        "@types/mocha": "^10.0.1",
-        "@types/node": "^18.16.2",
-        "@types/sinon": "^10.0.15",
-        "@types/temp": "^0.9.1",
-        "@types/uuid": "^9.0.3",
-        "@typescript-eslint/eslint-plugin": "^5.59.1",
-        "apache-arrow-old": "npm:apache-arrow@13.0.0",
-        "cargo-cp-artifact": "^0.1",
-        "chai": "^4.3.7",
-        "chai-as-promised": "^7.1.1",
-        "eslint": "^8.39.0",
-        "eslint-config-standard-with-typescript": "^34.0.1",
-        "eslint-plugin-import": "^2.26.0",
-        "eslint-plugin-n": "^15.7.0",
-        "eslint-plugin-promise": "^6.1.1",
-        "mocha": "^10.2.0",
-        "openai": "^4.24.1",
-        "sinon": "^15.1.0",
-        "temp": "^0.9.4",
-        "ts-node": "^10.9.1",
-        "ts-node-dev": "^2.0.0",
-        "typedoc": "^0.24.7",
-        "typedoc-plugin-markdown": "^3.15.3",
-        "typescript": "^5.1.0",
-        "uuid": "^9.0.0"
-      },
-      "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.21.2-beta.0",
-        "@lancedb/vectordb-darwin-x64": "0.21.2-beta.0",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.21.2-beta.0",
-        "@lancedb/vectordb-linux-x64-gnu": "0.21.2-beta.0",
-        "@lancedb/vectordb-win32-x64-msvc": "0.21.2-beta.0"
-      },
-      "peerDependencies": {
-        "@apache-arrow/ts": "^14.0.2",
-        "apache-arrow": "^14.0.2"
-      }
-    },
-    "../node/node_modules/apache-arrow": {
-      "version": "14.0.2",
-      "license": "Apache-2.0",
-      "dependencies": {
-        "@types/command-line-args": "5.2.0",
-        "@types/command-line-usage": "5.0.2",
-        "@types/node": "20.3.0",
-        "@types/pad-left": "2.1.1",
-        "command-line-args": "5.2.1",
-        "command-line-usage": "7.0.1",
-        "flatbuffers": "23.5.26",
-        "json-bignum": "^0.0.3",
-        "pad-left": "^2.1.0",
-        "tslib": "^2.5.3"
-      },
-      "bin": {
-        "arrow2csv": "bin/arrow2csv.js"
-      }
-    },
-    "node_modules/@types/node": {
-      "version": "20.11.8",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.8.tgz",
-      "integrity": "sha512-i7omyekpPTNdv4Jb/Rgqg0RU8YqLcNsI12quKSDkRXNfx7Wxdm6HhK1awT3xTgEkgxPn3bvnSpiEAc7a7Lpyow==",
-      "dev": true,
-      "dependencies": {
-        "undici-types": "~5.26.4"
-      }
-    },
-    "node_modules/apache-arrow": {
-      "resolved": "../node/node_modules/apache-arrow",
-      "link": true
-    },
-    "node_modules/typescript": {
-      "version": "5.3.3",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz",
-      "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==",
-      "dev": true,
-      "bin": {
-        "tsc": "bin/tsc",
-        "tsserver": "bin/tsserver"
-      },
-      "engines": {
-        "node": ">=14.17"
-      }
-    },
-    "node_modules/undici-types": {
-      "version": "5.26.5",
-      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
-      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
-      "dev": true
-    },
-    "node_modules/vectordb": {
-      "resolved": "../node",
-      "link": true
-    }
-  }
-}

docs/package.json

@@ -1,20 +0,0 @@
-{
-  "name": "lancedb-docs-test",
-  "version": "1.0.0",
-  "description": "auto-generated tests from doc",
-  "author": "dev@lancedb.com",
-  "license": "Apache 2",
-  "dependencies": {
-    "apache-arrow": "file:../node/node_modules/apache-arrow",
-    "vectordb": "file:../node"
-  },
-  "scripts": {
-    "build": "tsc -b && cd ../node && npm run build-release",
-    "example": "npm run build && node",
-    "test": "npm run build && ls dist/*.js | xargs -n 1 node"
-  },
-  "devDependencies": {
-    "@types/node": "^20.11.8",
-    "typescript": "^5.3.3"
-  }
-}

docs/requirements.txt

@@ -1,9 +1,4 @@
-mkdocs==1.5.3
+mkdocs==1.4.2
 mkdocs-jupyter==0.24.1
-mkdocs-material==9.5.3
-mkdocs-autorefs<=1.0
-mkdocstrings[python]==0.25.2
-griffe
-mkdocs-render-swagger-plugin
-pydantic
-mkdocs-redirects
+mkdocs-material==9.1.3
+mkdocstrings[python]==0.20.0

docs/robots.txt

@@ -1 +0,0 @@
-User-agent: *

docs/src/ann_indexes.md

@@ -0,0 +1,187 @@
+# ANN (Approximate Nearest Neighbor) Indexes
+
+You can create an index over your vector data to make search faster.
+Vector indexes are faster but less accurate than exhaustive search (KNN or Flat Search).
+LanceDB provides many parameters to fine-tune the index's size, the speed of queries, and the accuracy of results.
+
+Currently, LanceDB does *not* automatically create the ANN index.
+LanceDB has optimized code for KNN as well. For many use-cases, datasets under 100K vectors won't require index creation at all.
+If you can live with < 100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
+
+In the future we will look to automatically create and configure the ANN index.
+
+## Types of Index
+
+Lance can support multiple index types, the most widely used one is `IVF_PQ`.
+
+* `IVF_PQ`: use **Inverted File Index (IVF)** to first divide the dataset into `N` partitions,
+   and then use **Product Quantization** to compress vectors in each partition.
+* `DISKANN` (**Experimental**): organize the vector as a on-disk graph, where the vertices approximately
+   represent the nearest neighbors of each vector.
+
+## Creating an IVF_PQ Index
+
+Lance supports `IVF_PQ` index type by default.
+
+=== "Python"
+     Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
+
+     ```python
+     import lancedb
+     import numpy as np
+     uri = "data/sample-lancedb"
+     db = lancedb.connect(uri)
+
+     # Create 10,000 sample vectors
+     data = [{"vector": row, "item": f"item {i}"}
+        for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]
+
+     # Add the vectors to a table
+     tbl = db.create_table("my_vectors", data=data)
+
+     # Create and train the index - you need to have enough data in the table for an effective training step
+     tbl.create_index(num_partitions=256, num_sub_vectors=96)
+     ```
+
+=== "Javascript"
+     ```javascript
+     const vectordb = require('vectordb')
+     const db = await vectordb.connect('data/sample-lancedb')
+
+     let data = []
+     for (let i = 0; i < 10_000; i++) {
+         data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
+     }
+     const table = await db.createTable('my_vectors', data)
+     await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 256, num_sub_vectors: 96 })
+     ```
+
+- **metric** (default: "L2"): The distance metric to use. By default it uses euclidean distance "`L2`".
+We also support "cosine" and "dot" distance as well.
+- **num_partitions** (default: 256): The number of partitions of the index.
+- **num_sub_vectors** (default: 96): The number of sub-vectors (M) that will be created during Product Quantization (PQ).
+For D dimensional vector, it will be divided into `M` of `D/M` sub-vectors, each of which is presented by
+a single PQ code.
+
+<figure markdown>
+  ![IVF PQ](./assets/ivf_pq.png)
+  <figcaption>IVF_PQ index with <code>num_partitions=2, num_sub_vectors=4</code></figcaption>
+</figure>
+
+### Use GPU to build vector index
+
+Lance Python SDK has experimental GPU support for creating IVF index.
+You can specify the GPU device to train IVF partitions via
+
+- **accelerator**: Specify to `"cuda"`` to enable GPU training.
+
+## Querying an ANN Index
+
+Querying vector indexes is done via the [search](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.search) function.
+
+There are a couple of parameters that can be used to fine-tune the search:
+
+- **limit** (default: 10): The amount of results that will be returned
+- **nprobes** (default: 20): The number of probes used. A higher number makes search more accurate but also slower.<br/>
+  Most of the time, setting nprobes to cover 5-10% of the dataset should achieve high recall with low latency.<br/>
+  e.g., for 1M vectors divided up into 256 partitions, nprobes should be set to ~20-40.<br/>
+  Note: nprobes is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
+- **refine_factor** (default: None): Refine the results by reading extra elements and re-ranking them in memory.<br/>
+  A higher number makes search more accurate but also slower. If you find the recall is less than ideal, try refine_factor=10 to start.<br/>
+  e.g., for 1M vectors divided into 256 partitions, if you're looking for top 20, then refine_factor=200 reranks the whole partition.<br/>
+  Note: refine_factor is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
+
+=== "Python"
+     ```python
+     tbl.search(np.random.random((1536))) \
+         .limit(2) \
+         .nprobes(20) \
+         .refine_factor(10) \
+         .to_pandas()
+     ```
+     ```
+                                              vector       item       _distance
+     0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
+     1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
+     ```
+
+=== "Javascript"
+     ```javascript
+     const results_1 = await table
+         .search(Array(1536).fill(1.2))
+         .limit(2)
+         .nprobes(20)
+         .refineFactor(10)
+         .execute()
+     ```
+
+The search will return the data requested in addition to the distance of each item.
+
+
+### Filtering (where clause)
+
+You can further filter the elements returned by a search using a where clause.
+
+=== "Python"
+     ```python
+     tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
+     ```
+
+=== "Javascript"
+     ```javascript
+     const results_2 = await table
+         .search(Array(1536).fill(1.2))
+         .where("id != '1141'")
+         .execute()
+     ```
+
+### Projections (select clause)
+
+You can select the columns returned by the query using a select clause.
+
+=== "Python"
+     ```python
+     tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
+     ```
+     ```
+        vector                                             _distance
+     0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
+     1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
+     ...
+     ```
+
+=== "Javascript"
+     ```javascript
+     const results_3 = await table
+         .search(Array(1536).fill(1.2))
+         .select(["id"])
+         .execute()
+     ```
+
+## FAQ
+
+### When is it necessary to create an ANN vector index?
+
+`LanceDB` has manually-tuned SIMD code for computing vector distances.
+In our benchmarks, computing 100K pairs of 1K dimension vectors takes **less than 20ms**.
+For small datasets (< 100K rows) or applications that can accept 100ms latency, vector indices are usually not necessary.
+
+For large-scale or higher dimension vectors, it is beneficial to create vector index.
+
+### How big is my index, and how many memory will it take?
+
+In LanceDB, all vector indices are **disk-based**, meaning that when responding to a vector query, only the relevant pages from the index file are loaded from disk and cached in memory. Additionally, each sub-vector is usually encoded into 1 byte PQ code.
+
+For example, with a 1024-dimension dataset, if we choose `num_sub_vectors=64`, each sub-vector has `1024 / 64 = 16` float32 numbers.
+Product quantization can lead to approximately `16 * sizeof(float32) / 1 = 64` times of space reduction.
+
+### How to choose `num_partitions` and `num_sub_vectors` for `IVF_PQ` index?
+
+`num_partitions` is used to decide how many partitions the first level `IVF` index uses.
+Higher number of partitions could lead to more efficient I/O during queries and better accuracy, but it takes much more time to train.
+On `SIFT-1M` dataset, our benchmark shows that keeping each partition 1K-4K rows lead to a good latency / recall.
+
+`num_sub_vectors` specifies how many Product Quantization (PQ) short codes to generate on each vector. Because
+PQ is a lossy compression of the original vector, a higher `num_sub_vectors` usually results in
+less space distortion, and thus yields better accuracy. However, a higher `num_sub_vectors` also causes heavier I/O and
+more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.

docs/src/assets/colab.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="117" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="117" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0h30v20H0z"/><path fill="#007ec6" d="M30 0h87v20H30z"/><path fill="url(#b)" d="M0 0h117v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="110"><svg x="4px" y="0px" width="22px" height="20px" viewBox="-2 0 28 24" style="background-color: #fff;border-radius: 1px;"><path style="fill:#e8710a;" d="M1.977,16.77c-2.667-2.277-2.605-7.079,0-9.357C2.919,8.057,3.522,9.075,4.49,9.691c-1.152,1.6-1.146,3.201-0.004,4.803C3.522,15.111,2.918,16.126,1.977,16.77z"/><path style="fill:#f9ab00;" d="M12.257,17.114c-1.767-1.633-2.485-3.658-2.118-6.02c0.451-2.91,2.139-4.893,4.946-5.678c2.565-0.718,4.964-0.217,6.878,1.819c-0.884,0.743-1.707,1.547-2.434,2.446C18.488,8.827,17.319,8.435,16,8.856c-2.404,0.767-3.046,3.241-1.494,5.644c-0.241,0.275-0.493,0.541-0.721,0.826C13.295,15.939,12.511,16.3,12.257,17.114z"/><path style="fill:#e8710a;" d="M19.529,9.682c0.727-0.899,1.55-1.703,2.434-2.446c2.703,2.783,2.701,7.031-0.005,9.764c-2.648,2.674-6.936,2.725-9.701,0.115c0.254-0.814,1.038-1.175,1.528-1.788c0.228-0.285,0.48-0.552,0.721-0.826c1.053,0.916,2.254,1.268,3.6,0.83C20.502,14.551,21.151,11.927,19.529,9.682z"/><path style="fill:#f9ab00;" d="M4.49,9.691C3.522,9.075,2.919,8.057,1.977,7.413c2.209-2.398,5.721-2.942,8.476-1.355c0.555,0.32,0.719,0.606,0.285,1.128c-0.157,0.188-0.258,0.422-0.391,0.631c-0.299,0.47-0.509,1.067-0.929,1.371C8.933,9.539,8.523,8.847,8.021,8.746C6.673,8.475,5.509,8.787,4.49,9.691z"/><path style="fill:#f9ab00;" d="M1.977,16.77c0.941-0.644,1.545-1.659,2.509-2.277c1.373,1.152,2.85,1.433,4.45,0.499c0.332-0.194,0.503-0.088,0.673,0.19c0.386,0.635,0.753,1.285,1.181,1.89c0.34,0.48,0.222,0.715-0.253,1.006C7.84,19.73,4.205,19.188,1.977,16.77z"/></svg><text x="245" y="140" transform="scale(.1)" textLength="30"> </text><text x="725" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="770">Open in Colab</text><text x="725" y="140" transform="scale(.1)" textLength="770">Open in Colab</text></g> </svg>

docs/src/assets/ghost.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="88.25" height="28" role="img" aria-label="GHOST"><title>GHOST</title><g shape-rendering="crispEdges"><rect width="88.25" height="28" fill="#000"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="100"><image x="9" y="7" width="14" height="14" xlink:href="data:image/svg+xml;base64,PHN2ZyBmaWxsPSIjZjdkZjFlIiByb2xlPSJpbWciIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+R2hvc3Q8L3RpdGxlPjxwYXRoIGQ9Ik0xMiAwQzUuMzczIDAgMCA1LjM3MyAwIDEyczUuMzczIDEyIDEyIDEyIDEyLTUuMzczIDEyLTEyUzE4LjYyNyAwIDEyIDB6bS4yNTYgMi4zMTNjMi40Ny4wMDUgNS4xMTYgMi4wMDggNS44OTggMi45NjJsLjI0NC4zYzEuNjQgMS45OTQgMy41NjkgNC4zNCAzLjU2OSA2Ljk2NiAwIDMuNzE5LTIuOTggNS44MDgtNi4xNTggNy41MDgtMS40MzMuNzY2LTIuOTggMS41MDgtNC43NDggMS41MDgtNC41NDMgMC04LjM2Ni0zLjU2OS04LjM2Ni04LjExMiAwLS43MDYuMTctMS40MjUuMzQyLTIuMTUuMTIyLS41MTUuMjQ0LTEuMDMzLjMwNy0xLjU0OS41NDgtNC41MzkgMi45NjctNi43OTUgOC40MjItNy40MDhhNC4yOSA0LjI5IDAgMDEuNDktLjAyNloiLz48L3N2Zz4="/><text transform="scale(.1)" x="541.25" y="175" textLength="442.5" fill="#fff" font-weight="bold">GHOST</text></g></svg>

docs/src/assets/github.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="95.5" height="28" role="img" aria-label="GITHUB"><title>GITHUB</title><g shape-rendering="crispEdges"><rect width="95.5" height="28" fill="#121011"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="100"><image x="9" y="7" width="14" height="14" xlink:href="data:image/svg+xml;base64,PHN2ZyBmaWxsPSJ3aGl0ZSIgcm9sZT0iaW1nIiB2aWV3Qm94PSIwIDAgMjQgMjQiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+PHRpdGxlPkdpdEh1YjwvdGl0bGU+PHBhdGggZD0iTTEyIC4yOTdjLTYuNjMgMC0xMiA1LjM3My0xMiAxMiAwIDUuMzAzIDMuNDM4IDkuOCA4LjIwNSAxMS4zODUuNi4xMTMuODItLjI1OC44Mi0uNTc3IDAtLjI4NS0uMDEtMS4wNC0uMDE1LTIuMDQtMy4zMzguNzI0LTQuMDQyLTEuNjEtNC4wNDItMS42MUM0LjQyMiAxOC4wNyAzLjYzMyAxNy43IDMuNjMzIDE3LjdjLTEuMDg3LS43NDQuMDg0LS43MjkuMDg0LS43MjkgMS4yMDUuMDg0IDEuODM4IDEuMjM2IDEuODM4IDEuMjM2IDEuMDcgMS44MzUgMi44MDkgMS4zMDUgMy40OTUuOTk4LjEwOC0uNzc2LjQxNy0xLjMwNS43Ni0xLjYwNS0yLjY2NS0uMy01LjQ2Ni0xLjMzMi01LjQ2Ni01LjkzIDAtMS4zMS40NjUtMi4zOCAxLjIzNS0zLjIyLS4xMzUtLjMwMy0uNTQtMS41MjMuMTA1LTMuMTc2IDAgMCAxLjAwNS0uMzIyIDMuMyAxLjIzLjk2LS4yNjcgMS45OC0uMzk5IDMtLjQwNSAxLjAyLjAwNiAyLjA0LjEzOCAzIC40MDUgMi4yOC0xLjU1MiAzLjI4NS0xLjIzIDMuMjg1LTEuMjMuNjQ1IDEuNjUzLjI0IDIuODczLjEyIDMuMTc2Ljc2NS44NCAxLjIzIDEuOTEgMS4yMyAzLjIyIDAgNC42MS0yLjgwNSA1LjYyNS01LjQ3NSA1LjkyLjQyLjM2LjgxIDEuMDk2LjgxIDIuMjIgMCAxLjYwNi0uMDE1IDIuODk2LS4wMTUgMy4yODYgMCAuMzE1LjIxLjY5LjgyNS41N0MyMC41NjUgMjIuMDkyIDI0IDE3LjU5MiAyNCAxMi4yOTdjMC02LjYyNy01LjM3My0xMi0xMi0xMiIvPjwvc3ZnPg=="/><text transform="scale(.1)" x="577.5" y="175" textLength="515" fill="#fff" font-weight="bold">GITHUB</text></g></svg>