[python] Bump version: 0.1.13 → 0.1.14

- Move native packages to @lancedb org
- Move package-lock.json update to a reusable action and created a target to run it manually.

.github/workflows/make-release-commit.yml

@@ -25,38 +25,25 @@ jobs:
   bump-version:
     runs-on: ubuntu-latest
     steps:
-    - name: Check out main
+      - name: Check out main
-      uses: actions/checkout@v3
+        uses: actions/checkout@v3
-      with:
+        with:
-        ref: main
+          ref: main
-        persist-credentials: false
+          fetch-depth: 0
-        fetch-depth: 0
+          lfs: true
-        lfs: true
+      - name: Set git configs for bumpversion
-    - name: Set git configs for bumpversion
+        shell: bash
-      shell: bash
+        run: |
-      run: |
+          git config user.name 'Lance Release'
-        git config user.name 'Lance Release'
+          git config user.email 'lance-dev@lancedb.com'
-        git config user.email 'lance-dev@lancedb.com'
+      - name: Set up Python 3.10
-    - name: Set up Python 3.10
+        uses: actions/setup-python@v4
-      uses: actions/setup-python@v4
+        with:
-      with:
+          python-version: "3.10"
-        python-version: "3.10"
+      - name: Bump version, create tag and commit
-    - name: Bump version, create tag and commit
+        run: |
-      run: |
+          pip install bump2version
-        pip install bump2version
+          bumpversion --verbose ${{ inputs.part }}
-        bumpversion --verbose ${{ inputs.part }}
+          git push
-    - name: Update package-lock.json file
+      - uses: ./.github/workflows/update_package_lock
-      run: |
-        npm install
-        git add package-lock.json
-        # Add this change to the commit created by bumpversion
-        git commit --amend --no-edit
-      working-directory: node
-    - name: Push new version and tag
-      if: ${{ inputs.dry_run }} == "false"
-      uses: ad-m/github-push-action@master
-      with:
-        github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-        branch: main
-        tags: true
 

.github/workflows/node.yml

@@ -70,7 +70,7 @@ jobs:
         npm run tsc
         npm run build
         npm run pack-build
-        npm install --no-save ./dist/vectordb-*.tgz
+        npm install --no-save ./dist/lancedb-vectordb-*.tgz
         # Remove index.node to test with dependency installed
         rm index.node
     - name: Test
@@ -101,7 +101,7 @@ jobs:
         npm run tsc
         npm run build
         npm run pack-build
-        npm install --no-save ./dist/vectordb-*.tgz
+        npm install --no-save ./dist/lancedb-vectordb-*.tgz
         # Remove index.node to test with dependency installed
         rm index.node
     - name: Test

.github/workflows/npm-publish.yml

@@ -35,7 +35,7 @@ jobs:
         with:
           name: node-package
           path: |
-            node/vectordb-*.tgz
+            node/lancedb-vectordb-*.tgz
 
   node-macos:
     runs-on: macos-12
@@ -46,25 +46,25 @@ jobs:
       matrix:
         target: [x86_64-apple-darwin, aarch64-apple-darwin]
     steps:
-    - name: Checkout
+      - name: Checkout
-      uses: actions/checkout@v3
+        uses: actions/checkout@v3
-    - name: Install system dependencies
+      - name: Install system dependencies
-      run: brew install protobuf
+        run: brew install protobuf
-    - name: Install npm dependencies
+      - name: Install npm dependencies
-      run: |
+        run: |
-        cd node
+          cd node
-        npm ci
+          npm ci
-    - name: Install rustup target
+      - name: Install rustup target
-      if: ${{ matrix.target == 'aarch64-apple-darwin' }}
+        if: ${{ matrix.target == 'aarch64-apple-darwin' }}
-      run: rustup target add aarch64-apple-darwin
+        run: rustup target add aarch64-apple-darwin
-    - name: Build MacOS native node modules
+      - name: Build MacOS native node modules
-      run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
+        run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
-    - name: Upload Darwin Artifacts
+      - name: Upload Darwin Artifacts
-      uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v3
-      with:
+        with:
-        name: darwin-native
+          name: native-darwin
-        path: |
+          path: |
-          node/dist/vectordb-darwin*.tgz
+            node/dist/lancedb-vectordb-darwin*.tgz
 
   node-linux:
     name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }})
@@ -84,37 +84,37 @@ jobs:
           # Building on aarch64 is too slow for now
           # - aarch64
     steps:
-    - name: Checkout
+      - name: Checkout
-      uses: actions/checkout@v3
+        uses: actions/checkout@v3
-    - name: Change owner to root (for npm)
+      - name: Change owner to root (for npm)
-      # The docker container is run as root, so we need the files to be owned by root
+        # The docker container is run as root, so we need the files to be owned by root
-      # Otherwise npm is a nightmare: https://github.com/npm/cli/issues/3773
+        # Otherwise npm is a nightmare: https://github.com/npm/cli/issues/3773
-      run: sudo chown -R root:root .
+        run: sudo chown -R root:root .
-    - name: Set up QEMU
+      - name: Set up QEMU
-      if: ${{ matrix.arch == 'aarch64' }}
+        if: ${{ matrix.arch == 'aarch64' }}
-      uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v2
-      with:
+        with:
-        platforms: arm64
+          platforms: arm64
-    - name: Build Linux GNU native node modules
+      - name: Build Linux GNU native node modules
-      if: ${{ matrix.libc == 'gnu' }}
+        if: ${{ matrix.libc == 'gnu' }}
-      run: |
+        run: |
-        docker run \
+          docker run \
-          -v $(pwd):/io -w /io \
+            -v $(pwd):/io -w /io \
-          rust:1.70-bookworm \
+            rust:1.70-bookworm \
-          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu
+            bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu
-    - name: Build musl Linux native node modules
+      - name: Build musl Linux native node modules
-      if: ${{ matrix.libc == 'musl' }}
+        if: ${{ matrix.libc == 'musl' }}
-      run: |
+        run: |
-        docker run --platform linux/arm64/v8 \
+          docker run --platform linux/arm64/v8 \
-          -v $(pwd):/io -w /io \
+            -v $(pwd):/io -w /io \
-          quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
+            quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
-          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
+            bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
-    - name: Upload Linux Artifacts
+      - name: Upload Linux Artifacts
-      uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v3
-      with:
+        with:
-        name: linux-native
+          name: native-linux
-        path: |
+          path: |
-          node/dist/vectordb-linux*.tgz
+            node/dist/lancedb-vectordb-linux*.tgz
 
   node-windows:
     runs-on: windows-2022
@@ -145,12 +145,12 @@ jobs:
       - name: Upload Windows Artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: windows-native
+          name: native-windows
           path: |
-            node/dist/vectordb-win32*.tgz
+            node/dist/lancedb-vectordb-win32*.tgz
 
   release:
-    needs: [node, node-macos, node-linux]
+    needs: [node, node-macos, node-linux, node-windows]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -170,3 +170,11 @@ jobs:
           for filename in *.tgz; do
             npm publish $filename
           done
+
+  update-package-lock:
+    needs: [release]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: ./.github/workflows/update_package_lock

.github/workflows/update_package_lock/action.yml

@@ -0,0 +1,22 @@
+name: update_package_lock
+description: "Update node's package.lock"
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 20
+    - name: Set git configs
+      shell: bash
+      run: |
+        git config user.name 'Lance Release'
+        git config user.email 'lance-dev@lancedb.com'
+    - name: Update package-lock.json file
+      working-directory: ./node
+      run: |
+        npm install
+        git add package-lock.json
+        git commit -m "Updating package-lock.json"
+        git push
+      shell: bash

.github/workflows/update_package_lock_run.yml

@@ -0,0 +1,12 @@
+name: Update package-lock.json
+
+on:
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: ./.github/workflows/update_package_lock

node/native.js

@@ -17,7 +17,7 @@ const { currentTarget } = require('@neon-rs/load');
 let nativeLib;
 
 try {
-    nativeLib = require(`vectordb-${currentTarget()}`);
+    nativeLib = require(`@lancedb/vectordb-${currentTarget()}`);
 } catch (e) {
     try {
         // Might be developing locally, so try that. But don't expose that error
@@ -25,12 +25,12 @@ try {
         nativeLib = require("./index.node");
     } catch {
         throw new Error(`vectordb: failed to load native library.
-  You may need to run \`npm install vectordb-${currentTarget()}\`.
+  You may need to run \`npm install @lancedb/vectordb-${currentTarget()}\`.
-      
+
   If that does not work, please file a bug report at https://github.com/lancedb/lancedb/issues
       
   Source error: ${e}`);
-    }   
+    }
 }
 
 // Dynamic require for runtime.

node/package-lock.json

@@ -51,11 +51,11 @@
         "typescript": "*"
       },
       "optionalDependencies": {
-        "vectordb-darwin-arm64": "0.1.15",
+        "@lancedb/vectordb-darwin-arm64": "0.1.15",
-        "vectordb-darwin-x64": "0.1.15",
+        "@lancedb/vectordb-darwin-x64": "0.1.15",
-        "vectordb-linux-arm64-gnu": "0.1.15",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.1.15",
-        "vectordb-linux-x64-gnu": "0.1.15",
+        "@lancedb/vectordb-linux-x64-gnu": "0.1.15",
-        "vectordb-win32-x64-msvc": "0.1.15"
+        "@lancedb/vectordb-win32-x64-msvc": "0.1.15"
       }
     },
     "node_modules/@apache-arrow/ts": {

node/package.json

@@ -70,18 +70,18 @@
   ],
   "neon": {
     "targets": {
-      "x86_64-apple-darwin": "vectordb-darwin-x64",
+      "x86_64-apple-darwin": "@lancedb/vectordb-darwin-x64",
-      "aarch64-apple-darwin": "vectordb-darwin-arm64",
+      "aarch64-apple-darwin": "@lancedb/vectordb-darwin-arm64",
-      "x86_64-unknown-linux-gnu": "vectordb-linux-x64-gnu",
+      "x86_64-unknown-linux-gnu": "@lancedb/vectordb-linux-x64-gnu",
-      "aarch64-unknown-linux-gnu": "vectordb-linux-arm64-gnu",
+      "aarch64-unknown-linux-gnu": "@lancedb/vectordb-linux-arm64-gnu",
-      "x86_64-pc-windows-msvc": "vectordb-win32-x64-msvc"
+      "x86_64-pc-windows-msvc": "@lancedb/vectordb-win32-x64-msvc"
     }
   },
   "optionalDependencies": {
-    "vectordb-darwin-arm64": "0.1.15",
+    "@lancedb/vectordb-darwin-arm64": "0.1.15",
-    "vectordb-darwin-x64": "0.1.15",
+    "@lancedb/vectordb-darwin-x64": "0.1.15",
-    "vectordb-linux-arm64-gnu": "0.1.15",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.1.15",
-    "vectordb-linux-x64-gnu": "0.1.15",
+    "@lancedb/vectordb-linux-x64-gnu": "0.1.15",
-    "vectordb-win32-x64-msvc": "0.1.15"
+    "@lancedb/vectordb-win32-x64-msvc": "0.1.15"
   }
 }

node/src/remote/client.ts

@@ -41,7 +41,7 @@ export class HttpLancedbClient {
     filter?: string
   ): Promise<ArrowTable<any>> {
     const response = await axios.post(
-              `${this._url}/v1/table/${tableName}`,
+              `${this._url}/v1/table/${tableName}/query/`,
               {
                 vector,
                 k,

python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.13
+current_version = 0.1.14
 commit = True
 message = [python] Bump version: {current_version} → {new_version}
 tag = True

python/lancedb/db.py

@@ -327,6 +327,6 @@ class LanceDBConnection(DBConnection):
         name: str
             The name of the table.
         """
-        filesystem, path = pa.fs.FileSystem.from_uri(self.uri)
+        filesystem, path = fs_from_uri(self.uri)
         table_path = os.path.join(path, name + ".lance")
         filesystem.delete_dir(table_path)

python/lancedb/remote/client.py

@@ -141,5 +141,7 @@ class RestfulLanceDBClient:
     @_check_not_closed
     async def query(self, table_name: str, query: VectorQuery) -> VectorQueryResult:
         """Query a table."""
-        tbl = await self.post(f"/v1/table/{table_name}/", query, deserialize=_read_ipc)
+        tbl = await self.post(
+            f"/v1/table/{table_name}/query/", query, deserialize=_read_ipc
+        )
         return VectorQueryResult(tbl)

python/lancedb/remote/table.py

@@ -33,13 +33,13 @@ class RemoteTable(Table):
         self._name = name
 
     def __repr__(self) -> str:
-        return f"RemoteTable({self._conn.db_name}.{self.name})"
+        return f"RemoteTable({self._conn.db_name}.{self._name})"
 
     @cached_property
     def schema(self) -> pa.Schema:
         """Return the schema of the table."""
         resp = self._conn._loop.run_until_complete(
-            self._conn._client.get(f"/v1/table/{self._name}/describe")
+            self._conn._client.post(f"/v1/table/{self._name}/describe/")
         )
         schema = json_to_schema(resp["schema"])
         return schema
@@ -73,7 +73,7 @@ class RemoteTable(Table):
 
         self._conn._loop.run_until_complete(
             self._conn._client.post(
-                f"/v1/table/{self._name}/insert",
+                f"/v1/table/{self._name}/insert/",
                 data=payload,
                 params={"request_id": request_id, "mode": mode},
                 content_type=ARROW_STREAM_CONTENT_TYPE,

python/lancedb/table.py

@@ -23,12 +23,12 @@ import numpy as np
 import pandas as pd
 import pyarrow as pa
 import pyarrow.compute as pc
-import pyarrow.fs
 from lance import LanceDataset
 from lance.vector import vec_to_table
 
 from .common import DATA, VEC, VECTOR_COLUMN_NAME
 from .query import LanceFtsQueryBuilder, LanceQueryBuilder, Query
+from .util import fs_from_uri
 
 
 def _sanitize_data(data, schema, on_bad_vectors, fill_value):
@@ -527,7 +527,7 @@ class LanceTable(Table):
     @classmethod
     def open(cls, db, name):
         tbl = cls(db, name)
-        fs, path = pa.fs.FileSystem.from_uri(tbl._dataset_uri)
+        fs, path = fs_from_uri(tbl._dataset_uri)
         file_info = fs.get_file_info(path)
         if file_info.type != pa.fs.FileType.Directory:
             raise FileNotFoundError(

python/lancedb/util.py

@@ -71,7 +71,8 @@ def fs_from_uri(uri: str) -> Tuple[pa_fs.FileSystem, str]:
     Get a PyArrow FileSystem from a URI, handling extra environment variables.
     """
     if get_uri_scheme(uri) == "s3":
-        if os.environ["AWS_ENDPOINT"]:
+        fs = pa_fs.S3FileSystem(endpoint_override=os.environ.get("AWS_ENDPOINT"))
-            uri += "?endpoint_override=" + os.environ["AWS_ENDPOINT"]
+        path = get_uri_location(uri)
+        return fs, path
 
     return pa_fs.FileSystem.from_uri(uri)

python/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lancedb"
-version = "0.1.13"
+version = "0.1.14"
 dependencies = ["pylance~=0.5.8", "ratelimiter", "retry", "tqdm", "aiohttp", "pydantic", "attr", "semver"]
 description = "lancedb"
 authors = [