expose scalar index

Please note: this is not tested as we don't have a server here and
testing against a mock object wouldn't be that interesting.

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.6
+current_version = 0.3.7
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True

Cargo.toml

@@ -5,9 +5,10 @@ exclude = ["python"]
 resolver = "2"
 
 [workspace.dependencies]
-lance = { "version" = "=0.8.14", "features" = ["dynamodb"] }
-lance-linalg = { "version" = "=0.8.14" }
-lance-testing = { "version" = "=0.8.14" }
+lance = { "version" = "=0.8.16", "features" = ["dynamodb"] }
+lance-linalg = { "version" = "=0.8.16" }
+lance-index = { "version" = "=0.8.16" }
+lance-testing = { "version" = "=0.8.16" }
 # Note that this one does not include pyarrow
 arrow = { version = "47.0.0", optional = false }
 arrow-array = "47.0"

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.3.6",
+  "version": "0.3.7",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.3.6",
+      "version": "0.3.7",
       "cpu": [
         "x64",
         "arm64"
@@ -53,11 +53,11 @@
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.3.6",
-        "@lancedb/vectordb-darwin-x64": "0.3.6",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.3.6",
-        "@lancedb/vectordb-linux-x64-gnu": "0.3.6",
-        "@lancedb/vectordb-win32-x64-msvc": "0.3.6"
+        "@lancedb/vectordb-darwin-arm64": "0.3.7",
+        "@lancedb/vectordb-darwin-x64": "0.3.7",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.3.7",
+        "@lancedb/vectordb-linux-x64-gnu": "0.3.7",
+        "@lancedb/vectordb-win32-x64-msvc": "0.3.7"
       }
     },
     "node_modules/@apache-arrow/ts": {
@@ -317,9 +317,9 @@
       }
     },
     "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.3.6",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.3.6.tgz",
-      "integrity": "sha512-GR5v+4kHUCZ71gVxd3mLsUdlreXPUIbvBgvr+BmEXRbLfc7+JsFUjsRgxmoctQ0mXxkW67Sl7v6kQCWcBLCk/Q==",
+      "version": "0.3.7",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.3.7.tgz",
+      "integrity": "sha512-QsDxcbhrumJg+Cyflpnj8EY+bZojbco5K7VSeKvguqeXUGb62ksyOZuUTCn2sqJaCgy1KZ1qC5U8jBqfgZHc2w==",
       "cpu": [
         "arm64"
       ],
@@ -329,9 +329,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.3.6",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.3.6.tgz",
-      "integrity": "sha512-4qemi4jUXG8jOk7ecECmb0+5Nm0n7YF5/1X9/5uc81I+4What+yhZE9nEsmCGRBqmtuQXkYl35ePvQgj3rCQjQ==",
+      "version": "0.3.7",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.3.7.tgz",
+      "integrity": "sha512-fgv10kI04UycgpmhJLUcCswgvSdgsGuj65o+W5usmVdxYZiWpoXBBXRkWYMjUX5RNe3mY1Ff6QPBbToR0WkSUA==",
       "cpu": [
         "x64"
       ],
@@ -341,9 +341,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.3.6",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.3.6.tgz",
-      "integrity": "sha512-I/lFqIUcXYxJnUG5+DILzUzcfHRGHXL3kl5bs1MGkR9a7F3oPx1IAwY9wkskVnClM7XF9H7MVcFRVTjHUqoUwA==",
+      "version": "0.3.7",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.3.7.tgz",
+      "integrity": "sha512-pvw+31+VKEH3YmS/GLKzEGt/Y2+c/IaE6JL6tIjXi2KY+ZcWuyyXpYnYiHHDw2EP7ubKj6+fKIG1P9tlxMcGMQ==",
       "cpu": [
         "arm64"
       ],
@@ -353,9 +353,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.3.6",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.3.6.tgz",
-      "integrity": "sha512-UTA/4bpA3UoByhfDx//S5m4o6uQ1qfpneD0PbuftAjkt9eHg0ABIEpZdiTI3xUBdrjXSKZtpVTxOin9X39IBKQ==",
+      "version": "0.3.7",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.3.7.tgz",
+      "integrity": "sha512-kHFURhfhJRqw4k1auseqQgOzAHB4oYpyzLCX3TCR3uTxqRQ7gFxxlO0TnIcwNRqLcGb9GmWxWWoR8k1CdCXrMw==",
       "cpu": [
         "x64"
       ],
@@ -365,9 +365,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.3.6",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.3.6.tgz",
-      "integrity": "sha512-70IS0TX4BpjSX4GP1Pq835cqQ5LZpfOJuBNtGv93OxMTWTVQUxtp2MLNwOR6OJMGNQz6q84NNKrKOSf15ZGwGg==",
+      "version": "0.3.7",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.3.7.tgz",
+      "integrity": "sha512-zWfZ557v2Y+93dVrmqqnbiLeTOb0ptunAG0zGjyE+3oyi8j/4+bL56Fdv94k+dfNF4KrcqcULEcZhKik3/FQ9w==",
       "cpu": [
         "x64"
       ],
@@ -4869,33 +4869,33 @@
       }
     },
     "@lancedb/vectordb-darwin-arm64": {
-      "version": "0.3.6",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.3.6.tgz",
-      "integrity": "sha512-GR5v+4kHUCZ71gVxd3mLsUdlreXPUIbvBgvr+BmEXRbLfc7+JsFUjsRgxmoctQ0mXxkW67Sl7v6kQCWcBLCk/Q==",
+      "version": "0.3.7",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.3.7.tgz",
+      "integrity": "sha512-QsDxcbhrumJg+Cyflpnj8EY+bZojbco5K7VSeKvguqeXUGb62ksyOZuUTCn2sqJaCgy1KZ1qC5U8jBqfgZHc2w==",
       "optional": true
     },
     "@lancedb/vectordb-darwin-x64": {
-      "version": "0.3.6",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.3.6.tgz",
-      "integrity": "sha512-4qemi4jUXG8jOk7ecECmb0+5Nm0n7YF5/1X9/5uc81I+4What+yhZE9nEsmCGRBqmtuQXkYl35ePvQgj3rCQjQ==",
+      "version": "0.3.7",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.3.7.tgz",
+      "integrity": "sha512-fgv10kI04UycgpmhJLUcCswgvSdgsGuj65o+W5usmVdxYZiWpoXBBXRkWYMjUX5RNe3mY1Ff6QPBbToR0WkSUA==",
       "optional": true
     },
     "@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.3.6",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.3.6.tgz",
-      "integrity": "sha512-I/lFqIUcXYxJnUG5+DILzUzcfHRGHXL3kl5bs1MGkR9a7F3oPx1IAwY9wkskVnClM7XF9H7MVcFRVTjHUqoUwA==",
+      "version": "0.3.7",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.3.7.tgz",
+      "integrity": "sha512-pvw+31+VKEH3YmS/GLKzEGt/Y2+c/IaE6JL6tIjXi2KY+ZcWuyyXpYnYiHHDw2EP7ubKj6+fKIG1P9tlxMcGMQ==",
       "optional": true
     },
     "@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.3.6",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.3.6.tgz",
-      "integrity": "sha512-UTA/4bpA3UoByhfDx//S5m4o6uQ1qfpneD0PbuftAjkt9eHg0ABIEpZdiTI3xUBdrjXSKZtpVTxOin9X39IBKQ==",
+      "version": "0.3.7",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.3.7.tgz",
+      "integrity": "sha512-kHFURhfhJRqw4k1auseqQgOzAHB4oYpyzLCX3TCR3uTxqRQ7gFxxlO0TnIcwNRqLcGb9GmWxWWoR8k1CdCXrMw==",
       "optional": true
     },
     "@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.3.6",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.3.6.tgz",
-      "integrity": "sha512-70IS0TX4BpjSX4GP1Pq835cqQ5LZpfOJuBNtGv93OxMTWTVQUxtp2MLNwOR6OJMGNQz6q84NNKrKOSf15ZGwGg==",
+      "version": "0.3.7",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.3.7.tgz",
+      "integrity": "sha512-zWfZ557v2Y+93dVrmqqnbiLeTOb0ptunAG0zGjyE+3oyi8j/4+bL56Fdv94k+dfNF4KrcqcULEcZhKik3/FQ9w==",
       "optional": true
     },
     "@neon-rs/cli": {

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.3.6",
+  "version": "0.3.7",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -81,10 +81,10 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.3.6",
-    "@lancedb/vectordb-darwin-x64": "0.3.6",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.3.6",
-    "@lancedb/vectordb-linux-x64-gnu": "0.3.6",
-    "@lancedb/vectordb-win32-x64-msvc": "0.3.6"
+    "@lancedb/vectordb-darwin-arm64": "0.3.7",
+    "@lancedb/vectordb-darwin-x64": "0.3.7",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.3.7",
+    "@lancedb/vectordb-linux-x64-gnu": "0.3.7",
+    "@lancedb/vectordb-win32-x64-msvc": "0.3.7"
   }
 }

node/src/remote/client.ts

@@ -89,7 +89,8 @@ export class HttpLancedbClient {
       {
         headers: {
           'Content-Type': 'application/json',
-          'x-api-key': this._apiKey()
+          'x-api-key': this._apiKey(),
+          ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
         },
         params,
         timeout: 10000

node/src/remote/index.ts

@@ -237,7 +237,8 @@ export class RemoteTable<T = number[]> implements Table<T> {
   }
 
   async countRows (): Promise<number> {
-    throw new Error('Not implemented')
+    const result = await this._client.post(`/v1/table/${this._name}/describe/`)
+    return result.data?.stats?.num_rows
   }
 
   async delete (filter: string): Promise<void> {

python/lancedb/embeddings/base.py

@@ -33,6 +33,7 @@ class EmbeddingFunction(BaseModel, ABC):
     3. ndims method which returns the number of dimensions of the vector column
     """
 
+    __slots__ = ("__weakref__",)  # pydantic 1.x compatibility
     max_retries: int = (
         7  # Setitng 0 disables retires. Maybe this should not be enabled by default,
     )

python/lancedb/remote/table.py

@@ -44,6 +44,14 @@ class RemoteTable(Table):
         schema = json_to_schema(resp["schema"])
         return schema
 
+    @property
+    def version(self) -> int:
+        """Get the current version of the table"""
+        resp = self._conn._loop.run_until_complete(
+            self._conn._client.post(f"/v1/table/{self._name}/describe/")
+        )
+        return resp["version"]
+
     def to_arrow(self) -> pa.Table:
         """Return the table as an Arrow table."""
         raise NotImplementedError("to_arrow() is not supported on the LanceDB cloud")
@@ -64,7 +72,54 @@ class RemoteTable(Table):
         replace: bool = True,
         accelerator: Optional[str] = None,
     ):
-        raise NotImplementedError
+        """Create an index on the table.
+        Currently, the only parameters that matter are
+        the metric and the vector column name.
+
+        Parameters
+        ----------
+        metric : str
+            The metric to use for the index. Default is "L2".
+        num_partitions : int
+            The number of partitions to use for the index. Default is 256.
+        num_sub_vectors : int
+            The number of sub-vectors to use for the index. Default is 96.
+        vector_column_name : str
+            The name of the vector column. Default is "vector".
+        replace : bool
+            Whether to replace the existing index. Default is True.
+
+        Examples
+        --------
+        import lancedb
+        import uuid
+        from lancedb.schema import vector
+        conn = lancedb.connect("db://...", api_key="...", region="...")
+        table_name = uuid.uuid4().hex
+        schema = pa.schema(
+            [
+                    pa.field("id", pa.uint32(), False),
+                    pa.field("vector", vector(128), False),
+                    pa.field("s", pa.string(), False),
+            ]
+        )
+        table = conn.create_table(
+            table_name,
+            schema=schema,
+        )
+        table.create_index()
+        """
+        index_type = "vector"
+
+        data = {
+            "column": vector_column_name,
+            "index_type": index_type,
+            "metric_type": metric,
+        }
+        resp = self._conn._loop.run_until_complete(
+            self._conn._client.post(f"/v1/table/{self._name}/create_index/", data=data)
+        )
+        return resp
 
     def add(
         self,

rust/ffi/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb-node"
-version = "0.3.6"
+version = "0.3.7"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
 edition = "2018"
@@ -20,6 +20,7 @@ futures = "0.3"
 half = { workspace = true }
 lance = { workspace = true }
 lance-linalg = { workspace = true }
+lance-index = { workspace = true }
 vectordb = { path = "../../vectordb" }
 tokio = { version = "1.23", features = ["rt-multi-thread"] }
 neon = {version = "0.10.1", default-features = false, features = ["channel-api", "napi-6", "promise-api", "task-api"] }

rust/ffi/node/src/index/vector.rs

@@ -12,7 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use lance::index::vector::{ivf::IvfBuildParams, pq::PQBuildParams};
+use lance::index::vector::pq::PQBuildParams;
+use lance_index::vector::ivf::IvfBuildParams;
 use lance_linalg::distance::MetricType;
 use neon::context::FunctionContext;
 use neon::prelude::*;

rust/ffi/node/src/lib.rs

@@ -180,12 +180,13 @@ fn database_open_table(mut cx: FunctionContext) -> JsResult<JsPromise> {
 
     let aws_creds = get_aws_creds(&mut cx, 1)?;
 
-    let aws_region = get_aws_region(&mut cx, 4)?;
+    let _aws_region = get_aws_region(&mut cx, 4)?;
 
     let params = ReadParams {
-        store_options: Some(ObjectStoreParams::with_aws_credentials(
-            aws_creds, aws_region,
-        )),
+        store_options: Some(ObjectStoreParams{
+            aws_credentials: aws_creds,
+            ..ObjectStoreParams::default()
+        }),
         ..ReadParams::default()
     };
 

rust/ffi/node/src/table.rs

@@ -64,12 +64,13 @@ impl JsTable {
         let database = db.database.clone();
 
         let aws_creds = get_aws_creds(&mut cx, 3)?;
-        let aws_region = get_aws_region(&mut cx, 6)?;
+        let _aws_region = get_aws_region(&mut cx, 6)?;
 
         let params = WriteParams {
-            store_params: Some(ObjectStoreParams::with_aws_credentials(
-                aws_creds, aws_region,
-            )),
+            store_params: Some(ObjectStoreParams{
+                aws_credentials: aws_creds,
+                ..ObjectStoreParams::default()
+            }),
             mode,
             ..WriteParams::default()
         };
@@ -106,12 +107,13 @@ impl JsTable {
             s => return cx.throw_error(format!("invalid write mode {}", s)),
         };
         let aws_creds = get_aws_creds(&mut cx, 2)?;
-        let aws_region = get_aws_region(&mut cx, 5)?;
+        let _aws_region = get_aws_region(&mut cx, 5)?;
 
         let params = WriteParams {
-            store_params: Some(ObjectStoreParams::with_aws_credentials(
-                aws_creds, aws_region,
-            )),
+            store_params: Some(ObjectStoreParams{
+                aws_credentials: aws_creds,
+                ..ObjectStoreParams::default()
+            }),
             mode: write_mode,
             ..WriteParams::default()
         };

rust/vectordb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb"
-version = "0.3.6"
+version = "0.3.7"
 edition = "2021"
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
@@ -22,6 +22,7 @@ snafu = { workspace = true }
 half = { workspace = true }
 lance = { workspace = true }
 lance-linalg = { workspace = true }
+lance-index = { workspace = true }
 lance-testing = { workspace = true }
 tokio = { version = "1.23", features = ["rt-multi-thread"] }
 log = { workspace = true }

rust/vectordb/src/index/vector.rs

@@ -13,9 +13,9 @@
 // limitations under the License.
 
 use lance::format::{Index, Manifest};
-use lance::index::vector::ivf::IvfBuildParams;
 use lance::index::vector::pq::PQBuildParams;
 use lance::index::vector::VectorIndexParams;
+use lance_index::vector::ivf::IvfBuildParams;
 use lance_linalg::distance::MetricType;
 
 pub trait VectorIndexBuilder {
@@ -136,7 +136,6 @@ impl VectorIndex {
 mod tests {
     use super::*;
 
-    use lance::index::vector::ivf::IvfBuildParams;
     use lance::index::vector::pq::PQBuildParams;
     use lance::index::vector::StageParams;
 

rust/vectordb/src/table.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use chrono::Duration;
+use lance_index::IndexType;
 use std::sync::Arc;
 
 use arrow_array::{Float32Array, RecordBatchReader};
@@ -22,7 +23,7 @@ use lance::dataset::optimize::{
     compact_files, CompactionMetrics, CompactionOptions, IndexRemapperOptions,
 };
 use lance::dataset::{Dataset, WriteParams};
-use lance::index::{DatasetIndexExt, IndexType};
+use lance::index::DatasetIndexExt;
 use lance::io::object_store::WrappingObjectStore;
 use std::path::Path;