expose scalar index

Cargo.toml

@@ -5,9 +5,10 @@ exclude = ["python"]
 resolver = "2"
 
 [workspace.dependencies]
-lance = { "version" = "=0.8.14", "features" = ["dynamodb"] }
+lance = { "version" = "=0.8.16", "features" = ["dynamodb"] }
-lance-linalg = { "version" = "=0.8.14" }
+lance-linalg = { "version" = "=0.8.16" }
-lance-testing = { "version" = "=0.8.14" }
+lance-index = { "version" = "=0.8.16" }
+lance-testing = { "version" = "=0.8.16" }
 # Note that this one does not include pyarrow
 arrow = { version = "47.0.0", optional = false }
 arrow-array = "47.0"

python/lancedb/__init__.py

@@ -22,8 +22,6 @@ from .remote.db import RemoteDBConnection
 from .schema import vector  # noqa: F401
 from .utils import sentry_log  # noqa: F401
 
-import requests
-
 
 def connect(
     uri: URI,
@@ -72,26 +70,3 @@ def connect(
             raise ValueError(f"api_key is required to connected LanceDB cloud: {uri}")
         return RemoteDBConnection(uri, api_key, region, host_override)
     return LanceDBConnection(uri)
-
-def drop_database(uri: URI, api_key: str, region: str = "us-west-2"):
-    """Drop a LanceDB database.
-
-    Parameters
-    ----------
-    uri: str or Path
-        The uri of the database.
-
-    Examples
-    --------
-
-    >>> lancedb.drop_database(uri="db://", api_key="sk_...", region="...")
-
-    """
-    if isinstance(uri, str) and uri.startswith("db://"):
-        control_plane_url = f"control-plane.{region}.api.lancedb.com"
-        requests.delete(
-            f"https://{control_plane_url}/api/v1/auth/token/delete",
-            json={"api_key": api_key}
-        )
-    return LanceDBConnection(uri).drop_database()
-

python/lancedb/remote/table.py

@@ -72,7 +72,54 @@ class RemoteTable(Table):
         replace: bool = True,
         accelerator: Optional[str] = None,
     ):
-        raise NotImplementedError
+        """Create an index on the table.
+        Currently, the only parameters that matter are
+        the metric and the vector column name.
+
+        Parameters
+        ----------
+        metric : str
+            The metric to use for the index. Default is "L2".
+        num_partitions : int
+            The number of partitions to use for the index. Default is 256.
+        num_sub_vectors : int
+            The number of sub-vectors to use for the index. Default is 96.
+        vector_column_name : str
+            The name of the vector column. Default is "vector".
+        replace : bool
+            Whether to replace the existing index. Default is True.
+
+        Examples
+        --------
+        import lancedb
+        import uuid
+        from lancedb.schema import vector
+        conn = lancedb.connect("db://...", api_key="...", region="...")
+        table_name = uuid.uuid4().hex
+        schema = pa.schema(
+            [
+                    pa.field("id", pa.uint32(), False),
+                    pa.field("vector", vector(128), False),
+                    pa.field("s", pa.string(), False),
+            ]
+        )
+        table = conn.create_table(
+            table_name,
+            schema=schema,
+        )
+        table.create_index()
+        """
+        index_type = "vector"
+
+        data = {
+            "column": vector_column_name,
+            "index_type": index_type,
+            "metric_type": metric,
+        }
+        resp = self._conn._loop.run_until_complete(
+            self._conn._client.post(f"/v1/table/{self._name}/create_index/", data=data)
+        )
+        return resp
 
     def add(
         self,

rust/ffi/node/Cargo.toml

@@ -20,6 +20,7 @@ futures = "0.3"
 half = { workspace = true }
 lance = { workspace = true }
 lance-linalg = { workspace = true }
+lance-index = { workspace = true }
 vectordb = { path = "../../vectordb" }
 tokio = { version = "1.23", features = ["rt-multi-thread"] }
 neon = {version = "0.10.1", default-features = false, features = ["channel-api", "napi-6", "promise-api", "task-api"] }

rust/ffi/node/src/index/vector.rs

@@ -12,7 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use lance::index::vector::{ivf::IvfBuildParams, pq::PQBuildParams};
+use lance::index::vector::pq::PQBuildParams;
+use lance_index::vector::ivf::IvfBuildParams;
 use lance_linalg::distance::MetricType;
 use neon::context::FunctionContext;
 use neon::prelude::*;

rust/ffi/node/src/lib.rs

@@ -180,12 +180,13 @@ fn database_open_table(mut cx: FunctionContext) -> JsResult<JsPromise> {
 
     let aws_creds = get_aws_creds(&mut cx, 1)?;
 
-    let aws_region = get_aws_region(&mut cx, 4)?;
+    let _aws_region = get_aws_region(&mut cx, 4)?;
 
     let params = ReadParams {
-        store_options: Some(ObjectStoreParams::with_aws_credentials(
+        store_options: Some(ObjectStoreParams{
-            aws_creds, aws_region,
+            aws_credentials: aws_creds,
-        )),
+            ..ObjectStoreParams::default()
+        }),
         ..ReadParams::default()
     };
 

rust/ffi/node/src/table.rs

@@ -64,12 +64,13 @@ impl JsTable {
         let database = db.database.clone();
 
         let aws_creds = get_aws_creds(&mut cx, 3)?;
-        let aws_region = get_aws_region(&mut cx, 6)?;
+        let _aws_region = get_aws_region(&mut cx, 6)?;
 
         let params = WriteParams {
-            store_params: Some(ObjectStoreParams::with_aws_credentials(
+            store_params: Some(ObjectStoreParams{
-                aws_creds, aws_region,
+                aws_credentials: aws_creds,
-            )),
+                ..ObjectStoreParams::default()
+            }),
             mode,
             ..WriteParams::default()
         };
@@ -106,12 +107,13 @@ impl JsTable {
             s => return cx.throw_error(format!("invalid write mode {}", s)),
         };
         let aws_creds = get_aws_creds(&mut cx, 2)?;
-        let aws_region = get_aws_region(&mut cx, 5)?;
+        let _aws_region = get_aws_region(&mut cx, 5)?;
 
         let params = WriteParams {
-            store_params: Some(ObjectStoreParams::with_aws_credentials(
+            store_params: Some(ObjectStoreParams{
-                aws_creds, aws_region,
+                aws_credentials: aws_creds,
-            )),
+                ..ObjectStoreParams::default()
+            }),
             mode: write_mode,
             ..WriteParams::default()
         };

rust/vectordb/Cargo.toml

@@ -22,6 +22,7 @@ snafu = { workspace = true }
 half = { workspace = true }
 lance = { workspace = true }
 lance-linalg = { workspace = true }
+lance-index = { workspace = true }
 lance-testing = { workspace = true }
 tokio = { version = "1.23", features = ["rt-multi-thread"] }
 log = { workspace = true }

rust/vectordb/src/index/vector.rs

@@ -13,9 +13,9 @@
 // limitations under the License.
 
 use lance::format::{Index, Manifest};
-use lance::index::vector::ivf::IvfBuildParams;
 use lance::index::vector::pq::PQBuildParams;
 use lance::index::vector::VectorIndexParams;
+use lance_index::vector::ivf::IvfBuildParams;
 use lance_linalg::distance::MetricType;
 
 pub trait VectorIndexBuilder {
@@ -136,7 +136,6 @@ impl VectorIndex {
 mod tests {
     use super::*;
 
-    use lance::index::vector::ivf::IvfBuildParams;
     use lance::index::vector::pq::PQBuildParams;
     use lance::index::vector::StageParams;
 

rust/vectordb/src/table.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use chrono::Duration;
+use lance_index::IndexType;
 use std::sync::Arc;
 
 use arrow_array::{Float32Array, RecordBatchReader};
@@ -22,7 +23,7 @@ use lance::dataset::optimize::{
     compact_files, CompactionMetrics, CompactionOptions, IndexRemapperOptions,
 };
 use lance::dataset::{Dataset, WriteParams};
-use lance::index::{DatasetIndexExt, IndexType};
+use lance::index::DatasetIndexExt;
 use lance::io::object_store::WrappingObjectStore;
 use std::path::Path;