expose scalar index

Cargo.toml

@@ -5,9 +5,10 @@ exclude = ["python"]
 resolver = "2"
 
 [workspace.dependencies]
-lance = { "version" = "=0.8.14", "features" = ["dynamodb"] }
+lance = { "version" = "=0.8.16", "features" = ["dynamodb"] }
-lance-linalg = { "version" = "=0.8.14" }
+lance-linalg = { "version" = "=0.8.16" }
-lance-testing = { "version" = "=0.8.14" }
+lance-index = { "version" = "=0.8.16" }
+lance-testing = { "version" = "=0.8.16" }
 # Note that this one does not include pyarrow
 arrow = { version = "47.0.0", optional = false }
 arrow-array = "47.0"

rust/ffi/node/Cargo.toml

@@ -20,6 +20,7 @@ futures = "0.3"
 half = { workspace = true }
 lance = { workspace = true }
 lance-linalg = { workspace = true }
+lance-index = { workspace = true }
 vectordb = { path = "../../vectordb" }
 tokio = { version = "1.23", features = ["rt-multi-thread"] }
 neon = {version = "0.10.1", default-features = false, features = ["channel-api", "napi-6", "promise-api", "task-api"] }

rust/ffi/node/src/index/vector.rs

@@ -12,7 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use lance::index::vector::{ivf::IvfBuildParams, pq::PQBuildParams};
+use lance::index::vector::pq::PQBuildParams;
+use lance_index::vector::ivf::IvfBuildParams;
 use lance_linalg::distance::MetricType;
 use neon::context::FunctionContext;
 use neon::prelude::*;

rust/ffi/node/src/lib.rs

@@ -180,12 +180,13 @@ fn database_open_table(mut cx: FunctionContext) -> JsResult<JsPromise> {
 
     let aws_creds = get_aws_creds(&mut cx, 1)?;
 
-    let aws_region = get_aws_region(&mut cx, 4)?;
+    let _aws_region = get_aws_region(&mut cx, 4)?;
 
     let params = ReadParams {
-        store_options: Some(ObjectStoreParams::with_aws_credentials(
+        store_options: Some(ObjectStoreParams{
-            aws_creds, aws_region,
+            aws_credentials: aws_creds,
-        )),
+            ..ObjectStoreParams::default()
+        }),
         ..ReadParams::default()
     };
 

rust/ffi/node/src/table.rs

@@ -64,12 +64,13 @@ impl JsTable {
         let database = db.database.clone();
 
         let aws_creds = get_aws_creds(&mut cx, 3)?;
-        let aws_region = get_aws_region(&mut cx, 6)?;
+        let _aws_region = get_aws_region(&mut cx, 6)?;
 
         let params = WriteParams {
-            store_params: Some(ObjectStoreParams::with_aws_credentials(
+            store_params: Some(ObjectStoreParams{
-                aws_creds, aws_region,
+                aws_credentials: aws_creds,
-            )),
+                ..ObjectStoreParams::default()
+            }),
             mode,
             ..WriteParams::default()
         };
@@ -106,12 +107,13 @@ impl JsTable {
             s => return cx.throw_error(format!("invalid write mode {}", s)),
         };
         let aws_creds = get_aws_creds(&mut cx, 2)?;
-        let aws_region = get_aws_region(&mut cx, 5)?;
+        let _aws_region = get_aws_region(&mut cx, 5)?;
 
         let params = WriteParams {
-            store_params: Some(ObjectStoreParams::with_aws_credentials(
+            store_params: Some(ObjectStoreParams{
-                aws_creds, aws_region,
+                aws_credentials: aws_creds,
-            )),
+                ..ObjectStoreParams::default()
+            }),
             mode: write_mode,
             ..WriteParams::default()
         };

rust/vectordb/Cargo.toml

@@ -22,6 +22,7 @@ snafu = { workspace = true }
 half = { workspace = true }
 lance = { workspace = true }
 lance-linalg = { workspace = true }
+lance-index = { workspace = true }
 lance-testing = { workspace = true }
 tokio = { version = "1.23", features = ["rt-multi-thread"] }
 log = { workspace = true }

rust/vectordb/src/index/vector.rs

@@ -13,9 +13,9 @@
 // limitations under the License.
 
 use lance::format::{Index, Manifest};
-use lance::index::vector::ivf::IvfBuildParams;
 use lance::index::vector::pq::PQBuildParams;
 use lance::index::vector::VectorIndexParams;
+use lance_index::vector::ivf::IvfBuildParams;
 use lance_linalg::distance::MetricType;
 
 pub trait VectorIndexBuilder {
@@ -136,7 +136,6 @@ impl VectorIndex {
 mod tests {
     use super::*;
 
-    use lance::index::vector::ivf::IvfBuildParams;
     use lance::index::vector::pq::PQBuildParams;
     use lance::index::vector::StageParams;
 

rust/vectordb/src/table.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use chrono::Duration;
+use lance_index::IndexType;
 use std::sync::Arc;
 
 use arrow_array::{Float32Array, RecordBatchReader};
@@ -22,7 +23,7 @@ use lance::dataset::optimize::{
     compact_files, CompactionMetrics, CompactionOptions, IndexRemapperOptions,
 };
 use lance::dataset::{Dataset, WriteParams};
-use lance::index::{DatasetIndexExt, IndexType};
+use lance::index::DatasetIndexExt;
 use lance::io::object_store::WrappingObjectStore;
 use std::path::Path;