chore: add global cargo config to enable minimal cpu target (#925)

* Closes #895 
* Fix cargo clippy

.cargo/config.toml

@@ -0,0 +1,34 @@
+[profile.release]
+lto = "fat"
+codegen-units = 1
+
+[profile.release-with-debug]
+inherits = "release"
+debug = true
+# Prioritize compile time over runtime performance
+codegen-units = 16
+lto = "thin"
+
+[target.'cfg(all())']
+rustflags = [
+    "-Wclippy::all",
+    "-Wclippy::style",
+    "-Wclippy::fallible_impl_from",
+    "-Wclippy::manual_let_else",
+    "-Wclippy::redundant_pub_crate",
+    "-Wclippy::string_add_assign",
+    "-Wclippy::string_add",
+    "-Wclippy::string_lit_as_bytes",
+    "-Wclippy::string_to_string",
+    "-Wclippy::use_self",
+    "-Dclippy::cargo",
+    "-Dclippy::dbg_macro",
+    # not too much we can do to avoid multiple crate versions
+    "-Aclippy::multiple-crate-versions",
+]
+
+[target.x86_64-unknown-linux-gnu]
+rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"]
+
+[target.aarch64-apple-darwin]
+rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm,+dotprod"]

Cargo.toml

@@ -6,9 +6,12 @@ resolver = "2"
 
 [workspace.package]
 edition = "2021"
-authors = ["Lance Devs <dev@lancedb.com>"]
+authors = ["LanceDB Devs <dev@lancedb.com>"]
 license = "Apache-2.0"
 repository = "https://github.com/lancedb/lancedb"
+description = "Serverless, low-latency vector database for AI applications"
+keywords = ["lancedb", "lance", "database", "vector", "search"]
+categories = ["database-implementations"]
 
 [workspace.dependencies]
 lance = { "version" = "=0.9.12", "features" = ["dynamodb"] }

nodejs/Cargo.toml

@@ -1,9 +1,12 @@
 [package]
 name = "vectordb-nodejs"
-edition = "2021"
+edition.workspace = true
 version = "0.0.0"
 license.workspace = true
+description.workspace = true
 repository.workspace = true
+keywords.workspace = true
+categories.workspace = true
 
 [lib]
 crate-type = ["cdylib"]
@@ -14,15 +17,11 @@ futures.workspace = true
 lance-linalg.workspace = true
 lance.workspace = true
 vectordb = { path = "../rust/vectordb" }
-napi = { version = "2.14", default-features = false, features = [
+napi = { version = "2.15", default-features = false, features = [
     "napi7",
     "async"
 ] }
-napi-derive = "2.14"
+napi-derive = "2"
 
 [build-dependencies]
 napi-build = "2.1"
-
-[profile.release]
-lto = true
-strip = "symbols"

nodejs/jest.config.js

@@ -2,4 +2,6 @@
 module.exports = {
   preset: 'ts-jest',
   testEnvironment: 'node',
-};
+  moduleDirectories: ["node_modules", "./dist"],
+  moduleFileExtensions: ["js", "ts"],
+};

rust/ffi/node/Cargo.toml

@@ -2,8 +2,11 @@
 name = "vectordb-node"
 version = "0.4.8"
 description = "Serverless, low-latency vector database for AI applications"
-license = "Apache-2.0"
-edition = "2018"
+license.workspace = true
+edition.workspace = true
+repository.workspace = true
+keywords.workspace = true
+categories.workspace = true
 exclude = ["index.node"]
 
 [lib]

rust/ffi/node/src/arrow.rs

@@ -22,7 +22,7 @@ use arrow_schema::SchemaRef;
 
 use crate::error::Result;
 
-pub(crate) fn arrow_buffer_to_record_batch(slice: &[u8]) -> Result<(Vec<RecordBatch>, SchemaRef)> {
+pub fn arrow_buffer_to_record_batch(slice: &[u8]) -> Result<(Vec<RecordBatch>, SchemaRef)> {
     let mut batches: Vec<RecordBatch> = Vec::new();
     let file_reader = FileReader::try_new(Cursor::new(slice), None)?;
     let schema = file_reader.schema();
@@ -33,7 +33,7 @@ pub(crate) fn arrow_buffer_to_record_batch(slice: &[u8]) -> Result<(Vec<RecordBa
     Ok((batches, schema))
 }
 
-pub(crate) fn record_batch_to_buffer(batches: Vec<RecordBatch>) -> Result<Vec<u8>> {
+pub fn record_batch_to_buffer(batches: Vec<RecordBatch>) -> Result<Vec<u8>> {
     if batches.is_empty() {
         return Ok(Vec::new());
     }

rust/ffi/node/src/convert.rs

@@ -17,7 +17,7 @@ use neon::types::buffer::TypedArray;
 
 use crate::error::ResultExt;
 
-pub(crate) fn vec_str_to_array<'a, C: Context<'a>>(
+pub fn vec_str_to_array<'a, C: Context<'a>>(
     vec: &Vec<String>,
     cx: &mut C,
 ) -> JsResult<'a, JsArray> {
@@ -29,7 +29,7 @@ pub(crate) fn vec_str_to_array<'a, C: Context<'a>>(
     Ok(a)
 }
 
-pub(crate) fn js_array_to_vec(array: &JsArray, cx: &mut FunctionContext) -> Vec<f32> {
+pub fn js_array_to_vec(array: &JsArray, cx: &mut FunctionContext) -> Vec<f32> {
     let mut query_vec: Vec<f32> = Vec::new();
     for i in 0..array.len(cx) {
         let entry: Handle<JsNumber> = array.get(cx, i).unwrap();
@@ -39,7 +39,7 @@ pub(crate) fn js_array_to_vec(array: &JsArray, cx: &mut FunctionContext) -> Vec<
 }
 
 // Creates a new JsBuffer from a rust buffer with a special logic for electron
-pub(crate) fn new_js_buffer<'a>(
+pub fn new_js_buffer<'a>(
     buffer: Vec<u8>,
     cx: &mut TaskContext<'a>,
     is_electron: bool,

rust/ffi/node/src/error.rs

@@ -18,7 +18,6 @@ use neon::prelude::NeonResult;
 use snafu::Snafu;
 
 #[derive(Debug, Snafu)]
-#[snafu(visibility(pub(crate)))]
 pub enum Error {
     #[snafu(display("column '{name}' is missing"))]
     MissingColumn { name: String },

rust/ffi/node/src/index/scalar.rs

@@ -21,7 +21,7 @@ use neon::{
 use crate::{error::ResultExt, runtime, table::JsTable};
 use vectordb::Table;
 
-pub(crate) fn table_create_scalar_index(mut cx: FunctionContext) -> JsResult<JsPromise> {
+pub fn table_create_scalar_index(mut cx: FunctionContext) -> JsResult<JsPromise> {
     let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
     let column = cx.argument::<JsString>(0)?.value(&mut cx);
     let replace = cx.argument::<JsBoolean>(1)?.value(&mut cx);

rust/ffi/node/src/index/vector.rs

@@ -24,7 +24,7 @@ use crate::neon_ext::js_object_ext::JsObjectExt;
 use crate::runtime;
 use crate::table::JsTable;
 
-pub(crate) fn table_create_vector_index(mut cx: FunctionContext) -> JsResult<JsPromise> {
+pub fn table_create_vector_index(mut cx: FunctionContext) -> JsResult<JsPromise> {
     let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
     let index_params = cx.argument::<JsObject>(0)?;
 

rust/ffi/node/src/query.rs

@@ -13,7 +13,7 @@ use crate::neon_ext::js_object_ext::JsObjectExt;
 use crate::table::JsTable;
 use crate::{convert, runtime};
 
-pub(crate) struct JsQuery {}
+pub struct JsQuery {}
 
 impl JsQuery {
     pub(crate) fn js_search(mut cx: FunctionContext) -> JsResult<JsPromise> {

rust/ffi/node/src/table.rs

@@ -28,7 +28,7 @@ use vectordb::TableRef;
 use crate::error::ResultExt;
 use crate::{convert, get_aws_credential_provider, get_aws_region, runtime, JsDatabase};
 
-pub(crate) struct JsTable {
+pub struct JsTable {
     pub table: TableRef,
 }
 
@@ -36,7 +36,7 @@ impl Finalize for JsTable {}
 
 impl From<TableRef> for JsTable {
     fn from(table: TableRef) -> Self {
-        JsTable { table }
+        Self { table }
     }
 }
 
@@ -85,14 +85,14 @@ impl JsTable {
 
             deferred.settle_with(&channel, move |mut cx| {
                 let table = table_rst.or_throw(&mut cx)?;
-                Ok(cx.boxed(JsTable::from(table)))
+                Ok(cx.boxed(Self::from(table)))
             });
         });
         Ok(promise)
     }
 
     pub(crate) fn js_add(mut cx: FunctionContext) -> JsResult<JsPromise> {
-        let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
+        let js_table = cx.this().downcast_or_throw::<JsBox<Self>, _>(&mut cx)?;
         let buffer = cx.argument::<JsBuffer>(0)?;
         let write_mode = cx.argument::<JsString>(1)?.value(&mut cx);
         let (batches, schema) =
@@ -125,14 +125,14 @@ impl JsTable {
 
             deferred.settle_with(&channel, move |mut cx| {
                 add_result.or_throw(&mut cx)?;
-                Ok(cx.boxed(JsTable::from(table)))
+                Ok(cx.boxed(Self::from(table)))
             });
         });
         Ok(promise)
     }
 
     pub(crate) fn js_count_rows(mut cx: FunctionContext) -> JsResult<JsPromise> {
-        let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
+        let js_table = cx.this().downcast_or_throw::<JsBox<Self>, _>(&mut cx)?;
         let rt = runtime(&mut cx)?;
         let (deferred, promise) = cx.promise();
         let channel = cx.channel();
@@ -150,7 +150,7 @@ impl JsTable {
     }
 
     pub(crate) fn js_delete(mut cx: FunctionContext) -> JsResult<JsPromise> {
-        let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
+        let js_table = cx.this().downcast_or_throw::<JsBox<Self>, _>(&mut cx)?;
         let rt = runtime(&mut cx)?;
         let (deferred, promise) = cx.promise();
         let predicate = cx.argument::<JsString>(0)?.value(&mut cx);
@@ -162,14 +162,14 @@ impl JsTable {
 
             deferred.settle_with(&channel, move |mut cx| {
                 delete_result.or_throw(&mut cx)?;
-                Ok(cx.boxed(JsTable::from(table)))
+                Ok(cx.boxed(Self::from(table)))
             })
         });
         Ok(promise)
     }
 
     pub(crate) fn js_merge_insert(mut cx: FunctionContext) -> JsResult<JsPromise> {
-        let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
+        let js_table = cx.this().downcast_or_throw::<JsBox<Self>, _>(&mut cx)?;
         let rt = runtime(&mut cx)?;
         let (deferred, promise) = cx.promise();
         let channel = cx.channel();
@@ -209,14 +209,14 @@ impl JsTable {
 
             deferred.settle_with(&channel, move |mut cx| {
                 merge_insert_result.or_throw(&mut cx)?;
-                Ok(cx.boxed(JsTable::from(table)))
+                Ok(cx.boxed(Self::from(table)))
             })
         });
         Ok(promise)
     }
 
     pub(crate) fn js_update(mut cx: FunctionContext) -> JsResult<JsPromise> {
-        let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
+        let js_table = cx.this().downcast_or_throw::<JsBox<Self>, _>(&mut cx)?;
         let table = js_table.table.clone();
 
         let rt = runtime(&mut cx)?;
@@ -275,7 +275,7 @@ impl JsTable {
                 .await;
             deferred.settle_with(&channel, move |mut cx| {
                 update_result.or_throw(&mut cx)?;
-                Ok(cx.boxed(JsTable::from(table)))
+                Ok(cx.boxed(Self::from(table)))
             })
         });
 
@@ -283,7 +283,7 @@ impl JsTable {
     }
 
     pub(crate) fn js_cleanup(mut cx: FunctionContext) -> JsResult<JsPromise> {
-        let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
+        let js_table = cx.this().downcast_or_throw::<JsBox<Self>, _>(&mut cx)?;
         let rt = runtime(&mut cx)?;
         let (deferred, promise) = cx.promise();
         let table = js_table.table.clone();
@@ -321,7 +321,7 @@ impl JsTable {
                 let old_versions = cx.number(prune_stats.old_versions as f64);
                 output_metrics.set(&mut cx, "oldVersions", old_versions)?;
 
-                let output_table = cx.boxed(JsTable::from(table));
+                let output_table = cx.boxed(Self::from(table));
 
                 let output = JsObject::new(&mut cx);
                 output.set(&mut cx, "metrics", output_metrics)?;
@@ -334,7 +334,7 @@ impl JsTable {
     }
 
     pub(crate) fn js_compact(mut cx: FunctionContext) -> JsResult<JsPromise> {
-        let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
+        let js_table = cx.this().downcast_or_throw::<JsBox<Self>, _>(&mut cx)?;
         let rt = runtime(&mut cx)?;
         let (deferred, promise) = cx.promise();
         let table = js_table.table.clone();
@@ -393,7 +393,7 @@ impl JsTable {
                 let files_added = cx.number(stats.files_added as f64);
                 output_metrics.set(&mut cx, "filesAdded", files_added)?;
 
-                let output_table = cx.boxed(JsTable::from(table));
+                let output_table = cx.boxed(Self::from(table));
 
                 let output = JsObject::new(&mut cx);
                 output.set(&mut cx, "metrics", output_metrics)?;
@@ -406,7 +406,7 @@ impl JsTable {
     }
 
     pub(crate) fn js_list_indices(mut cx: FunctionContext) -> JsResult<JsPromise> {
-        let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
+        let js_table = cx.this().downcast_or_throw::<JsBox<Self>, _>(&mut cx)?;
         let rt = runtime(&mut cx)?;
         let (deferred, promise) = cx.promise();
         // let predicate = cx.argument::<JsString>(0)?.value(&mut cx);
@@ -445,7 +445,7 @@ impl JsTable {
     }
 
     pub(crate) fn js_index_stats(mut cx: FunctionContext) -> JsResult<JsPromise> {
-        let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
+        let js_table = cx.this().downcast_or_throw::<JsBox<Self>, _>(&mut cx)?;
         let rt = runtime(&mut cx)?;
         let (deferred, promise) = cx.promise();
         let index_uuid = cx.argument::<JsString>(0)?.value(&mut cx);
@@ -493,7 +493,7 @@ impl JsTable {
     }
 
     pub(crate) fn js_schema(mut cx: FunctionContext) -> JsResult<JsPromise> {
-        let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
+        let js_table = cx.this().downcast_or_throw::<JsBox<Self>, _>(&mut cx)?;
         let rt = runtime(&mut cx)?;
         let (deferred, promise) = cx.promise();
         let channel = cx.channel();

rust/vectordb/Cargo.toml

@@ -1,12 +1,12 @@
 [package]
 name = "vectordb"
 version = "0.4.8"
-edition = "2021"
+edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
-license = "Apache-2.0"
-repository = "https://github.com/lancedb/lancedb"
-keywords = ["lancedb", "lance", "database", "search"]
-categories = ["database-implementations"]
+license.workspace = true
+repository.workspace = true
+keywords.workspace = true
+categories.workspace = true
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [dependencies]

rust/vectordb/src/connection.rs

@@ -188,12 +188,12 @@ impl Database {
     /// # Returns
     ///
     /// * A [Database] object.
-    pub async fn connect(uri: &str) -> Result<Database> {
+    pub async fn connect(uri: &str) -> Result<Self> {
         let options = ConnectOptions::new(uri);
         Self::connect_with_options(&options).await
     }
 
-    pub async fn connect_with_options(options: &ConnectOptions) -> Result<Database> {
+    pub async fn connect_with_options(options: &ConnectOptions) -> Result<Self> {
         let uri = &options.uri;
         let parse_res = url::Url::parse(uri);
 
@@ -276,7 +276,7 @@ impl Database {
                     None => None,
                 };
 
-                Ok(Database {
+                Ok(Self {
                     uri: table_base_uri,
                     query_string,
                     base_path,
@@ -288,7 +288,7 @@ impl Database {
         }
     }
 
-    async fn open_path(path: &str) -> Result<Database> {
+    async fn open_path(path: &str) -> Result<Self> {
         let (object_store, base_path) = ObjectStore::from_uri(path).await?;
         if object_store.is_local() {
             Self::try_create_dir(path).context(CreateDirSnafu { path })?;

rust/vectordb/src/index.rs

@@ -69,7 +69,7 @@ pub struct IndexBuilder {
 
 impl IndexBuilder {
     pub(crate) fn new(table: Arc<dyn Table>, columns: &[&str]) -> Self {
-        IndexBuilder {
+        Self {
             table,
             columns: columns.iter().map(|c| c.to_string()).collect(),
             name: None,

rust/vectordb/src/index/vector.rs

@@ -23,13 +23,13 @@ pub struct VectorIndex {
 }
 
 impl VectorIndex {
-    pub fn new_from_format(manifest: &Manifest, index: &Index) -> VectorIndex {
+    pub fn new_from_format(manifest: &Manifest, index: &Index) -> Self {
         let fields = index
             .fields
             .iter()
             .map(|i| manifest.schema.fields[*i as usize].name.clone())
             .collect();
-        VectorIndex {
+        Self {
             columns: fields,
             index_name: index.name.clone(),
             index_uuid: index.uuid.to_string(),

rust/vectordb/src/query.rs

@@ -62,7 +62,7 @@ impl Query {
     /// * `dataset` - Lance dataset.
     ///
     pub(crate) fn new(dataset: Arc<Dataset>) -> Self {
-        Query {
+        Self {
             dataset,
             query_vector: None,
             column: None,

rust/vectordb/src/table.rs

@@ -385,7 +385,7 @@ impl NativeTable {
                     message: e.to_string(),
                 },
             })?;
-        Ok(NativeTable {
+        Ok(Self {
             name: name.to_string(),
             uri: uri.to_string(),
             dataset: Arc::new(Mutex::new(dataset)),
@@ -427,7 +427,7 @@ impl NativeTable {
                     message: e.to_string(),
                 },
             })?;
-        Ok(NativeTable {
+        Ok(Self {
             name: name.to_string(),
             uri: uri.to_string(),
             dataset: Arc::new(Mutex::new(dataset)),
@@ -501,7 +501,7 @@ impl NativeTable {
                     message: e.to_string(),
                 },
             })?;
-        Ok(NativeTable {
+        Ok(Self {
             name: name.to_string(),
             uri: uri.to_string(),
             dataset: Arc::new(Mutex::new(dataset)),