From 0b0f42537eefaced53f44f49cf6358046fd5cf39 Mon Sep 17 00:00:00 2001 From: Lei Xu Date: Sun, 4 Feb 2024 14:21:27 -0800 Subject: [PATCH] chore: add global cargo config to enable minimal cpu target (#925) * Closes #895 * Fix cargo clippy --- .cargo/config.toml | 34 +++++++++++++++++++++++++++ Cargo.toml | 5 +++- nodejs/Cargo.toml | 13 +++++------ nodejs/jest.config.js | 4 +++- rust/ffi/node/Cargo.toml | 7 ++++-- rust/ffi/node/src/arrow.rs | 4 ++-- rust/ffi/node/src/convert.rs | 6 ++--- rust/ffi/node/src/error.rs | 1 - rust/ffi/node/src/index/scalar.rs | 2 +- rust/ffi/node/src/index/vector.rs | 2 +- rust/ffi/node/src/query.rs | 2 +- rust/ffi/node/src/table.rs | 38 +++++++++++++++---------------- rust/vectordb/Cargo.toml | 10 ++++---- rust/vectordb/src/connection.rs | 8 +++---- rust/vectordb/src/index.rs | 2 +- rust/vectordb/src/index/vector.rs | 4 ++-- rust/vectordb/src/query.rs | 2 +- rust/vectordb/src/table.rs | 6 ++--- 18 files changed, 95 insertions(+), 55 deletions(-) create mode 100644 .cargo/config.toml diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 00000000..8dca69f5 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,34 @@ +[profile.release] +lto = "fat" +codegen-units = 1 + +[profile.release-with-debug] +inherits = "release" +debug = true +# Prioritize compile time over runtime performance +codegen-units = 16 +lto = "thin" + +[target.'cfg(all())'] +rustflags = [ + "-Wclippy::all", + "-Wclippy::style", + "-Wclippy::fallible_impl_from", + "-Wclippy::manual_let_else", + "-Wclippy::redundant_pub_crate", + "-Wclippy::string_add_assign", + "-Wclippy::string_add", + "-Wclippy::string_lit_as_bytes", + "-Wclippy::string_to_string", + "-Wclippy::use_self", + "-Dclippy::cargo", + "-Dclippy::dbg_macro", + # not too much we can do to avoid multiple crate versions + "-Aclippy::multiple-crate-versions", +] + +[target.x86_64-unknown-linux-gnu] +rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"] + +[target.aarch64-apple-darwin] +rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm,+dotprod"] diff --git a/Cargo.toml b/Cargo.toml index 1d505920..972aa328 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,9 +6,12 @@ resolver = "2" [workspace.package] edition = "2021" -authors = ["Lance Devs "] +authors = ["LanceDB Devs "] license = "Apache-2.0" repository = "https://github.com/lancedb/lancedb" +description = "Serverless, low-latency vector database for AI applications" +keywords = ["lancedb", "lance", "database", "vector", "search"] +categories = ["database-implementations"] [workspace.dependencies] lance = { "version" = "=0.9.12", "features" = ["dynamodb"] } diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index fe7b1876..d58fd15d 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,9 +1,12 @@ [package] name = "vectordb-nodejs" -edition = "2021" +edition.workspace = true version = "0.0.0" license.workspace = true +description.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true [lib] crate-type = ["cdylib"] @@ -14,15 +17,11 @@ futures.workspace = true lance-linalg.workspace = true lance.workspace = true vectordb = { path = "../rust/vectordb" } -napi = { version = "2.14", default-features = false, features = [ +napi = { version = "2.15", default-features = false, features = [ "napi7", "async" ] } -napi-derive = "2.14" +napi-derive = "2" [build-dependencies] napi-build = "2.1" - -[profile.release] -lto = true -strip = "symbols" diff --git a/nodejs/jest.config.js b/nodejs/jest.config.js index b413e106..2c7632a3 100644 --- a/nodejs/jest.config.js +++ b/nodejs/jest.config.js @@ -2,4 +2,6 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', -}; \ No newline at end of file + moduleDirectories: ["node_modules", "./dist"], + moduleFileExtensions: ["js", "ts"], +}; diff --git a/rust/ffi/node/Cargo.toml b/rust/ffi/node/Cargo.toml index 3b52f58b..73ecfdab 100644 --- a/rust/ffi/node/Cargo.toml +++ b/rust/ffi/node/Cargo.toml @@ -2,8 +2,11 @@ name = "vectordb-node" version = "0.4.8" description = "Serverless, low-latency vector database for AI applications" -license = "Apache-2.0" -edition = "2018" +license.workspace = true +edition.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true exclude = ["index.node"] [lib] diff --git a/rust/ffi/node/src/arrow.rs b/rust/ffi/node/src/arrow.rs index 2c90abf7..036f7104 100644 --- a/rust/ffi/node/src/arrow.rs +++ b/rust/ffi/node/src/arrow.rs @@ -22,7 +22,7 @@ use arrow_schema::SchemaRef; use crate::error::Result; -pub(crate) fn arrow_buffer_to_record_batch(slice: &[u8]) -> Result<(Vec, SchemaRef)> { +pub fn arrow_buffer_to_record_batch(slice: &[u8]) -> Result<(Vec, SchemaRef)> { let mut batches: Vec = Vec::new(); let file_reader = FileReader::try_new(Cursor::new(slice), None)?; let schema = file_reader.schema(); @@ -33,7 +33,7 @@ pub(crate) fn arrow_buffer_to_record_batch(slice: &[u8]) -> Result<(Vec) -> Result> { +pub fn record_batch_to_buffer(batches: Vec) -> Result> { if batches.is_empty() { return Ok(Vec::new()); } diff --git a/rust/ffi/node/src/convert.rs b/rust/ffi/node/src/convert.rs index 2736d2ca..3c0ea2f0 100644 --- a/rust/ffi/node/src/convert.rs +++ b/rust/ffi/node/src/convert.rs @@ -17,7 +17,7 @@ use neon::types::buffer::TypedArray; use crate::error::ResultExt; -pub(crate) fn vec_str_to_array<'a, C: Context<'a>>( +pub fn vec_str_to_array<'a, C: Context<'a>>( vec: &Vec, cx: &mut C, ) -> JsResult<'a, JsArray> { @@ -29,7 +29,7 @@ pub(crate) fn vec_str_to_array<'a, C: Context<'a>>( Ok(a) } -pub(crate) fn js_array_to_vec(array: &JsArray, cx: &mut FunctionContext) -> Vec { +pub fn js_array_to_vec(array: &JsArray, cx: &mut FunctionContext) -> Vec { let mut query_vec: Vec = Vec::new(); for i in 0..array.len(cx) { let entry: Handle = array.get(cx, i).unwrap(); @@ -39,7 +39,7 @@ pub(crate) fn js_array_to_vec(array: &JsArray, cx: &mut FunctionContext) -> Vec< } // Creates a new JsBuffer from a rust buffer with a special logic for electron -pub(crate) fn new_js_buffer<'a>( +pub fn new_js_buffer<'a>( buffer: Vec, cx: &mut TaskContext<'a>, is_electron: bool, diff --git a/rust/ffi/node/src/error.rs b/rust/ffi/node/src/error.rs index 14c5cc80..ce447f02 100644 --- a/rust/ffi/node/src/error.rs +++ b/rust/ffi/node/src/error.rs @@ -18,7 +18,6 @@ use neon::prelude::NeonResult; use snafu::Snafu; #[derive(Debug, Snafu)] -#[snafu(visibility(pub(crate)))] pub enum Error { #[snafu(display("column '{name}' is missing"))] MissingColumn { name: String }, diff --git a/rust/ffi/node/src/index/scalar.rs b/rust/ffi/node/src/index/scalar.rs index 19477b69..c9c743fe 100644 --- a/rust/ffi/node/src/index/scalar.rs +++ b/rust/ffi/node/src/index/scalar.rs @@ -21,7 +21,7 @@ use neon::{ use crate::{error::ResultExt, runtime, table::JsTable}; use vectordb::Table; -pub(crate) fn table_create_scalar_index(mut cx: FunctionContext) -> JsResult { +pub fn table_create_scalar_index(mut cx: FunctionContext) -> JsResult { let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; let column = cx.argument::(0)?.value(&mut cx); let replace = cx.argument::(1)?.value(&mut cx); diff --git a/rust/ffi/node/src/index/vector.rs b/rust/ffi/node/src/index/vector.rs index 110a34e5..7185c892 100644 --- a/rust/ffi/node/src/index/vector.rs +++ b/rust/ffi/node/src/index/vector.rs @@ -24,7 +24,7 @@ use crate::neon_ext::js_object_ext::JsObjectExt; use crate::runtime; use crate::table::JsTable; -pub(crate) fn table_create_vector_index(mut cx: FunctionContext) -> JsResult { +pub fn table_create_vector_index(mut cx: FunctionContext) -> JsResult { let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; let index_params = cx.argument::(0)?; diff --git a/rust/ffi/node/src/query.rs b/rust/ffi/node/src/query.rs index 06988680..c9ba7b8b 100644 --- a/rust/ffi/node/src/query.rs +++ b/rust/ffi/node/src/query.rs @@ -13,7 +13,7 @@ use crate::neon_ext::js_object_ext::JsObjectExt; use crate::table::JsTable; use crate::{convert, runtime}; -pub(crate) struct JsQuery {} +pub struct JsQuery {} impl JsQuery { pub(crate) fn js_search(mut cx: FunctionContext) -> JsResult { diff --git a/rust/ffi/node/src/table.rs b/rust/ffi/node/src/table.rs index 9f6f9f6f..8139cfda 100644 --- a/rust/ffi/node/src/table.rs +++ b/rust/ffi/node/src/table.rs @@ -28,7 +28,7 @@ use vectordb::TableRef; use crate::error::ResultExt; use crate::{convert, get_aws_credential_provider, get_aws_region, runtime, JsDatabase}; -pub(crate) struct JsTable { +pub struct JsTable { pub table: TableRef, } @@ -36,7 +36,7 @@ impl Finalize for JsTable {} impl From for JsTable { fn from(table: TableRef) -> Self { - JsTable { table } + Self { table } } } @@ -85,14 +85,14 @@ impl JsTable { deferred.settle_with(&channel, move |mut cx| { let table = table_rst.or_throw(&mut cx)?; - Ok(cx.boxed(JsTable::from(table))) + Ok(cx.boxed(Self::from(table))) }); }); Ok(promise) } pub(crate) fn js_add(mut cx: FunctionContext) -> JsResult { - let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; + let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; let buffer = cx.argument::(0)?; let write_mode = cx.argument::(1)?.value(&mut cx); let (batches, schema) = @@ -125,14 +125,14 @@ impl JsTable { deferred.settle_with(&channel, move |mut cx| { add_result.or_throw(&mut cx)?; - Ok(cx.boxed(JsTable::from(table))) + Ok(cx.boxed(Self::from(table))) }); }); Ok(promise) } pub(crate) fn js_count_rows(mut cx: FunctionContext) -> JsResult { - let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; + let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; let rt = runtime(&mut cx)?; let (deferred, promise) = cx.promise(); let channel = cx.channel(); @@ -150,7 +150,7 @@ impl JsTable { } pub(crate) fn js_delete(mut cx: FunctionContext) -> JsResult { - let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; + let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; let rt = runtime(&mut cx)?; let (deferred, promise) = cx.promise(); let predicate = cx.argument::(0)?.value(&mut cx); @@ -162,14 +162,14 @@ impl JsTable { deferred.settle_with(&channel, move |mut cx| { delete_result.or_throw(&mut cx)?; - Ok(cx.boxed(JsTable::from(table))) + Ok(cx.boxed(Self::from(table))) }) }); Ok(promise) } pub(crate) fn js_merge_insert(mut cx: FunctionContext) -> JsResult { - let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; + let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; let rt = runtime(&mut cx)?; let (deferred, promise) = cx.promise(); let channel = cx.channel(); @@ -209,14 +209,14 @@ impl JsTable { deferred.settle_with(&channel, move |mut cx| { merge_insert_result.or_throw(&mut cx)?; - Ok(cx.boxed(JsTable::from(table))) + Ok(cx.boxed(Self::from(table))) }) }); Ok(promise) } pub(crate) fn js_update(mut cx: FunctionContext) -> JsResult { - let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; + let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; let table = js_table.table.clone(); let rt = runtime(&mut cx)?; @@ -275,7 +275,7 @@ impl JsTable { .await; deferred.settle_with(&channel, move |mut cx| { update_result.or_throw(&mut cx)?; - Ok(cx.boxed(JsTable::from(table))) + Ok(cx.boxed(Self::from(table))) }) }); @@ -283,7 +283,7 @@ impl JsTable { } pub(crate) fn js_cleanup(mut cx: FunctionContext) -> JsResult { - let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; + let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; let rt = runtime(&mut cx)?; let (deferred, promise) = cx.promise(); let table = js_table.table.clone(); @@ -321,7 +321,7 @@ impl JsTable { let old_versions = cx.number(prune_stats.old_versions as f64); output_metrics.set(&mut cx, "oldVersions", old_versions)?; - let output_table = cx.boxed(JsTable::from(table)); + let output_table = cx.boxed(Self::from(table)); let output = JsObject::new(&mut cx); output.set(&mut cx, "metrics", output_metrics)?; @@ -334,7 +334,7 @@ impl JsTable { } pub(crate) fn js_compact(mut cx: FunctionContext) -> JsResult { - let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; + let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; let rt = runtime(&mut cx)?; let (deferred, promise) = cx.promise(); let table = js_table.table.clone(); @@ -393,7 +393,7 @@ impl JsTable { let files_added = cx.number(stats.files_added as f64); output_metrics.set(&mut cx, "filesAdded", files_added)?; - let output_table = cx.boxed(JsTable::from(table)); + let output_table = cx.boxed(Self::from(table)); let output = JsObject::new(&mut cx); output.set(&mut cx, "metrics", output_metrics)?; @@ -406,7 +406,7 @@ impl JsTable { } pub(crate) fn js_list_indices(mut cx: FunctionContext) -> JsResult { - let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; + let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; let rt = runtime(&mut cx)?; let (deferred, promise) = cx.promise(); // let predicate = cx.argument::(0)?.value(&mut cx); @@ -445,7 +445,7 @@ impl JsTable { } pub(crate) fn js_index_stats(mut cx: FunctionContext) -> JsResult { - let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; + let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; let rt = runtime(&mut cx)?; let (deferred, promise) = cx.promise(); let index_uuid = cx.argument::(0)?.value(&mut cx); @@ -493,7 +493,7 @@ impl JsTable { } pub(crate) fn js_schema(mut cx: FunctionContext) -> JsResult { - let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; + let js_table = cx.this().downcast_or_throw::, _>(&mut cx)?; let rt = runtime(&mut cx)?; let (deferred, promise) = cx.promise(); let channel = cx.channel(); diff --git a/rust/vectordb/Cargo.toml b/rust/vectordb/Cargo.toml index 88cde224..c0a7b13b 100644 --- a/rust/vectordb/Cargo.toml +++ b/rust/vectordb/Cargo.toml @@ -1,12 +1,12 @@ [package] name = "vectordb" version = "0.4.8" -edition = "2021" +edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" -license = "Apache-2.0" -repository = "https://github.com/lancedb/lancedb" -keywords = ["lancedb", "lance", "database", "search"] -categories = ["database-implementations"] +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] diff --git a/rust/vectordb/src/connection.rs b/rust/vectordb/src/connection.rs index 670b059e..0e29415d 100644 --- a/rust/vectordb/src/connection.rs +++ b/rust/vectordb/src/connection.rs @@ -188,12 +188,12 @@ impl Database { /// # Returns /// /// * A [Database] object. - pub async fn connect(uri: &str) -> Result { + pub async fn connect(uri: &str) -> Result { let options = ConnectOptions::new(uri); Self::connect_with_options(&options).await } - pub async fn connect_with_options(options: &ConnectOptions) -> Result { + pub async fn connect_with_options(options: &ConnectOptions) -> Result { let uri = &options.uri; let parse_res = url::Url::parse(uri); @@ -276,7 +276,7 @@ impl Database { None => None, }; - Ok(Database { + Ok(Self { uri: table_base_uri, query_string, base_path, @@ -288,7 +288,7 @@ impl Database { } } - async fn open_path(path: &str) -> Result { + async fn open_path(path: &str) -> Result { let (object_store, base_path) = ObjectStore::from_uri(path).await?; if object_store.is_local() { Self::try_create_dir(path).context(CreateDirSnafu { path })?; diff --git a/rust/vectordb/src/index.rs b/rust/vectordb/src/index.rs index 78225fb9..e953f6e6 100644 --- a/rust/vectordb/src/index.rs +++ b/rust/vectordb/src/index.rs @@ -69,7 +69,7 @@ pub struct IndexBuilder { impl IndexBuilder { pub(crate) fn new(table: Arc, columns: &[&str]) -> Self { - IndexBuilder { + Self { table, columns: columns.iter().map(|c| c.to_string()).collect(), name: None, diff --git a/rust/vectordb/src/index/vector.rs b/rust/vectordb/src/index/vector.rs index 9afcc467..cf8771cb 100644 --- a/rust/vectordb/src/index/vector.rs +++ b/rust/vectordb/src/index/vector.rs @@ -23,13 +23,13 @@ pub struct VectorIndex { } impl VectorIndex { - pub fn new_from_format(manifest: &Manifest, index: &Index) -> VectorIndex { + pub fn new_from_format(manifest: &Manifest, index: &Index) -> Self { let fields = index .fields .iter() .map(|i| manifest.schema.fields[*i as usize].name.clone()) .collect(); - VectorIndex { + Self { columns: fields, index_name: index.name.clone(), index_uuid: index.uuid.to_string(), diff --git a/rust/vectordb/src/query.rs b/rust/vectordb/src/query.rs index 7ce900b7..85061167 100644 --- a/rust/vectordb/src/query.rs +++ b/rust/vectordb/src/query.rs @@ -62,7 +62,7 @@ impl Query { /// * `dataset` - Lance dataset. /// pub(crate) fn new(dataset: Arc) -> Self { - Query { + Self { dataset, query_vector: None, column: None, diff --git a/rust/vectordb/src/table.rs b/rust/vectordb/src/table.rs index d4a3ae28..97eaecd0 100644 --- a/rust/vectordb/src/table.rs +++ b/rust/vectordb/src/table.rs @@ -385,7 +385,7 @@ impl NativeTable { message: e.to_string(), }, })?; - Ok(NativeTable { + Ok(Self { name: name.to_string(), uri: uri.to_string(), dataset: Arc::new(Mutex::new(dataset)), @@ -427,7 +427,7 @@ impl NativeTable { message: e.to_string(), }, })?; - Ok(NativeTable { + Ok(Self { name: name.to_string(), uri: uri.to_string(), dataset: Arc::new(Mutex::new(dataset)), @@ -501,7 +501,7 @@ impl NativeTable { message: e.to_string(), }, })?; - Ok(NativeTable { + Ok(Self { name: name.to_string(), uri: uri.to_string(), dataset: Arc::new(Mutex::new(dataset)),