add index_stats to python api

Added a small bit of documentation for the `dim` feature which is
provided by the new `text-embedding-3` model series that allows users to
shorten an embedding.

Happy to discuss a bit on the phrasing but I struggled quite a bit with
getting it to work so wanted to help others who might want to use the
newer model too

.github/workflows/remote-integration.yml

@@ -1,37 +0,0 @@
-name: LanceDb Cloud Integration Test
-
-on:
-  workflow_run:
-    workflows: [Rust]
-    types:
-      - completed
-
-env:
-  LANCEDB_PROJECT: ${{ secrets.LANCEDB_PROJECT }}
-  LANCEDB_API_KEY: ${{ secrets.LANCEDB_API_KEY }}
-  LANCEDB_REGION: ${{ secrets.LANCEDB_REGION }}
-
-jobs:
-  test:
-    timeout-minutes: 30
-    runs-on: ubuntu-22.04
-    defaults:
-      run:
-        shell: bash
-        working-directory: rust
-    steps:
-    - uses: actions/checkout@v4
-      with:
-          fetch-depth: 0
-          lfs: true
-    - uses: Swatinem/rust-cache@v2
-      with:
-          workspaces: rust
-    - name: Install dependencies
-      run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-    - name: Build
-      run: cargo build --all-features
-    - name: Run Integration test
-      run: cargo test --tests -- --ignored

docs/src/embeddings/default_embedding_functions.md

@@ -47,6 +47,7 @@ LanceDB registers the OpenAI embeddings function in the registry by default, as
 | Parameter | Type | Default Value | Description |
 |---|---|---|---|
 | `name` | `str` | `"text-embedding-ada-002"` | The name of the model. |
+| `dim` | `int` |  Model default   | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it |
 
 
 ```python

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.4.11",
+  "version": "0.4.12",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.4.11",
+      "version": "0.4.12",
       "cpu": [
         "x64",
         "arm64"
@@ -52,11 +52,11 @@
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.4.11",
+        "@lancedb/vectordb-darwin-arm64": "0.4.12",
-        "@lancedb/vectordb-darwin-x64": "0.4.11",
+        "@lancedb/vectordb-darwin-x64": "0.4.12",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.4.11",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.4.12",
-        "@lancedb/vectordb-linux-x64-gnu": "0.4.11",
+        "@lancedb/vectordb-linux-x64-gnu": "0.4.12",
-        "@lancedb/vectordb-win32-x64-msvc": "0.4.11"
+        "@lancedb/vectordb-win32-x64-msvc": "0.4.12"
       },
       "peerDependencies": {
         "@apache-arrow/ts": "^14.0.2",
@@ -334,9 +334,9 @@
       }
     },
     "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.4.11",
+      "version": "0.4.12",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.11.tgz",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.12.tgz",
-      "integrity": "sha512-JDOKmFnuJPFkA7ZmrzBJolROwSjWr7yMvAbi40uLBc25YbbVezodd30u2EFtIwWwtk1GqNYRZ49FZOElKYeC/Q==",
+      "integrity": "sha512-38/rkJRlWXkPWXuj9onzvbrhnIWcIUQjgEp5G9v5ixPosBowm7A4j8e2Q8CJMsVSNcVX2JLqwWVldiWegZFuYw==",
       "cpu": [
         "arm64"
       ],
@@ -346,9 +346,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.4.11",
+      "version": "0.4.12",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.4.11.tgz",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.4.12.tgz",
-      "integrity": "sha512-iy6r+8tp2v1EFgJV52jusXtxgO6NY6SkpOdX41xPqN2mQWMkfUAR9Xtks1mgknjPOIKH4MRc8ZS0jcW/UWmilQ==",
+      "integrity": "sha512-psE48dztyO450hXWdv9Rl9aayM2HQ1uF9wErfC0gKmDUh1N0NdVq2viDuFpZxnmCis/nvGwKlYiYT9OnYNCJ9g==",
       "cpu": [
         "x64"
       ],
@@ -358,9 +358,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.4.11",
+      "version": "0.4.12",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.4.11.tgz",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.4.12.tgz",
-      "integrity": "sha512-5K6IVcTMuH0SZBjlqB5Gg39WC889FpTwIWKufxzQMMXrzxo5J3lKUHVoR28RRlNhDF2d9kZXBEyCpIfDFsV9iQ==",
+      "integrity": "sha512-xwkgF6MiF5aAdG9JG8v4ke652YxUJrhs9z4OrsEfrENnvsIQd2C5UyKMepVLdvij4BI/XPFRFWXdjPvP7S9rTA==",
       "cpu": [
         "arm64"
       ],
@@ -370,9 +370,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.4.11",
+      "version": "0.4.12",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.4.11.tgz",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.4.12.tgz",
-      "integrity": "sha512-hF9ZChsdqKqqnivOzd9mE7lC3PmhZadXtwThi2RrsPiOLoEaGDfmr6Ni3amVQnB3bR8YEJtTxdQxe0NC4uW/8g==",
+      "integrity": "sha512-gJqYR0aymrS+C60xc4EQPzmQ5/69XfeFv2ofBvAj7qW+c6BcnoAcfVl+7s1IrcWeGz251sm5cD5Lx4AzJd89dA==",
       "cpu": [
         "x64"
       ],
@@ -382,9 +382,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.4.11",
+      "version": "0.4.12",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.4.11.tgz",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.4.12.tgz",
-      "integrity": "sha512-0+9ut1ccKoqIyGxsVixwx3771Z+DXpl5WfSmOeA8kf3v3jlOg2H+0YUahiXLDid2ju+yeLPrAUYm7A1gKHVhew==",
+      "integrity": "sha512-LhCzpyEeBUyO6L2fuVqeP3mW8kYDryyU9PNqcM01m88sZB1Do6AlwiM+GjPRQ0SpzD0LK9oxQqSmJrdcNGqjbw==",
       "cpu": [
         "x64"
       ],

python/python/lancedb/remote/table.py

@@ -68,10 +68,14 @@ class RemoteTable(Table):
 
     def list_indices(self):
         """List all the indices on the table"""
-        print(self._name)
         resp = self._conn._client.post(f"/v1/table/{self._name}/index/list/")
         return resp
     
+    def index_stats(self, index_uuid: str):
+        """List all the indices on the table"""
+        resp = self._conn._client.post(f"/v1/table/{self._name}/index/{index_uuid}/stats/")
+        return resp
+
     def create_scalar_index(
         self,
         column: str,

rust/ffi/node/src/table.rs

@@ -323,7 +323,7 @@ impl JsTable {
             .and_then(|val| val.downcast::<JsNumber, _>(&mut cx).ok())
             .map(|val| val.value(&mut cx) as i64)
             .unwrap_or_else(|| 2 * 7 * 24 * 60); // 2 weeks
-        let older_than = chrono::Duration::minutes(older_than);
+        let older_than = chrono::Duration::try_minutes(older_than).unwrap();
         let delete_unverified: Option<bool> = Some(
             cx.argument_opt(1)
                 .and_then(|val| val.downcast::<JsBoolean, _>(&mut cx).ok())

rust/lancedb/src/connection.rs

@@ -796,10 +796,10 @@ mod tests {
         let tmp_dir = tempdir().unwrap();
         let mut names = Vec::with_capacity(100);
         for _ in 0..100 {
-            let name = uuid::Uuid::new_v4().to_string();
+            let mut name = uuid::Uuid::new_v4().to_string();
             names.push(name.clone());
-            let table_name = name + ".lance";
+            name.push_str(".lance");
-            create_dir_all(tmp_dir.path().join(&table_name)).unwrap();
+            create_dir_all(tmp_dir.path().join(&name)).unwrap();
         }
         names.sort();
 

rust/lancedb/src/ipc.rs

@@ -65,7 +65,6 @@ mod tests {
 
     use super::*;
     use arrow_array::{Float32Array, Int64Array, RecordBatch};
-    use arrow_ipc::writer::StreamWriter;
     use arrow_schema::{DataType, Field, Schema};
     use std::sync::Arc;
 

rust/lancedb/src/table.rs

@@ -504,6 +504,13 @@ impl Table {
     }
 }
 
+impl From<NativeTable> for Table {
+    fn from(table: NativeTable) -> Self {
+        Self {
+            inner: Arc::new(table),
+        }
+    }
+}
 /// A table in a LanceDB database.
 #[derive(Debug, Clone)]
 pub struct NativeTable {
@@ -1141,7 +1148,7 @@ impl TableInternal for NativeTable {
                     .compaction;
                 stats.prune = self
                     .optimize(OptimizeAction::Prune {
-                        older_than: Duration::days(7),
+                        older_than: Duration::try_days(7).unwrap(),
                         delete_unverified: None,
                     })
                     .await?

rust/lancedb/tests/lancedb_cloud.rs

@@ -1,67 +0,0 @@
-// Copyright 2024 LanceDB Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::sync::Arc;
-
-use arrow_array::RecordBatchIterator;
-
-#[tokio::test]
-#[ignore]
-async fn cloud_integration_test() {
-    let project = std::env::var("LANCEDB_PROJECT")
-        .expect("the LANCEDB_PROJECT env must be set to run the cloud integration test");
-    let api_key = std::env::var("LANCEDB_API_KEY")
-        .expect("the LANCEDB_API_KEY env must be set to run the cloud integration test");
-    let region = std::env::var("LANCEDB_REGION")
-        .expect("the LANCEDB_REGION env must be set to run the cloud integration test");
-    let host_override = std::env::var("LANCEDB_HOST_OVERRIDE")
-        .map(Some)
-        .unwrap_or(None);
-    if host_override.is_none() {
-        println!("No LANCEDB_HOST_OVERRIDE has been set.  Running integration test against LanceDb Cloud production instance");
-    }
-
-    let mut builder = lancedb::connect(&format!("db://{}", project))
-        .api_key(&api_key)
-        .region(&region);
-    if let Some(host_override) = &host_override {
-        builder = builder.host_override(host_override);
-    }
-    let db = builder.execute().await.unwrap();
-
-    let schema = Arc::new(arrow_schema::Schema::new(vec![
-        arrow_schema::Field::new("id", arrow_schema::DataType::Int64, false),
-        arrow_schema::Field::new("name", arrow_schema::DataType::Utf8, false),
-    ]));
-    let initial_data = arrow::record_batch::RecordBatch::try_new(
-        schema.clone(),
-        vec![
-            Arc::new(arrow_array::Int64Array::from(vec![1, 2, 3])),
-            Arc::new(arrow_array::StringArray::from(vec!["a", "b", "c"])),
-        ],
-    );
-    let rbr = RecordBatchIterator::new(vec![initial_data], schema);
-
-    let name = uuid::Uuid::new_v4().to_string();
-    let tbl = db
-        .create_table(name.clone(), Box::new(rbr))
-        .execute()
-        .await
-        .unwrap();
-
-    assert_eq!(tbl.name(), name);
-
-    let table_names = db.table_names().execute().await.unwrap();
-    assert!(table_names.contains(&name));
-}