Bump version: 0.2.1 → 0.2.2

This adds LanceTable.restore as a temporary feature. It reads data from
a previous version and creates
a new snapshot version using that data. This makes the version writeable
unlike checkout. This should be replaced once the feature is implemented
in pylance.

Co-authored-by: Chang She <chang@lancedb.com>

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.19
+current_version = 0.2.2
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True

Cargo.toml

@@ -6,7 +6,7 @@ members = [
 resolver = "2"
 
 [workspace.dependencies]
-lance = "=0.6.1"
+lance = "=0.6.2"
 arrow-array = "43.0"
 arrow-data = "43.0"
 arrow-schema = "43.0"
@@ -14,4 +14,3 @@ arrow-ipc = "43.0"
 half = { "version" = "=2.2.1", default-features = false }
 object_store = "0.6.1"
 snafu = "0.7.4"
-

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.1.19",
+  "version": "0.2.1",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.1.19",
+      "version": "0.2.1",
       "cpu": [
         "x64",
         "arm64"
@@ -51,11 +51,11 @@
         "typescript": "*"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.1.19",
-        "@lancedb/vectordb-darwin-x64": "0.1.19",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.1.19",
-        "@lancedb/vectordb-linux-x64-gnu": "0.1.19",
-        "@lancedb/vectordb-win32-x64-msvc": "0.1.19"
+        "@lancedb/vectordb-darwin-arm64": "0.2.1",
+        "@lancedb/vectordb-darwin-x64": "0.2.1",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.2.1",
+        "@lancedb/vectordb-linux-x64-gnu": "0.2.1",
+        "@lancedb/vectordb-win32-x64-msvc": "0.2.1"
       }
     },
     "node_modules/@apache-arrow/ts": {
@@ -315,9 +315,9 @@
       }
     },
     "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.1.19.tgz",
-      "integrity": "sha512-efQhJkBKvMNhjFq3Sw3/qHo9D9gb9UqiIr98n3STsbNxBQjMnWemXn91Ckl40siRG1O8qXcINW7Qs/EGmus+kg==",
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.2.1.tgz",
+      "integrity": "sha512-Hee08Bhz+SDrlt6JdatFNMozHklqbuJ7pJKD8z7CItnHu02kN+1ZwqimBCarwnh5ZAMwBG0LeLTLFOKuTMgphQ==",
       "cpu": [
         "arm64"
       ],
@@ -327,9 +327,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.1.19.tgz",
-      "integrity": "sha512-r6OZNVyemAssABz2w7CRhe7dyREwBEfTytn+ux1zzTnzsgMgDovCQ0rQ3WZcxWvcy7SFCxiemA9IP1b/lsb4tQ==",
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.2.1.tgz",
+      "integrity": "sha512-G9vIPMT9kf0o4VWlXh4CqWWNiWLeshepWAZtbcuemLoBR+Va/5GdnRgn8RBkbAXTb2nia7bzC6BvdcjwDGd/3w==",
       "cpu": [
         "x64"
       ],
@@ -339,9 +339,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.1.19.tgz",
-      "integrity": "sha512-mL/hRmZp6Kw7hmGJBdOZfp/tTYiCdlOcs8DA/+nr2eiXERv0gIhyiKvr2P5DwbBmut3qXEkDalMHTo95BSdL2A==",
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.2.1.tgz",
+      "integrity": "sha512-50a4Tw3x9bk7+V+/thdGyt5AvCdE5p3Y3KOMRrSiXXTnXT9MG0EVU3HVidP8Spwmdemb4kRXcCskOQ4l2EIajg==",
       "cpu": [
         "arm64"
       ],
@@ -351,9 +351,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.1.19.tgz",
-      "integrity": "sha512-AG0FHksbbr+cHVKPi4B8cmBtqb6T9E0uaK4kyZkXrX52/xtv9RYVZcykaB/tSSm0XNFPWWRnx9R8UqNZV/hxMA==",
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.2.1.tgz",
+      "integrity": "sha512-nVPPACGuLjCORh8UdjeTvTpIzzNFPwaQrc6FGPjTNjvPQj5oJnMsjTJuJBgnFvJuHD0tDlfGFgJ4Vl176YVPjw==",
       "cpu": [
         "x64"
       ],
@@ -363,9 +363,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.1.19.tgz",
-      "integrity": "sha512-PDWZ2hvLVXH4Z4WIO1rsWY8ev3NpNm7aXlaey32P+l1Iz9Hia9+F2GBpp2UiEQKfvbk82ucAvBLRmpSsHY8Tlw==",
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.2.1.tgz",
+      "integrity": "sha512-+pH4UKtgZZ6oKSGg1uUtrdBw0LoAszZeT+Fn5mq3RpYY3On+LUuNWyIgencKmOJhuvVoysos/EvDtC5j0RdVOQ==",
       "cpu": [
         "x64"
       ],
@@ -4852,33 +4852,33 @@
       }
     },
     "@lancedb/vectordb-darwin-arm64": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.1.19.tgz",
-      "integrity": "sha512-efQhJkBKvMNhjFq3Sw3/qHo9D9gb9UqiIr98n3STsbNxBQjMnWemXn91Ckl40siRG1O8qXcINW7Qs/EGmus+kg==",
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.2.1.tgz",
+      "integrity": "sha512-Hee08Bhz+SDrlt6JdatFNMozHklqbuJ7pJKD8z7CItnHu02kN+1ZwqimBCarwnh5ZAMwBG0LeLTLFOKuTMgphQ==",
       "optional": true
     },
     "@lancedb/vectordb-darwin-x64": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.1.19.tgz",
-      "integrity": "sha512-r6OZNVyemAssABz2w7CRhe7dyREwBEfTytn+ux1zzTnzsgMgDovCQ0rQ3WZcxWvcy7SFCxiemA9IP1b/lsb4tQ==",
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.2.1.tgz",
+      "integrity": "sha512-G9vIPMT9kf0o4VWlXh4CqWWNiWLeshepWAZtbcuemLoBR+Va/5GdnRgn8RBkbAXTb2nia7bzC6BvdcjwDGd/3w==",
       "optional": true
     },
     "@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.1.19.tgz",
-      "integrity": "sha512-mL/hRmZp6Kw7hmGJBdOZfp/tTYiCdlOcs8DA/+nr2eiXERv0gIhyiKvr2P5DwbBmut3qXEkDalMHTo95BSdL2A==",
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.2.1.tgz",
+      "integrity": "sha512-50a4Tw3x9bk7+V+/thdGyt5AvCdE5p3Y3KOMRrSiXXTnXT9MG0EVU3HVidP8Spwmdemb4kRXcCskOQ4l2EIajg==",
       "optional": true
     },
     "@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.1.19.tgz",
-      "integrity": "sha512-AG0FHksbbr+cHVKPi4B8cmBtqb6T9E0uaK4kyZkXrX52/xtv9RYVZcykaB/tSSm0XNFPWWRnx9R8UqNZV/hxMA==",
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.2.1.tgz",
+      "integrity": "sha512-nVPPACGuLjCORh8UdjeTvTpIzzNFPwaQrc6FGPjTNjvPQj5oJnMsjTJuJBgnFvJuHD0tDlfGFgJ4Vl176YVPjw==",
       "optional": true
     },
     "@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.1.19.tgz",
-      "integrity": "sha512-PDWZ2hvLVXH4Z4WIO1rsWY8ev3NpNm7aXlaey32P+l1Iz9Hia9+F2GBpp2UiEQKfvbk82ucAvBLRmpSsHY8Tlw==",
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.2.1.tgz",
+      "integrity": "sha512-+pH4UKtgZZ6oKSGg1uUtrdBw0LoAszZeT+Fn5mq3RpYY3On+LUuNWyIgencKmOJhuvVoysos/EvDtC5j0RdVOQ==",
       "optional": true
     },
     "@neon-rs/cli": {

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.1.19",
+  "version": "0.2.2",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -78,10 +78,10 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.1.19",
-    "@lancedb/vectordb-darwin-x64": "0.1.19",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.1.19",
-    "@lancedb/vectordb-linux-x64-gnu": "0.1.19",
-    "@lancedb/vectordb-win32-x64-msvc": "0.1.19"
+    "@lancedb/vectordb-darwin-arm64": "0.2.2",
+    "@lancedb/vectordb-darwin-x64": "0.2.2",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.2.2",
+    "@lancedb/vectordb-linux-x64-gnu": "0.2.2",
+    "@lancedb/vectordb-win32-x64-msvc": "0.2.2"
   }
 }

node/src/index.ts

@@ -256,7 +256,16 @@ export class LocalConnection implements Connection {
   async openTable<T> (name: string, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
   async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
   async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
-    const tbl = await databaseOpenTable.call(this._db, name)
+    // TODO: move this thing into rust
+    const callArgs = [this._db, name]
+    if (this._options.awsCredentials !== undefined) {
+      callArgs.push(this._options.awsCredentials.accessKeyId)
+      callArgs.push(this._options.awsCredentials.secretKey)
+      if (this._options.awsCredentials.sessionToken !== undefined) {
+        callArgs.push(this._options.awsCredentials.sessionToken)
+      }
+    }
+    const tbl = await databaseOpenTable.call(...callArgs)
     if (embeddings !== undefined) {
       return new LocalTable(tbl, name, this._options, embeddings)
     } else {

node/src/query.ts

@@ -112,7 +112,8 @@ export class Query<T = number[]> {
       this._queryVector = this._query as number[]
     }
 
-    const buffer = await tableSearch.call(this._tbl, this)
+    const isElectron = this.isElectron()
+    const buffer = await tableSearch.call(this._tbl, this, isElectron)
     const data = tableFromIPC(buffer)
 
     return data.toArray().map((entry: Record<string, unknown>) => {
@@ -127,4 +128,14 @@ export class Query<T = number[]> {
       return newObject as unknown as T
     })
   }
+
+  // See https://github.com/electron/electron/issues/2288
+  private isElectron (): boolean {
+    try {
+      // eslint-disable-next-line no-prototype-builtins
+      return (process?.versions?.hasOwnProperty('electron') || navigator?.userAgent?.toLowerCase()?.includes(' electron'))
+    } catch (e) {
+      return false
+    }
+  }
 }

python/lancedb/table.py

@@ -268,10 +268,11 @@ class LanceTable(Table):
         self.name = name
         self._version = version
 
-    def _reset_dataset(self):
+    def _reset_dataset(self, version=None):
         try:
             if "_dataset" in self.__dict__:
                 del self.__dict__["_dataset"]
+            self._version = version
         except AttributeError:
             pass
 
@@ -297,7 +298,9 @@ class LanceTable(Table):
     def checkout(self, version: int):
         """Checkout a version of the table. This is an in-place operation.
 
-        This allows viewing previous versions of the table.
+        This allows viewing previous versions of the table. If you wish to
+        keep writing to the dataset starting from an old version, then use
+        the `restore` function instead.
 
         Parameters
         ----------
@@ -325,7 +328,49 @@ class LanceTable(Table):
         max_ver = max([v["version"] for v in self._dataset.versions()])
         if version < 1 or version > max_ver:
             raise ValueError(f"Invalid version {version}")
-        self._version = version
+        self._reset_dataset(version=version)
+
+    def restore(self, version: int):
+        """Restore a version of the table. This is an in-place operation.
+
+        This creates a new version where the data is equivalent to the
+        specified previous version. Note that this creates a new snapshot.
+
+        Parameters
+        ----------
+        version : int
+            The version to restore.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> db = lancedb.connect("./.lancedb")
+        >>> table = db.create_table("my_table", [{"vector": [1.1, 0.9], "type": "vector"}])
+        >>> table.version
+        1
+        >>> table.to_pandas()
+               vector    type
+        0  [1.1, 0.9]  vector
+        >>> table.add([{"vector": [0.5, 0.2], "type": "vector"}])
+        >>> table.version
+        2
+        >>> table.restore(1)
+        >>> table.to_pandas()
+               vector    type
+        0  [1.1, 0.9]  vector
+        >>> len(table.list_versions())
+        3
+        """
+        max_ver = max([v["version"] for v in self._dataset.versions()])
+        if version < 1 or version >= max_ver:
+            raise ValueError(f"Invalid version {version}")
+        if version == max_ver:
+            self._reset_dataset()
+            return
+        self.checkout(version)
+        data = self.to_arrow()
+        self.checkout(max_ver)
+        self.add(data, mode="overwrite")
         self._reset_dataset()
 
     def __len__(self):

python/tests/test_table.py

@@ -268,3 +268,15 @@ def test_add_with_nans(db):
     arrow_tbl = table.to_lance().to_table(filter="item == 'bar'")
     v = arrow_tbl["vector"].to_pylist()[0]
     assert np.allclose(v, np.array([0.0, 0.0]))
+
+
+def test_restore(db):
+    table = LanceTable.create(
+        db,
+        "my_table",
+        data=[{"vector": [1.1, 0.9], "type": "vector"}],
+    )
+    table.add([{"vector": [0.5, 0.2], "type": "vector"}])
+    table.restore(1)
+    assert len(table.list_versions()) == 3
+    assert len(table) == 1

rust/ffi/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb-node"
-version = "0.1.19"
+version = "0.2.2"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
 edition = "2018"

rust/ffi/node/src/query.rs

@@ -7,6 +7,7 @@ use lance::index::vector::MetricType;
 use neon::context::FunctionContext;
 use neon::handle::Handle;
 use neon::prelude::*;
+use neon::types::buffer::TypedArray;
 
 use crate::arrow::record_batch_to_buffer;
 use crate::error::ResultExt;
@@ -47,6 +48,8 @@ impl JsQuery {
             .map(|s| s.value(&mut cx))
             .map(|s| MetricType::try_from(s.as_str()).unwrap());
 
+        let is_electron = cx.argument::<JsBoolean>(1).or_throw(&mut cx)?.value(&mut cx);
+
         let rt = runtime(&mut cx)?;
 
         let (deferred, promise) = cx.promise();
@@ -76,9 +79,22 @@ impl JsQuery {
             deferred.settle_with(&channel, move |mut cx| {
                 let results = results.or_throw(&mut cx)?;
                 let buffer = record_batch_to_buffer(results).or_throw(&mut cx)?;
-                Ok(JsBuffer::external(&mut cx, buffer))
+                Self::new_js_buffer(buffer, &mut cx, is_electron)
             });
         });
         Ok(promise)
     }
+
+    // Creates a new JsBuffer from a rust buffer with a special logic for electron
+    fn new_js_buffer<'a>(buffer: Vec<u8>, cx: &mut TaskContext<'a>, is_electron: bool) -> NeonResult<Handle<'a, JsBuffer>> {
+        if is_electron {
+            // Electron does not support `external`: https://github.com/neon-bindings/neon/pull/937
+            let mut js_buffer = JsBuffer::new(cx, buffer.len()).or_throw(cx)?;
+            let buffer_data = js_buffer.as_mut_slice(cx);
+            buffer_data.copy_from_slice(buffer.as_slice());
+            Ok(js_buffer)
+        } else {
+            Ok(JsBuffer::external(cx, buffer))
+        }
+    }
 }

rust/vectordb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb"
-version = "0.1.19"
+version = "0.2.2"
 edition = "2021"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"