feat!: upgrade lance to 0.16 (#1519)

nodejs/Cargo.toml

@@ -20,7 +20,6 @@ napi = { version = "2.16.8", default-features = false, features = [
     "async",
 ] }
 napi-derive = "2.16.4"
-
 # Prevent dynamic linking of lzma, which comes from datafusion
 lzma-sys = { version = "*", features = ["static"] }
 

nodejs/lancedb/connection.ts

@@ -44,10 +44,20 @@ export interface CreateTableOptions {
    * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
    */
   storageOptions?: Record<string, string>;
+  /**
+   * The version of the data storage format to use.
+   *
+   * The default is `legacy`, which is Lance format v1.
+   * `stable` is the new format, which is Lance format v2.
+   */
+  dataStorageVersion?: string;
+
   /**
    * If true then data files will be written with the legacy format
    *
    * The default is true while the new format is in beta
+   *
+   * Deprecated.
    */
   useLegacyFormat?: boolean;
   schema?: SchemaLike;
@@ -247,12 +257,19 @@ export class LocalConnection extends Connection {
       throw new Error("data is required");
     }
     const { buf, mode } = await Table.parseTableData(data, options);
+    let dataStorageVersion = "legacy";
+    if (options?.dataStorageVersion !== undefined) {
+      dataStorageVersion = options.dataStorageVersion;
+    } else if (options?.useLegacyFormat !== undefined) {
+      dataStorageVersion = options.useLegacyFormat ? "legacy" : "stable";
+    }
+
     const innerTable = await this.inner.createTable(
       nameOrOptions,
       buf,
       mode,
       cleanseStorageOptions(options?.storageOptions),
-      options?.useLegacyFormat,
+      dataStorageVersion,
     );
 
     return new LocalTable(innerTable);
@@ -276,6 +293,13 @@ export class LocalConnection extends Connection {
       metadata = registry.getTableMetadata([embeddingFunction]);
     }
 
+    let dataStorageVersion = "legacy";
+    if (options?.dataStorageVersion !== undefined) {
+      dataStorageVersion = options.dataStorageVersion;
+    } else if (options?.useLegacyFormat !== undefined) {
+      dataStorageVersion = options.useLegacyFormat ? "legacy" : "stable";
+    }
+
     const table = makeEmptyTable(schema, metadata);
     const buf = await fromTableToBuffer(table);
     const innerTable = await this.inner.createEmptyTable(
@@ -283,7 +307,7 @@ export class LocalConnection extends Connection {
       buf,
       mode,
       cleanseStorageOptions(options?.storageOptions),
-      options?.useLegacyFormat,
+      dataStorageVersion,
     );
     return new LocalTable(innerTable);
   }

nodejs/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@lancedb/lancedb",
-  "version": "0.7.2",
+  "version": "0.8.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@lancedb/lancedb",
-      "version": "0.7.2",
+      "version": "0.8.0",
       "cpu": [
         "x64",
         "arm64"

nodejs/src/connection.rs

@@ -13,13 +13,16 @@
 // limitations under the License.
 
 use std::collections::HashMap;
+use std::str::FromStr;
 
 use napi::bindgen_prelude::*;
 use napi_derive::*;
 
 use crate::table::Table;
 use crate::ConnectionOptions;
-use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection, CreateTableMode};
+use lancedb::connection::{
+    ConnectBuilder, Connection as LanceDBConnection, CreateTableMode, LanceFileVersion,
+};
 use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
 
 #[napi]
@@ -120,7 +123,7 @@ impl Connection {
         buf: Buffer,
         mode: String,
         storage_options: Option<HashMap<String, String>>,
-        use_legacy_format: Option<bool>,
+        data_storage_options: Option<String>,
     ) -> napi::Result<Table> {
         let batches = ipc_file_to_batches(buf.to_vec())
             .map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
@@ -131,8 +134,11 @@ impl Connection {
                 builder = builder.storage_option(key, value);
             }
         }
-        if let Some(use_legacy_format) = use_legacy_format {
-            builder = builder.use_legacy_format(use_legacy_format);
+        if let Some(data_storage_option) = data_storage_options.as_ref() {
+            builder = builder.data_storage_version(
+                LanceFileVersion::from_str(data_storage_option)
+                    .map_err(|e| napi::Error::from_reason(format!("{}", e)))?,
+            );
         }
         let tbl = builder
             .execute()
@@ -148,7 +154,7 @@ impl Connection {
         schema_buf: Buffer,
         mode: String,
         storage_options: Option<HashMap<String, String>>,
-        use_legacy_format: Option<bool>,
+        data_storage_options: Option<String>,
     ) -> napi::Result<Table> {
         let schema = ipc_file_to_schema(schema_buf.to_vec()).map_err(|e| {
             napi::Error::from_reason(format!("Failed to marshal schema from JS to Rust: {}", e))
@@ -163,8 +169,11 @@ impl Connection {
                 builder = builder.storage_option(key, value);
             }
         }
-        if let Some(use_legacy_format) = use_legacy_format {
-            builder = builder.use_legacy_format(use_legacy_format);
+        if let Some(data_storage_option) = data_storage_options.as_ref() {
+            builder = builder.data_storage_version(
+                LanceFileVersion::from_str(data_storage_option)
+                    .map_err(|e| napi::Error::from_reason(format!("{}", e)))?,
+            );
         }
         let tbl = builder
             .execute()

nodejs/src/table.rs

@@ -293,6 +293,7 @@ impl Table {
             .optimize(OptimizeAction::Prune {
                 older_than,
                 delete_unverified: None,
+                error_if_tagged_old_versions: None,
             })
             .await
             .default_error()?