feat!: upgrade lance to 0.16 (#1519)

python/python/lancedb/_lancedb.pyi

@@ -24,7 +24,7 @@ class Connection(object):
         mode: str,
         data: pa.RecordBatchReader,
         storage_options: Optional[Dict[str, str]] = None,
-        use_legacy_format: Optional[bool] = None,
+        data_storage_version: Optional[str] = None,
     ) -> Table: ...
     async def create_empty_table(
         self,
@@ -32,7 +32,7 @@ class Connection(object):
         mode: str,
         schema: pa.Schema,
         storage_options: Optional[Dict[str, str]] = None,
-        use_legacy_format: Optional[bool] = None,
+        data_storage_version: Optional[str] = None,
     ) -> Table: ...
 
 class Table:

python/python/lancedb/db.py

@@ -560,6 +560,7 @@ class AsyncConnection(object):
         fill_value: Optional[float] = None,
         storage_options: Optional[Dict[str, str]] = None,
         *,
+        data_storage_version: Optional[str] = None,
         use_legacy_format: Optional[bool] = None,
     ) -> AsyncTable:
         """Create an [AsyncTable][lancedb.table.AsyncTable] in the database.
@@ -603,9 +604,15 @@ class AsyncConnection(object):
             connection will be inherited by the table, but can be overridden here.
             See available options at
             https://lancedb.github.io/lancedb/guides/storage/
-        use_legacy_format: bool, optional, default True
+        data_storage_version: optional, str, default "legacy"
+            The version of the data storage format to use. Newer versions are more
+            efficient but require newer versions of lance to read.  The default is
+            "legacy" which will use the legacy v1 version.  See the user guide
+            for more details.
+        use_legacy_format: bool, optional, default True. (Deprecated)
             If True, use the legacy format for the table. If False, use the new format.
             The default is True while the new format is in beta.
+            This method is deprecated, use `data_storage_version` instead.
 
 
         Returns
@@ -765,13 +772,18 @@ class AsyncConnection(object):
         if mode == "create" and exist_ok:
             mode = "exist_ok"
 
+        if not data_storage_version:
+            data_storage_version = (
+                "legacy" if use_legacy_format is None or use_legacy_format else "stable"
+            )
+
         if data is None:
             new_table = await self._inner.create_empty_table(
                 name,
                 mode,
                 schema,
                 storage_options=storage_options,
-                use_legacy_format=use_legacy_format,
+                data_storage_version=data_storage_version,
             )
         else:
             data = data_to_reader(data, schema)
@@ -780,7 +792,7 @@ class AsyncConnection(object):
                 mode,
                 data,
                 storage_options=storage_options,
-                use_legacy_format=use_legacy_format,
+                data_storage_version=data_storage_version,
             )
 
         return AsyncTable(new_table)

python/python/tests/test_table.py

@@ -730,7 +730,7 @@ def test_create_scalar_index(db):
     indices = table.to_lance().list_indices()
     assert len(indices) == 1
     scalar_index = indices[0]
-    assert scalar_index["type"] == "Scalar"
+    assert scalar_index["type"] == "BTree"
 
     # Confirm that prefiltering still works with the scalar index column
     results = table.search().where("x = 'c'").to_arrow()