feat!: refactor ConnectionInternal into a Database trait (#2067)

This opens up the door for more custom database implementations than the
two we have today. The biggest change should be inivisble:
`ConnectionInternal` has been renamed to `Database`, made public, and
refactored

However, there are a few breaking changes. `data_storage_version` and
`enable_v2_manifest_paths` have been moved from options on
`create_table` to options for the database which are now set via
`storage_options`.

Before:
```
db = connect(uri)
tbl = db.create_table("my_table", data, data_storage_version="legacy", enable_v2_manifest_paths=True)
```

After:
```
db = connect(uri, storage_options={
  "new_table_enable_v2_manifest_paths": "true",
  "new_table_data_storage_version": "legacy"
})
tbl = db.create_table("my_table", data)
```

BREAKING CHANGE: the data_storage_version, enable_v2_manifest_paths
options have moved from options to create_table to storage_options.
BREAKING CHANGE: the use_legacy_format option has been removed,
data_storage_version has replaced it for some time now

python/.gitignore

@@ -0,0 +1,2 @@
+# Test data created by some example tests
+data/

python/python/lancedb/_lancedb.pyi

@@ -15,8 +15,6 @@ class Connection(object):
         mode: str,
         data: pa.RecordBatchReader,
         storage_options: Optional[Dict[str, str]] = None,
-        data_storage_version: Optional[str] = None,
-        enable_v2_manifest_paths: Optional[bool] = None,
     ) -> Table: ...
     async def create_empty_table(
         self,
@@ -24,8 +22,6 @@ class Connection(object):
         mode: str,
         schema: pa.Schema,
         storage_options: Optional[Dict[str, str]] = None,
-        data_storage_version: Optional[str] = None,
-        enable_v2_manifest_paths: Optional[bool] = None,
     ) -> Table: ...
     async def rename_table(self, old_name: str, new_name: str) -> None: ...
     async def drop_table(self, name: str) -> None: ...

python/python/lancedb/db.py

@@ -119,19 +119,11 @@ class DBConnection(EnforceOverrides):
             See available options at
             <https://lancedb.github.io/lancedb/guides/storage/>
         data_storage_version: optional, str, default "stable"
-            The version of the data storage format to use. Newer versions are more
-            efficient but require newer versions of lance to read.  The default is
-            "stable" which will use the legacy v2 version.  See the user guide
-            for more details.
-        enable_v2_manifest_paths: bool, optional, default False
-            Use the new V2 manifest paths. These paths provide more efficient
-            opening of datasets with many versions on object stores.  WARNING:
-            turning this on will make the dataset unreadable for older versions
-            of LanceDB (prior to 0.13.0). To migrate an existing dataset, instead
-            use the
-            [Table.migrate_manifest_paths_v2][lancedb.table.Table.migrate_v2_manifest_paths]
-            method.
-
+            Deprecated.  Set `storage_options` when connecting to the database and set
+            `new_table_data_storage_version` in the options.
+        enable_v2_manifest_paths: optional, bool, default False
+            Deprecated.  Set `storage_options` when connecting to the database and set
+            `new_table_enable_v2_manifest_paths` in the options.
         Returns
         -------
         LanceTable
@@ -452,8 +444,6 @@ class LanceDBConnection(DBConnection):
             fill_value=fill_value,
             embedding_functions=embedding_functions,
             storage_options=storage_options,
-            data_storage_version=data_storage_version,
-            enable_v2_manifest_paths=enable_v2_manifest_paths,
         )
         return tbl
 
@@ -595,9 +585,6 @@ class AsyncConnection(object):
         storage_options: Optional[Dict[str, str]] = None,
         *,
         embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
-        data_storage_version: Optional[str] = None,
-        use_legacy_format: Optional[bool] = None,
-        enable_v2_manifest_paths: Optional[bool] = None,
     ) -> AsyncTable:
         """Create an [AsyncTable][lancedb.table.AsyncTable] in the database.
 
@@ -640,23 +627,6 @@ class AsyncConnection(object):
             connection will be inherited by the table, but can be overridden here.
             See available options at
             <https://lancedb.github.io/lancedb/guides/storage/>
-        data_storage_version: optional, str, default "stable"
-            The version of the data storage format to use. Newer versions are more
-            efficient but require newer versions of lance to read.  The default is
-            "stable" which will use the legacy v2 version.  See the user guide
-            for more details.
-        use_legacy_format: bool, optional, default False. (Deprecated)
-            If True, use the legacy format for the table. If False, use the new format.
-            This method is deprecated, use `data_storage_version` instead.
-        enable_v2_manifest_paths: bool, optional, default False
-            Use the new V2 manifest paths. These paths provide more efficient
-            opening of datasets with many versions on object stores.  WARNING:
-            turning this on will make the dataset unreadable for older versions
-            of LanceDB (prior to 0.13.0). To migrate an existing dataset, instead
-            use the
-            [AsyncTable.migrate_manifest_paths_v2][lancedb.table.AsyncTable.migrate_manifest_paths_v2]
-            method.
-
 
         Returns
         -------
@@ -795,17 +765,12 @@ class AsyncConnection(object):
         if mode == "create" and exist_ok:
             mode = "exist_ok"
 
-        if not data_storage_version:
-            data_storage_version = "legacy" if use_legacy_format else "stable"
-
         if data is None:
             new_table = await self._inner.create_empty_table(
                 name,
                 mode,
                 schema,
                 storage_options=storage_options,
-                data_storage_version=data_storage_version,
-                enable_v2_manifest_paths=enable_v2_manifest_paths,
             )
         else:
             data = data_to_reader(data, schema)
@@ -814,8 +779,6 @@ class AsyncConnection(object):
                 mode,
                 data,
                 storage_options=storage_options,
-                data_storage_version=data_storage_version,
-                enable_v2_manifest_paths=enable_v2_manifest_paths,
             )
 
         return AsyncTable(new_table)

python/python/lancedb/table.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 import inspect
+import warnings
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from datetime import datetime, timedelta
@@ -2085,10 +2086,37 @@ class LanceTable(Table):
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
         embedding_functions: list of EmbeddingFunctionModel, default None
             The embedding functions to use when creating the table.
+        data_storage_version: optional, str, default "stable"
+            Deprecated.  Set `storage_options` when connecting to the database and set
+            `new_table_data_storage_version` in the options.
+        enable_v2_manifest_paths: optional, bool, default False
+            Deprecated.  Set `storage_options` when connecting to the database and set
+            `new_table_enable_v2_manifest_paths` in the options.
         """
         self = cls.__new__(cls)
         self._conn = db
 
+        if data_storage_version is not None:
+            warnings.warn(
+                "setting data_storage_version directly on create_table is deprecated. ",
+                "Use database_options instead.",
+                DeprecationWarning,
+            )
+            if storage_options is None:
+                storage_options = {}
+            storage_options["new_table_data_storage_version"] = data_storage_version
+        if enable_v2_manifest_paths is not None:
+            warnings.warn(
+                "setting enable_v2_manifest_paths directly on create_table is ",
+                "deprecated. Use database_options instead.",
+                DeprecationWarning,
+            )
+            if storage_options is None:
+                storage_options = {}
+            storage_options["new_table_enable_v2_manifest_paths"] = (
+                enable_v2_manifest_paths
+            )
+
         self._table = LOOP.run(
             self._conn._conn.create_table(
                 name,
@@ -2100,8 +2128,6 @@ class LanceTable(Table):
                 fill_value=fill_value,
                 embedding_functions=embedding_functions,
                 storage_options=storage_options,
-                data_storage_version=data_storage_version,
-                enable_v2_manifest_paths=enable_v2_manifest_paths,
             )
         )
         return self

python/python/tests/test_db.py

@@ -299,12 +299,12 @@ def test_create_exist_ok(tmp_db: lancedb.DBConnection):
 @pytest.mark.asyncio
 async def test_connect(tmp_path):
     db = await lancedb.connect_async(tmp_path)
-    assert str(db) == f"NativeDatabase(uri={tmp_path}, read_consistency_interval=None)"
+    assert str(db) == f"ListingDatabase(uri={tmp_path}, read_consistency_interval=None)"
 
     db = await lancedb.connect_async(
         tmp_path, read_consistency_interval=timedelta(seconds=5)
     )
-    assert str(db) == f"NativeDatabase(uri={tmp_path}, read_consistency_interval=5s)"
+    assert str(db) == f"ListingDatabase(uri={tmp_path}, read_consistency_interval=5s)"
 
 
 @pytest.mark.asyncio
@@ -396,13 +396,16 @@ async def test_create_exist_ok_async(tmp_db_async: lancedb.AsyncConnection):
 
 @pytest.mark.asyncio
 async def test_create_table_v2_manifest_paths_async(tmp_path):
-    db = await lancedb.connect_async(tmp_path)
+    db_with_v2_paths = await lancedb.connect_async(
+        tmp_path, storage_options={"new_table_enable_v2_manifest_paths": "true"}
+    )
+    db_no_v2_paths = await lancedb.connect_async(
+        tmp_path, storage_options={"new_table_enable_v2_manifest_paths": "false"}
+    )
     # Create table in v2 mode with v2 manifest paths enabled
-    tbl = await db.create_table(
+    tbl = await db_with_v2_paths.create_table(
         "test_v2_manifest_paths",
         data=[{"id": 0}],
-        use_legacy_format=False,
-        enable_v2_manifest_paths=True,
     )
     assert await tbl.uses_v2_manifest_paths()
     manifests_dir = tmp_path / "test_v2_manifest_paths.lance" / "_versions"
@@ -410,11 +413,9 @@ async def test_create_table_v2_manifest_paths_async(tmp_path):
         assert re.match(r"\d{20}\.manifest", manifest)
 
     # Start a table in V1 mode then migrate
-    tbl = await db.create_table(
+    tbl = await db_no_v2_paths.create_table(
         "test_v2_migration",
         data=[{"id": 0}],
-        use_legacy_format=False,
-        enable_v2_manifest_paths=False,
     )
     assert not await tbl.uses_v2_manifest_paths()
     manifests_dir = tmp_path / "test_v2_migration.lance" / "_versions"
@@ -583,7 +584,7 @@ def test_empty_or_nonexistent_table(mem_db: lancedb.DBConnection):
 
 
 @pytest.mark.asyncio
-async def test_create_in_v2_mode(mem_db_async: lancedb.AsyncConnection):
+async def test_create_in_v2_mode():
     def make_data():
         for i in range(10):
             yield pa.record_batch([pa.array([x for x in range(1024)])], names=["x"])
@@ -594,10 +595,13 @@ async def test_create_in_v2_mode(mem_db_async: lancedb.AsyncConnection):
     schema = pa.schema([pa.field("x", pa.int64())])
 
     # Create table in v1 mode
-    tbl = await mem_db_async.create_table(
-        "test", data=make_data(), schema=schema, data_storage_version="legacy"
+
+    v1_db = await lancedb.connect_async(
+        "memory://", storage_options={"new_table_data_storage_version": "legacy"}
     )
 
+    tbl = await v1_db.create_table("test", data=make_data(), schema=schema)
+
     async def is_in_v2_mode(tbl):
         batches = (
             await tbl.query().limit(10 * 1024).to_batches(max_batch_length=1024 * 10)
@@ -610,10 +614,12 @@ async def test_create_in_v2_mode(mem_db_async: lancedb.AsyncConnection):
     assert not await is_in_v2_mode(tbl)
 
     # Create table in v2 mode
-    tbl = await mem_db_async.create_table(
-        "test_v2", data=make_data(), schema=schema, use_legacy_format=False
+    v2_db = await lancedb.connect_async(
+        "memory://", storage_options={"new_table_data_storage_version": "stable"}
     )
 
+    tbl = await v2_db.create_table("test_v2", data=make_data(), schema=schema)
+
     assert await is_in_v2_mode(tbl)
 
     # Add data (should remain in v2 mode)
@@ -622,20 +628,18 @@ async def test_create_in_v2_mode(mem_db_async: lancedb.AsyncConnection):
     assert await is_in_v2_mode(tbl)
 
     # Create empty table in v2 mode and add data
-    tbl = await mem_db_async.create_table(
-        "test_empty_v2", data=None, schema=schema, use_legacy_format=False
-    )
+    tbl = await v2_db.create_table("test_empty_v2", data=None, schema=schema)
     await tbl.add(make_table())
 
     assert await is_in_v2_mode(tbl)
 
-    # Create empty table uses v1 mode by default
-    tbl = await mem_db_async.create_table(
-        "test_empty_v2_default", data=None, schema=schema, data_storage_version="legacy"
-    )
+    # Db uses v2 mode by default
+    db = await lancedb.connect_async("memory://")
+
+    tbl = await db.create_table("test_empty_v2_default", data=None, schema=schema)
     await tbl.add(make_table())
 
-    assert not await is_in_v2_mode(tbl)
+    assert await is_in_v2_mode(tbl)
 
 
 def test_replace_index(mem_db: lancedb.DBConnection):

python/src/connection.rs

@@ -1,10 +1,10 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-use std::{collections::HashMap, str::FromStr, sync::Arc, time::Duration};
+use std::{collections::HashMap, sync::Arc, time::Duration};
 
 use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::FromPyArrow};
-use lancedb::connection::{Connection as LanceConnection, CreateTableMode, LanceFileVersion};
+use lancedb::{connection::Connection as LanceConnection, database::CreateTableMode};
 use pyo3::{
     exceptions::{PyRuntimeError, PyValueError},
     pyclass, pyfunction, pymethods, Bound, FromPyObject, PyAny, PyRef, PyResult, Python,
@@ -80,15 +80,13 @@ impl Connection {
         future_into_py(self_.py(), async move { op.execute().await.infer_error() })
     }
 
-    #[pyo3(signature = (name, mode, data, storage_options=None, data_storage_version=None, enable_v2_manifest_paths=None))]
+    #[pyo3(signature = (name, mode, data, storage_options=None))]
     pub fn create_table<'a>(
         self_: PyRef<'a, Self>,
         name: String,
         mode: &str,
         data: Bound<'_, PyAny>,
         storage_options: Option<HashMap<String, String>>,
-        data_storage_version: Option<String>,
-        enable_v2_manifest_paths: Option<bool>,
     ) -> PyResult<Bound<'a, PyAny>> {
         let inner = self_.get_inner()?.clone();
 
@@ -101,32 +99,19 @@ impl Connection {
             builder = builder.storage_options(storage_options);
         }
 
-        if let Some(enable_v2_manifest_paths) = enable_v2_manifest_paths {
-            builder = builder.enable_v2_manifest_paths(enable_v2_manifest_paths);
-        }
-
-        if let Some(data_storage_version) = data_storage_version.as_ref() {
-            builder = builder.data_storage_version(
-                LanceFileVersion::from_str(data_storage_version)
-                    .map_err(|e| PyValueError::new_err(e.to_string()))?,
-            );
-        }
-
         future_into_py(self_.py(), async move {
             let table = builder.execute().await.infer_error()?;
             Ok(Table::new(table))
         })
     }
 
-    #[pyo3(signature = (name, mode, schema, storage_options=None, data_storage_version=None, enable_v2_manifest_paths=None))]
+    #[pyo3(signature = (name, mode, schema, storage_options=None))]
     pub fn create_empty_table<'a>(
         self_: PyRef<'a, Self>,
         name: String,
         mode: &str,
         schema: Bound<'_, PyAny>,
         storage_options: Option<HashMap<String, String>>,
-        data_storage_version: Option<String>,
-        enable_v2_manifest_paths: Option<bool>,
     ) -> PyResult<Bound<'a, PyAny>> {
         let inner = self_.get_inner()?.clone();
 
@@ -140,17 +125,6 @@ impl Connection {
             builder = builder.storage_options(storage_options);
         }
 
-        if let Some(enable_v2_manifest_paths) = enable_v2_manifest_paths {
-            builder = builder.enable_v2_manifest_paths(enable_v2_manifest_paths);
-        }
-
-        if let Some(data_storage_version) = data_storage_version.as_ref() {
-            builder = builder.data_storage_version(
-                LanceFileVersion::from_str(data_storage_version)
-                    .map_err(|e| PyValueError::new_err(e.to_string()))?,
-            );
-        }
-
         future_into_py(self_.py(), async move {
             let table = builder.execute().await.infer_error()?;
             Ok(Table::new(table))