feat(python): add Permutation.from_table() with chainable HF/torch-style API

Hides the two-step `permutation_builder(t).shuffle().execute()` /
`Permutation.from_tables(t, perm_tbl)` dance behind a single chainable
entry point that ML engineers already expect from HuggingFace and
PyTorch:

  perm = Permutation.from_table(table).shuffle(seed=42)

Builder operations (shuffle, filter, split_*) are accumulated lazily and
the underlying permutation table is built only once on first read.
Subsequent read / transform / format calls are forwarded transparently
to the materialized Permutation.

Closes lancedb/lancedb#3244

Cargo.toml

@@ -13,40 +13,40 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"
 
 [workspace.dependencies]
-lance = { "version" = "=6.0.0-beta.7", default-features = false, "tag" = "v6.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=6.0.0-beta.7", "tag" = "v6.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=6.0.0-beta.7", "tag" = "v6.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=6.0.0-beta.7", "tag" = "v6.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=6.0.0-beta.7", default-features = false, "tag" = "v6.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=6.0.0-beta.7", "tag" = "v6.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=6.0.0-beta.7", "tag" = "v6.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=6.0.0-beta.7", "tag" = "v6.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=6.0.0-beta.7", default-features = false, "tag" = "v6.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=6.0.0-beta.7", "tag" = "v6.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=6.0.0-beta.7", "tag" = "v6.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=6.0.0-beta.7", "tag" = "v6.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=6.0.0-beta.7", "tag" = "v6.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=6.0.0-beta.7", "tag" = "v6.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=6.0.0-beta.4", default-features = false, "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=6.0.0-beta.4", default-features = false, "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=6.0.0-beta.4", default-features = false, "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
-arrow = { version = "58.0.0", optional = false }
-arrow-array = "58.0.0"
-arrow-data = "58.0.0"
-arrow-ipc = "58.0.0"
-arrow-ord = "58.0.0"
-arrow-schema = "58.0.0"
-arrow-select = "58.0.0"
-arrow-cast = "58.0.0"
+arrow = { version = "57.2", optional = false }
+arrow-array = "57.2"
+arrow-data = "57.2"
+arrow-ipc = "57.2"
+arrow-ord = "57.2"
+arrow-schema = "57.2"
+arrow-select = "57.2"
+arrow-cast = "57.2"
 async-trait = "0"
-datafusion = { version = "53.0.0", default-features = false }
-datafusion-catalog = "53.0.0"
-datafusion-common = { version = "53.0.0", default-features = false }
-datafusion-execution = "53.0.0"
-datafusion-expr = "53.0.0"
-datafusion-functions = "53.0.0"
-datafusion-physical-plan = "53.0.0"
-datafusion-physical-expr = "53.0.0"
-datafusion-sql = "53.0.0"
+datafusion = { version = "52.1", default-features = false }
+datafusion-catalog = "52.1"
+datafusion-common = { version = "52.1", default-features = false }
+datafusion-execution = "52.1"
+datafusion-expr = "52.1"
+datafusion-functions = "52.1"
+datafusion-physical-plan = "52.1"
+datafusion-physical-expr = "52.1"
+datafusion-sql = "52.1"
 env_logger = "0.11"
 half = { "version" = "2.7.1", default-features = false, features = [
     "num-traits",

java/pom.xml

@@ -28,7 +28,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>6.0.0-beta.7</lance-core.version>
+        <lance-core.version>6.0.0-beta.4</lance-core.version>
         <spotless.skip>false</spotless.skip>
         <spotless.version>2.30.0</spotless.version>
         <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>

nodejs/Cargo.toml

@@ -16,7 +16,7 @@ crate-type = ["cdylib"]
 async-trait.workspace = true
 arrow-ipc.workspace = true
 arrow-array.workspace = true
-arrow-buffer = "58.0.0"
+arrow-buffer = "57.2"
 half.workspace = true
 arrow-schema.workspace = true
 env_logger.workspace = true

python/.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.31.0-beta.11"
+current_version = "0.31.0-beta.10"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.31.0-beta.11"
+version = "0.31.0-beta.10"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"
@@ -15,7 +15,7 @@ name = "_lancedb"
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow = { version = "58.0.0", features = ["pyarrow"] }
+arrow = { version = "57.2", features = ["pyarrow"] }
 async-trait = "0.1"
 bytes = "1"
 lancedb = { path = "../rust/lancedb", default-features = false }
@@ -25,8 +25,8 @@ lance-namespace-impls.workspace = true
 lance-io.workspace = true
 env_logger.workspace = true
 log.workspace = true
-pyo3 = { version = "0.28", features = ["extension-module", "abi3-py39"] }
-pyo3-async-runtimes = { version = "0.28", features = [
+pyo3 = { version = "0.26", features = ["extension-module", "abi3-py39"] }
+pyo3-async-runtimes = { version = "0.26", features = [
     "attributes",
     "tokio-runtime",
 ] }
@@ -38,7 +38,7 @@ snafu.workspace = true
 tokio = { version = "1.40", features = ["sync"] }
 
 [build-dependencies]
-pyo3-build-config = { version = "0.28", features = [
+pyo3-build-config = { version = "0.26", features = [
     "extension-module",
     "abi3-py39",
 ] }

python/python/lancedb/permutation.py

@@ -425,6 +425,35 @@ class Permutation:
         """
         return Permutation.from_tables(table, None, None)
 
+    @classmethod
+    def from_table(cls, table: LanceTable) -> "_PermutationFromTable":
+        """
+        Create a permutation directly from a base table, with HuggingFace /
+        PyTorch-style chaining for ``shuffle``, ``filter``, and ``split_*``.
+
+        This is a convenience wrapper that hides the two-step
+        ``permutation_builder(table).shuffle().execute()`` /
+        ``Permutation.from_tables(table, perm_tbl)`` dance. The returned object
+        accumulates builder operations and only materializes the underlying
+        permutation table on first read (any access of an attribute that is
+        not a builder operation), so chained calls do not pay an extra
+        ``execute()`` for each step.
+
+        After the first read all ``Permutation`` methods (``select_columns``,
+        ``with_format``, ``map``, ``__iter__``, ``fetch``, ``num_rows``, ...)
+        are forwarded transparently.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> db = lancedb.connect("memory:///")
+        >>> tbl = db.create_table("tbl", data=[{"x": x} for x in range(100)])
+        >>> perm = Permutation.from_table(tbl).shuffle(seed=42)
+        >>> perm.num_rows
+        100
+        """
+        return _PermutationFromTable(table)
+
     @classmethod
     def from_tables(
         cls,
@@ -842,3 +871,85 @@ class Permutation:
         Repeat the permutation `times` times
         """
         raise Exception("with_repeat is not yet implemented")
+
+
+class _PermutationFromTable:
+    """
+    Result of [Permutation.from_table](#from_table).
+
+    Records pending builder operations (``shuffle``, ``filter``, ``split_*``)
+    and lazily executes them on first read. After materialization all
+    Permutation reads / transforms (``select_columns``, ``with_format``,
+    ``map``, ``__iter__``, ``fetch``, ``num_rows``, ...) are forwarded to the
+    underlying [Permutation].
+    """
+
+    __slots__ = ("_base_table", "_pending_ops", "_materialized")
+
+    def __init__(
+        self,
+        base_table: LanceTable,
+        _pending_ops: Optional[list[tuple[str, tuple, dict]]] = None,
+    ):
+        self._base_table = base_table
+        self._pending_ops: list[tuple[str, tuple, dict]] = (
+            list(_pending_ops) if _pending_ops is not None else []
+        )
+        self._materialized: Optional[Permutation] = None
+
+    def _with_op(
+        self, name: str, args: tuple, kwargs: dict
+    ) -> "_PermutationFromTable":
+        return _PermutationFromTable(
+            self._base_table, _pending_ops=self._pending_ops + [(name, args, kwargs)]
+        )
+
+    def shuffle(self, *args, **kwargs) -> "_PermutationFromTable":
+        return self._with_op("shuffle", args, kwargs)
+
+    def filter(self, *args, **kwargs) -> "_PermutationFromTable":
+        return self._with_op("filter", args, kwargs)
+
+    def split_random(self, *args, **kwargs) -> "_PermutationFromTable":
+        return self._with_op("split_random", args, kwargs)
+
+    def split_sequential(self, *args, **kwargs) -> "_PermutationFromTable":
+        return self._with_op("split_sequential", args, kwargs)
+
+    def split_hash(self, *args, **kwargs) -> "_PermutationFromTable":
+        return self._with_op("split_hash", args, kwargs)
+
+    def split_calculated(self, *args, **kwargs) -> "_PermutationFromTable":
+        return self._with_op("split_calculated", args, kwargs)
+
+    def _materialize(self) -> Permutation:
+        if self._materialized is None:
+            if self._pending_ops:
+                builder = permutation_builder(self._base_table)
+                for name, args, kwargs in self._pending_ops:
+                    builder = getattr(builder, name)(*args, **kwargs)
+                perm_tbl = builder.execute()
+                self._materialized = Permutation.from_tables(
+                    self._base_table, perm_tbl
+                )
+            else:
+                self._materialized = Permutation.identity(self._base_table)
+        return self._materialized
+
+    def __getattr__(self, name: str) -> Any:
+        # Avoid recursion on dunder/private state.
+        if name.startswith("_"):
+            raise AttributeError(name)
+        return getattr(self._materialize(), name)
+
+    def __iter__(self):
+        return iter(self._materialize())
+
+    def __len__(self) -> int:
+        return len(self._materialize())
+
+    def __getitem__(self, index):
+        return self._materialize()[index]
+
+    def __getitems__(self, indices):
+        return self._materialize().__getitems__(indices)

python/python/tests/test_permutation.py

@@ -1095,3 +1095,60 @@ def test_getitems_invalid_offset(some_permutation: Permutation):
     """Test __getitems__ with an out-of-range offset raises an error."""
     with pytest.raises(Exception):
         some_permutation.__getitems__([999999])
+
+
+def test_from_table_identity(mem_db):
+    """Permutation.from_table without ops behaves like identity."""
+    tbl = mem_db.create_table("tbl", pa.table({"x": range(10)}))
+    perm = Permutation.from_table(tbl)
+    assert perm.num_rows == 10
+    assert perm.column_names == ["x"]
+
+
+def test_from_table_shuffle_seeded(mem_db):
+    """from_table().shuffle(seed=...) is reproducible and reorders rows."""
+    tbl = mem_db.create_table("tbl", pa.table({"x": range(100)}))
+    perm = Permutation.from_table(tbl).shuffle(seed=42)
+    rows = [r["x"] for r in perm.__getitems__(list(range(100)))]
+    assert sorted(rows) == list(range(100))
+    assert rows != list(range(100))
+
+    # Same seed → same order
+    rows2 = [
+        r["x"]
+        for r in Permutation.from_table(tbl)
+        .shuffle(seed=42)
+        .__getitems__(list(range(100)))
+    ]
+    assert rows == rows2
+
+
+def test_from_table_filter(mem_db):
+    """from_table().filter(...) limits the rows."""
+    tbl = mem_db.create_table("tbl", pa.table({"x": range(100)}))
+    perm = Permutation.from_table(tbl).filter("x < 25")
+    assert perm.num_rows == 25
+
+
+def test_from_table_chained_ops(mem_db):
+    """Chained shuffle + filter materializes once."""
+    tbl = mem_db.create_table("tbl", pa.table({"x": range(100)}))
+    perm = Permutation.from_table(tbl).filter("x >= 50").shuffle(seed=7)
+    assert perm.num_rows == 50
+    rows = [r["x"] for r in perm.__getitems__(list(range(50)))]
+    assert sorted(rows) == list(range(50, 100))
+
+
+def test_from_table_forwards_read_methods(mem_db):
+    """from_table() result transparently forwards Permutation read methods."""
+    tbl = mem_db.create_table("tbl", pa.table({"x": range(10), "y": range(10)}))
+    perm = Permutation.from_table(tbl).select_columns(["x"])
+    assert perm.column_names == ["x"]
+
+
+def test_from_table_split_random(mem_db):
+    """from_table().split_random(...) returns rows from the first split."""
+    tbl = mem_db.create_table("tbl", pa.table({"x": range(100)}))
+    perm = Permutation.from_table(tbl).split_random(ratios=[0.3, 0.7], seed=1)
+    # Default split is 0 — ratio 0.3 → ~30 rows
+    assert 25 <= perm.num_rows <= 35

python/src/expr.rs

@@ -17,7 +17,7 @@ use pyo3::{Bound, PyAny, PyResult, exceptions::PyValueError, prelude::*, pyfunct
 /// [`expr_lit`] and combined with the methods on this struct.  On the Python
 /// side a thin wrapper class (`lancedb.expr.Expr`) delegates to these methods
 /// and adds Python operator overloads.
-#[pyclass(name = "PyExpr", from_py_object)]
+#[pyclass(name = "PyExpr")]
 #[derive(Clone)]
 pub struct PyExpr(pub DfExpr);
 

python/src/header.rs

@@ -33,7 +33,7 @@ impl PyHeaderProvider {
                 Ok(headers_py) => {
                     // Convert Python dict to Rust HashMap
                     let bound_headers = headers_py.bind(py);
-                    let dict: &Bound<PyDict> = bound_headers.cast().map_err(|e| {
+                    let dict: &Bound<PyDict> = bound_headers.downcast().map_err(|e| {
                         format!("HeaderProvider.get_headers must return a dict: {}", e)
                     })?;
 

python/src/index.rs

@@ -13,7 +13,7 @@ use pyo3::{
     Bound, FromPyObject, PyAny, PyResult, Python,
     exceptions::{PyKeyError, PyValueError},
     intern, pyclass, pymethods,
-    types::{PyAnyMethods, PyString},
+    types::PyAnyMethods,
 };
 
 use crate::util::parse_distance_type;
@@ -22,7 +22,7 @@ pub fn class_name(ob: &'_ Bound<'_, PyAny>) -> PyResult<String> {
     let full_name = ob
         .getattr(intern!(ob.py(), "__class__"))?
         .getattr(intern!(ob.py(), "__name__"))?;
-    let full_name = full_name.cast::<PyString>()?.to_string_lossy();
+    let full_name = full_name.downcast()?.to_string_lossy();
 
     match full_name.rsplit_once('.') {
         Some((_, name)) => Ok(name.to_string()),

python/src/namespace.rs

@@ -183,7 +183,7 @@ async fn call_py_method_primitive<Req, Resp>(
 ) -> lance_core::Result<Resp>
 where
     Req: serde::Serialize + Send + 'static,
-    Resp: for<'a, 'py> pyo3::FromPyObject<'a, 'py> + Send + 'static,
+    Resp: for<'py> pyo3::FromPyObject<'py> + Send + 'static,
 {
     let request_json = serde_json::to_string(&request).map_err(|e| {
         lance_core::Error::io(format!(
@@ -203,7 +203,7 @@ where
 
             // Call the Python method
             let result = py_namespace.call_method1(py, method_name, (request_arg,))?;
-            let value: Resp = result.extract(py).map_err(Into::into)?;
+            let value: Resp = result.extract(py)?;
             Ok::<_, PyErr>(value)
         })
     })

python/src/permutation.rs

@@ -25,12 +25,12 @@ use pyo3_async_runtimes::tokio::future_into_py;
 
 fn table_from_py<'a>(table: Bound<'a, PyAny>) -> PyResult<Bound<'a, Table>> {
     if table.hasattr("_inner")? {
-        Ok(table.getattr("_inner")?.cast_into::<Table>()?)
+        Ok(table.getattr("_inner")?.downcast_into::<Table>()?)
     } else if table.hasattr("_table")? {
         Ok(table
             .getattr("_table")?
             .getattr("_inner")?
-            .cast_into::<Table>()?)
+            .downcast_into::<Table>()?)
     } else {
         Err(PyRuntimeError::new_err(
             "Provided table does not appear to be a Table or RemoteTable instance",
@@ -90,9 +90,9 @@ impl PyAsyncPermutationBuilder {
             database
                 .getattr("_conn")?
                 .getattr("_inner")?
-                .cast_into::<Connection>()?
+                .downcast_into::<Connection>()?
         } else {
-            database.getattr("_inner")?.cast_into::<Connection>()?
+            database.getattr("_inner")?.downcast_into::<Connection>()?
         };
         let database = conn.borrow().database()?;
         slf.modify(|builder| builder.persist(database, table_name))
@@ -243,7 +243,7 @@ impl PyPermutationReader {
         let Some(selection) = selection else {
             return Ok(Select::All);
         };
-        let selection = selection.cast_into::<PyDict>()?;
+        let selection = selection.downcast_into::<PyDict>()?;
         let selection = selection
             .iter()
             .map(|(key, value)| {

python/src/query.rs

@@ -33,7 +33,7 @@ use pyo3::pyfunction;
 use pyo3::pymethods;
 use pyo3::types::PyList;
 use pyo3::types::{PyDict, PyString};
-use pyo3::{Borrowed, FromPyObject, exceptions::PyRuntimeError};
+use pyo3::{FromPyObject, exceptions::PyRuntimeError};
 use pyo3::{PyErr, pyclass};
 use pyo3::{exceptions::PyValueError, intern};
 use pyo3_async_runtimes::tokio::future_into_py;
@@ -43,12 +43,9 @@ use crate::util::parse_distance_type;
 use crate::{arrow::RecordBatchStream, util::PyLanceDB};
 use crate::{error::PythonErrorExt, index::class_name};
 
-impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB<FtsQuery> {
-    type Error = PyErr;
-
-    fn extract(ob: Borrowed<'a, 'py, PyAny>) -> PyResult<Self> {
-        let ob = ob.to_owned();
-        match class_name(&ob)?.as_str() {
+impl FromPyObject<'_> for PyLanceDB<FtsQuery> {
+    fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult<Self> {
+        match class_name(ob)?.as_str() {
             "MatchQuery" => {
                 let query = ob.getattr("query")?.extract()?;
                 let column = ob.getattr("column")?.extract()?;
@@ -427,7 +424,7 @@ impl Query {
                 "Query text is required for nearest_to_text",
             ))?;
 
-        let query = if let Ok(query_text) = fts_query.cast::<PyString>() {
+        let query = if let Ok(query_text) = fts_query.downcast::<PyString>() {
             let mut query_text = query_text.to_string();
             let columns = query
                 .get_item("columns")?
@@ -609,7 +606,7 @@ impl TakeQuery {
     }
 }
 
-#[pyclass(from_py_object)]
+#[pyclass]
 #[derive(Clone)]
 pub struct FTSQuery {
     inner: LanceDbQuery,
@@ -738,7 +735,7 @@ impl FTSQuery {
     }
 }
 
-#[pyclass(from_py_object)]
+#[pyclass]
 #[derive(Clone)]
 pub struct VectorQuery {
     inner: LanceDbVectorQuery,

python/src/session.rs

@@ -11,7 +11,7 @@ use pyo3::{PyResult, pyclass, pymethods};
 /// Sessions allow you to configure cache sizes for index and metadata caches,
 /// which can significantly impact memory use and performance. They can
 /// also be re-used across multiple connections to share the same cache state.
-#[pyclass(from_py_object)]
+#[pyclass]
 #[derive(Clone)]
 pub struct Session {
     pub(crate) inner: Arc<LanceSession>,

python/src/table.rs

@@ -29,7 +29,7 @@ use pyo3_async_runtimes::tokio::future_into_py;
 mod scannable;
 
 /// Statistics about a compaction operation.
-#[pyclass(get_all, from_py_object)]
+#[pyclass(get_all)]
 #[derive(Clone, Debug)]
 pub struct CompactionStats {
     /// The number of fragments removed
@@ -43,7 +43,7 @@ pub struct CompactionStats {
 }
 
 /// Statistics about a cleanup operation
-#[pyclass(get_all, from_py_object)]
+#[pyclass(get_all)]
 #[derive(Clone, Debug)]
 pub struct RemovalStats {
     /// The number of bytes removed
@@ -53,7 +53,7 @@ pub struct RemovalStats {
 }
 
 /// Statistics about an optimize operation
-#[pyclass(get_all, from_py_object)]
+#[pyclass(get_all)]
 #[derive(Clone, Debug)]
 pub struct OptimizeStats {
     /// Statistics about the compaction operation
@@ -62,7 +62,7 @@ pub struct OptimizeStats {
     pub prune: RemovalStats,
 }
 
-#[pyclass(get_all, from_py_object)]
+#[pyclass(get_all)]
 #[derive(Clone, Debug)]
 pub struct UpdateResult {
     pub rows_updated: u64,
@@ -88,7 +88,7 @@ impl From<lancedb::table::UpdateResult> for UpdateResult {
     }
 }
 
-#[pyclass(get_all, from_py_object)]
+#[pyclass(get_all)]
 #[derive(Clone, Debug)]
 pub struct AddResult {
     pub version: u64,
@@ -109,7 +109,7 @@ impl From<lancedb::table::AddResult> for AddResult {
     }
 }
 
-#[pyclass(get_all, from_py_object)]
+#[pyclass(get_all)]
 #[derive(Clone, Debug)]
 pub struct DeleteResult {
     pub num_deleted_rows: u64,
@@ -135,7 +135,7 @@ impl From<lancedb::table::DeleteResult> for DeleteResult {
     }
 }
 
-#[pyclass(get_all, from_py_object)]
+#[pyclass(get_all)]
 #[derive(Clone, Debug)]
 pub struct MergeResult {
     pub version: u64,
@@ -171,7 +171,7 @@ impl From<lancedb::table::MergeResult> for MergeResult {
     }
 }
 
-#[pyclass(get_all, from_py_object)]
+#[pyclass(get_all)]
 #[derive(Clone, Debug)]
 pub struct AddColumnsResult {
     pub version: u64,
@@ -192,7 +192,7 @@ impl From<lancedb::table::AddColumnsResult> for AddColumnsResult {
     }
 }
 
-#[pyclass(get_all, from_py_object)]
+#[pyclass(get_all)]
 #[derive(Clone, Debug)]
 pub struct AlterColumnsResult {
     pub version: u64,
@@ -213,7 +213,7 @@ impl From<lancedb::table::AlterColumnsResult> for AlterColumnsResult {
     }
 }
 
-#[pyclass(get_all, from_py_object)]
+#[pyclass(get_all)]
 #[derive(Clone, Debug)]
 pub struct DropColumnsResult {
     pub version: u64,

python/src/table/scannable.rs

@@ -126,11 +126,8 @@ impl Scannable for PyScannable {
     }
 }
 
-impl<'a, 'py> FromPyObject<'a, 'py> for PyScannable {
-    type Error = pyo3::PyErr;
-
-    fn extract(ob: pyo3::Borrowed<'a, 'py, PyAny>) -> pyo3::PyResult<Self> {
-        let ob = ob.to_owned();
+impl<'py> FromPyObject<'py> for PyScannable {
+    fn extract_bound(ob: &pyo3::Bound<'py, PyAny>) -> pyo3::PyResult<Self> {
         // Convert from Scannable dataclass.
         let schema: PyArrowType<Schema> = ob.getattr("schema")?.extract()?;
         let schema = Arc::new(schema.0);

rust/lancedb/src/remote/table/insert.rs

@@ -43,7 +43,7 @@ pub struct RemoteInsertExec<S: HttpSend = Sender> {
     client: RestfulLanceDbClient<S>,
     input: Arc<dyn ExecutionPlan>,
     overwrite: bool,
-    properties: Arc<PlanProperties>,
+    properties: PlanProperties,
     add_result: Arc<Mutex<Option<AddResult>>>,
     metrics: ExecutionPlanMetricsSet,
     upload_id: Option<String>,
@@ -118,7 +118,7 @@ impl<S: HttpSend + 'static> RemoteInsertExec<S> {
             client,
             input,
             overwrite,
-            properties: Arc::new(properties),
+            properties,
             add_result: Arc::new(Mutex::new(None)),
             metrics: ExecutionPlanMetricsSet::new(),
             upload_id,
@@ -232,7 +232,7 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteInsertExec<S> {
         self
     }
 
-    fn properties(&self) -> &Arc<PlanProperties> {
+    fn properties(&self) -> &PlanProperties {
         &self.properties
     }
 

rust/lancedb/src/table/datafusion.rs

@@ -39,26 +39,21 @@ use lance_index::scalar::FullTextSearchQuery;
 struct MetadataEraserExec {
     input: Arc<dyn ExecutionPlan>,
     schema: Arc<ArrowSchema>,
-    properties: Arc<PlanProperties>,
+    properties: PlanProperties,
 }
 
 impl MetadataEraserExec {
     fn compute_properties_from_input(
         input: &Arc<dyn ExecutionPlan>,
         schema: &Arc<ArrowSchema>,
-    ) -> Arc<PlanProperties> {
+    ) -> PlanProperties {
         let input_properties = input.properties();
         let eq_properties = input_properties
             .eq_properties
             .clone()
             .with_new_schema(schema.clone())
             .unwrap();
-        Arc::new(
-            input_properties
-                .as_ref()
-                .clone()
-                .with_eq_properties(eq_properties),
-        )
+        input_properties.clone().with_eq_properties(eq_properties)
     }
 
     fn new(input: Arc<dyn ExecutionPlan>) -> Self {
@@ -92,7 +87,7 @@ impl ExecutionPlan for MetadataEraserExec {
         self
     }
 
-    fn properties(&self) -> &Arc<PlanProperties> {
+    fn properties(&self) -> &PlanProperties {
         &self.properties
     }
 

rust/lancedb/src/table/datafusion/insert.rs

@@ -81,7 +81,7 @@ pub struct InsertExec {
     dataset: Arc<Dataset>,
     input: Arc<dyn ExecutionPlan>,
     write_params: WriteParams,
-    properties: Arc<PlanProperties>,
+    properties: PlanProperties,
     partial_transactions: Arc<Mutex<Vec<Transaction>>>,
     metrics: ExecutionPlanMetricsSet,
 }
@@ -107,7 +107,7 @@ impl InsertExec {
             dataset,
             input,
             write_params,
-            properties: Arc::new(properties),
+            properties,
             partial_transactions: Arc::new(Mutex::new(Vec::with_capacity(num_partitions))),
             metrics: ExecutionPlanMetricsSet::new(),
         }
@@ -136,7 +136,7 @@ impl ExecutionPlan for InsertExec {
         self
     }
 
-    fn properties(&self) -> &Arc<PlanProperties> {
+    fn properties(&self) -> &PlanProperties {
         &self.properties
     }
 

rust/lancedb/src/table/datafusion/scannable_exec.rs

@@ -20,7 +20,7 @@ pub(crate) struct ScannableExec {
     // We don't require Scannable to be Sync, so we wrap it in a Mutex to allow safe concurrent access.
     source: Mutex<Box<dyn Scannable>>,
     num_rows: Option<usize>,
-    properties: Arc<PlanProperties>,
+    properties: PlanProperties,
     tracker: Option<Arc<WriteProgressTracker>>,
 }
 
@@ -49,7 +49,7 @@ impl ScannableExec {
         Self {
             source,
             num_rows,
-            properties: Arc::new(properties),
+            properties,
             tracker,
         }
     }
@@ -70,7 +70,7 @@ impl ExecutionPlan for ScannableExec {
         self
     }
 
-    fn properties(&self) -> &Arc<PlanProperties> {
+    fn properties(&self) -> &PlanProperties {
         &self.properties
     }