fix(python): support Polars 1.32 table scans (#3801)

## Root cause

`Table.to_polars()` disabled PyArrow predicate pushdown by selecting the
non-PyArrow Polars scan callback. Polars 1.32.3 invokes that callback
with `batch_size` both positionally and through its partial, so
collecting the returned lazy frame raises `TypeError:
_scan_pyarrow_dataset_impl() got multiple values for argument
batch_size`.

## Fix

- Keep the compatible PyArrow callback path.
- Add an identity `map_batches` barrier so predicates stay in Polars
instead of reaching the LanceDB adapter as unsupported PyArrow
expressions.
- Extend the tested Polars range through 1.32.3 and retain lazy-frame
regression coverage.

## Validation

- `python/tests/test_table.py::test_polars` with Polars 1.32.3
- `python/tests/test_table.py::test_polars` with the locked Polars 1.3.0
baseline
- `ruff format --check` on the changed Python files
- `ruff check .`
- `uv lock --check`

Fixes #2619

<!-- lance-gatekeeper-fix:v1 agent=0d42bcda944ac42765b25f2c19ff729f
generation=1 -->

---------

Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>

python/pyproject.toml

@@ -60,7 +60,7 @@ tests = [
     "pytest-asyncio>=0.21",
     "duckdb>=0.9.0",
     "pytz>=2023.3",
-    "polars>=0.19, <=1.3.0",
+    "polars>=0.19, <=1.32.3",
     "pyarrow<25",
     "pyarrow-stubs>=16.0",
     "pylance==9.0.0rc1",

python/python/lancedb/table.py

@@ -108,6 +108,11 @@ def _should_push_down_query_table(
     return namespace_client is not None and "QueryTable" in pushdown_operations
 
 
+def _polars_predicate_pushdown_barrier(frame: Any) -> Any:
+    """Return a Polars frame unchanged while blocking predicate pushdown."""
+    return frame
+
+
 _MODEL_BACKED_TOKENIZER_PREFIXES = ("jieba", "lindera")
 _MODEL_BACKED_TOKENIZER_ERRORS = (
     "unknown base tokenizer",
@@ -864,12 +869,18 @@ class Table(ABC):
         """
         raise NotImplementedError
 
-    def to_polars(self, **kwargs) -> "pl.DataFrame":
-        """Return the table as a polars.DataFrame.
+    def to_polars(self, **kwargs) -> "pl.LazyFrame":
+        """Return the table as a Polars LazyFrame.
+
+        Note
+        ----
+        The Polars streaming engine is not supported because it does not currently
+        implement Python PyArrow dataset scans. Use the default engine when collecting
+        this LazyFrame.
 
         Returns
         -------
-        polars.DataFrame
+        polars.LazyFrame
         """
         raise NotImplementedError
 
@@ -2569,6 +2580,9 @@ class LanceTable(Table):
         2. Currently we've disabled push-down of the filters from polars
            because polars pushdown into pyarrow uses pyarrow compute
            expressions rather than SQl strings (which LanceDB supports)
+        3. The Polars streaming engine is not supported because it does not
+           currently implement Python PyArrow dataset scans. Use the default
+           engine when collecting this LazyFrame.
 
         Returns
         -------
@@ -2577,8 +2591,12 @@ class LanceTable(Table):
         from lancedb.integrations.pyarrow import PyarrowDatasetAdapter
 
         dataset = PyarrowDatasetAdapter(self)
-        return pl.scan_pyarrow_dataset(
-            dataset, allow_pyarrow_filter=False, batch_size=batch_size
+        # Polars 1.32's non-PyArrow callback path passes batch_size twice.  Keep
+        # the compatible PyArrow path, but block predicates because this adapter
+        # cannot translate PyArrow expressions into LanceDB filters.
+        return pl.scan_pyarrow_dataset(dataset, batch_size=batch_size).map_batches(
+            _polars_predicate_pushdown_barrier,
+            predicate_pushdown=False,
         )
 
     # New unified API overload

python/python/tests/test_table.py

@@ -929,6 +929,7 @@ def test_polars(mem_db: DBConnection):
 
     # enter table to polars dataframe
     result = table.to_polars()
+    assert isinstance(result, pl.LazyFrame)
     assert np.allclose(result.collect()["vector"].to_list(), data["vector"])
 
     # make sure filtering isn't broken

python/uv.lock

@@ -1998,7 +1998,7 @@ requires-dist = [
     { name = "pillow", marker = "extra == 'clip'", specifier = ">=12.1.1" },
     { name = "pillow", marker = "extra == 'embeddings'", specifier = ">=12.1.1" },
     { name = "pillow", marker = "extra == 'siglip'", specifier = ">=12.1.1" },
-    { name = "polars", marker = "extra == 'tests'", specifier = ">=0.19,<=1.3.0" },
+    { name = "polars", marker = "extra == 'tests'", specifier = ">=0.19,<=1.32.3" },
     { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.5.0" },
     { name = "pyarrow", specifier = ">=16" },
     { name = "pyarrow", marker = "extra == 'tests'", specifier = "<25" },