From 564e5d0d56802bfd2c025e9337b8dd810d42759a Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:32:17 +0800 Subject: [PATCH] fix(python): support Polars 1.32 table scans (#3801) ## Root cause `Table.to_polars()` disabled PyArrow predicate pushdown by selecting the non-PyArrow Polars scan callback. Polars 1.32.3 invokes that callback with `batch_size` both positionally and through its partial, so collecting the returned lazy frame raises `TypeError: _scan_pyarrow_dataset_impl() got multiple values for argument batch_size`. ## Fix - Keep the compatible PyArrow callback path. - Add an identity `map_batches` barrier so predicates stay in Polars instead of reaching the LanceDB adapter as unsupported PyArrow expressions. - Extend the tested Polars range through 1.32.3 and retain lazy-frame regression coverage. ## Validation - `python/tests/test_table.py::test_polars` with Polars 1.32.3 - `python/tests/test_table.py::test_polars` with the locked Polars 1.3.0 baseline - `ruff format --check` on the changed Python files - `ruff check .` - `uv lock --check` Fixes #2619 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- python/pyproject.toml | 2 +- python/python/lancedb/table.py | 28 +++++++++++++++++++++++----- python/python/tests/test_table.py | 1 + python/uv.lock | 2 +- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 348058957..ce71484de 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -60,7 +60,7 @@ tests = [ "pytest-asyncio>=0.21", "duckdb>=0.9.0", "pytz>=2023.3", - "polars>=0.19, <=1.3.0", + "polars>=0.19, <=1.32.3", "pyarrow<25", "pyarrow-stubs>=16.0", "pylance==9.0.0rc1", diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index ae36bac7a..59e2650eb 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -108,6 +108,11 @@ def _should_push_down_query_table( return namespace_client is not None and "QueryTable" in pushdown_operations +def _polars_predicate_pushdown_barrier(frame: Any) -> Any: + """Return a Polars frame unchanged while blocking predicate pushdown.""" + return frame + + _MODEL_BACKED_TOKENIZER_PREFIXES = ("jieba", "lindera") _MODEL_BACKED_TOKENIZER_ERRORS = ( "unknown base tokenizer", @@ -864,12 +869,18 @@ class Table(ABC): """ raise NotImplementedError - def to_polars(self, **kwargs) -> "pl.DataFrame": - """Return the table as a polars.DataFrame. + def to_polars(self, **kwargs) -> "pl.LazyFrame": + """Return the table as a Polars LazyFrame. + + Note + ---- + The Polars streaming engine is not supported because it does not currently + implement Python PyArrow dataset scans. Use the default engine when collecting + this LazyFrame. Returns ------- - polars.DataFrame + polars.LazyFrame """ raise NotImplementedError @@ -2569,6 +2580,9 @@ class LanceTable(Table): 2. Currently we've disabled push-down of the filters from polars because polars pushdown into pyarrow uses pyarrow compute expressions rather than SQl strings (which LanceDB supports) + 3. The Polars streaming engine is not supported because it does not + currently implement Python PyArrow dataset scans. Use the default + engine when collecting this LazyFrame. Returns ------- @@ -2577,8 +2591,12 @@ class LanceTable(Table): from lancedb.integrations.pyarrow import PyarrowDatasetAdapter dataset = PyarrowDatasetAdapter(self) - return pl.scan_pyarrow_dataset( - dataset, allow_pyarrow_filter=False, batch_size=batch_size + # Polars 1.32's non-PyArrow callback path passes batch_size twice. Keep + # the compatible PyArrow path, but block predicates because this adapter + # cannot translate PyArrow expressions into LanceDB filters. + return pl.scan_pyarrow_dataset(dataset, batch_size=batch_size).map_batches( + _polars_predicate_pushdown_barrier, + predicate_pushdown=False, ) # New unified API overload diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 8fc06ea69..4ad5d7c3d 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -929,6 +929,7 @@ def test_polars(mem_db: DBConnection): # enter table to polars dataframe result = table.to_polars() + assert isinstance(result, pl.LazyFrame) assert np.allclose(result.collect()["vector"].to_list(), data["vector"]) # make sure filtering isn't broken diff --git a/python/uv.lock b/python/uv.lock index 551dc3f68..2cdcb182e 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -1998,7 +1998,7 @@ requires-dist = [ { name = "pillow", marker = "extra == 'clip'", specifier = ">=12.1.1" }, { name = "pillow", marker = "extra == 'embeddings'", specifier = ">=12.1.1" }, { name = "pillow", marker = "extra == 'siglip'", specifier = ">=12.1.1" }, - { name = "polars", marker = "extra == 'tests'", specifier = ">=0.19,<=1.3.0" }, + { name = "polars", marker = "extra == 'tests'", specifier = ">=0.19,<=1.32.3" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.5.0" }, { name = "pyarrow", specifier = ">=16" }, { name = "pyarrow", marker = "extra == 'tests'", specifier = "<25" },