diff --git a/python/pyproject.toml b/python/pyproject.toml index 348058957..ce71484de 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -60,7 +60,7 @@ tests = [ "pytest-asyncio>=0.21", "duckdb>=0.9.0", "pytz>=2023.3", - "polars>=0.19, <=1.3.0", + "polars>=0.19, <=1.32.3", "pyarrow<25", "pyarrow-stubs>=16.0", "pylance==9.0.0rc1", diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index ae36bac7a..59e2650eb 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -108,6 +108,11 @@ def _should_push_down_query_table( return namespace_client is not None and "QueryTable" in pushdown_operations +def _polars_predicate_pushdown_barrier(frame: Any) -> Any: + """Return a Polars frame unchanged while blocking predicate pushdown.""" + return frame + + _MODEL_BACKED_TOKENIZER_PREFIXES = ("jieba", "lindera") _MODEL_BACKED_TOKENIZER_ERRORS = ( "unknown base tokenizer", @@ -864,12 +869,18 @@ class Table(ABC): """ raise NotImplementedError - def to_polars(self, **kwargs) -> "pl.DataFrame": - """Return the table as a polars.DataFrame. + def to_polars(self, **kwargs) -> "pl.LazyFrame": + """Return the table as a Polars LazyFrame. + + Note + ---- + The Polars streaming engine is not supported because it does not currently + implement Python PyArrow dataset scans. Use the default engine when collecting + this LazyFrame. Returns ------- - polars.DataFrame + polars.LazyFrame """ raise NotImplementedError @@ -2569,6 +2580,9 @@ class LanceTable(Table): 2. Currently we've disabled push-down of the filters from polars because polars pushdown into pyarrow uses pyarrow compute expressions rather than SQl strings (which LanceDB supports) + 3. The Polars streaming engine is not supported because it does not + currently implement Python PyArrow dataset scans. Use the default + engine when collecting this LazyFrame. Returns ------- @@ -2577,8 +2591,12 @@ class LanceTable(Table): from lancedb.integrations.pyarrow import PyarrowDatasetAdapter dataset = PyarrowDatasetAdapter(self) - return pl.scan_pyarrow_dataset( - dataset, allow_pyarrow_filter=False, batch_size=batch_size + # Polars 1.32's non-PyArrow callback path passes batch_size twice. Keep + # the compatible PyArrow path, but block predicates because this adapter + # cannot translate PyArrow expressions into LanceDB filters. + return pl.scan_pyarrow_dataset(dataset, batch_size=batch_size).map_batches( + _polars_predicate_pushdown_barrier, + predicate_pushdown=False, ) # New unified API overload diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 8fc06ea69..4ad5d7c3d 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -929,6 +929,7 @@ def test_polars(mem_db: DBConnection): # enter table to polars dataframe result = table.to_polars() + assert isinstance(result, pl.LazyFrame) assert np.allclose(result.collect()["vector"].to_list(), data["vector"]) # make sure filtering isn't broken diff --git a/python/uv.lock b/python/uv.lock index 551dc3f68..2cdcb182e 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -1998,7 +1998,7 @@ requires-dist = [ { name = "pillow", marker = "extra == 'clip'", specifier = ">=12.1.1" }, { name = "pillow", marker = "extra == 'embeddings'", specifier = ">=12.1.1" }, { name = "pillow", marker = "extra == 'siglip'", specifier = ">=12.1.1" }, - { name = "polars", marker = "extra == 'tests'", specifier = ">=0.19,<=1.3.0" }, + { name = "polars", marker = "extra == 'tests'", specifier = ">=0.19,<=1.32.3" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.5.0" }, { name = "pyarrow", specifier = ">=16" }, { name = "pyarrow", marker = "extra == 'tests'", specifier = "<25" },