From 9f4cd1618bddf1041c8ebdaa0b0ccb33656dc57c Mon Sep 17 00:00:00 2001 From: adenzhou1350 Date: Fri, 23 Oct 2026 03:19:06 +0800 Subject: [PATCH] fix(python): handle empty batch iterables with a schema (#4538) ## Summary `create_table(data=iter(()), schema=...)` currently raises `RuntimeError: coroutine raised StopIteration`, although an empty list with the same schema works. A tuple of batches also fails because the helper calls `next()` before obtaining an iterator. Normalize the iterable once and use the supplied schema for an empty reader. Without a schema, raise a clear `ValueError`. Nonempty schema inference and batch casting are unchanged. Tests cover synchronous/asynchronous creation, empty/nonempty iterators and tuples, and schema-aware `add()` sanitization. The default schema-less `add()` rejection is deliberately unchanged. ## Validation - Built the current Rust/PyO3 extension from base `9f37f392` with the repository's `ci` profile; both Python-only arms use the same verified extension and locked Python dependency versions. - Complete `python/tests/test_table.py` + `python/tests/test_scannable.py`: **268 passed, 0 skipped**. The unchanged production code with identical tests has **24 failures / 244 passes**. - Both arms used four-CPU affinity. An initial unrestricted-affinity run exhausted thread creation under the private memory limit; that failed attempt is not counted as a pass. - Package-wide Ruff check/format pass. Repository-wide checks still report pre-existing findings in unchanged `docs/web-tests`; those files are not modified here. AI-assisted change, reproduced against unchanged production code and validated with a native source build. This does not claim the full repository/platform matrix or a performance improvement. Signed-off-by: Xucheng Zhou --- python/python/lancedb/table.py | 15 +++++-- python/python/tests/test_table.py | 75 +++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 124fa34b5..c7de4161d 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -469,7 +469,7 @@ def _into_pyarrow_reader( ): return data.collect().to_arrow().to_reader() elif isinstance(data, Iterable): - return _iterator_to_reader(data) + return _iterator_to_reader(data, schema) else: raise TypeError( f"Unknown data type {type(data)}. " @@ -479,10 +479,19 @@ def _into_pyarrow_reader( ) -def _iterator_to_reader(data: Iterable) -> pa.RecordBatchReader: +def _iterator_to_reader( + data: Iterable, schema: Optional[pa.Schema] = None +) -> pa.RecordBatchReader: # Each batch is treated as it's own reader, mainly so we can # re-use the _into_pyarrow_reader logic. - first = _into_pyarrow_reader(next(data)) + data = iter(data) + try: + first_batch = next(data) + except StopIteration: + if schema is None: + raise ValueError("Cannot create table from empty iterable without a schema") + return pa.RecordBatchReader.from_batches(schema, []) + first = _into_pyarrow_reader(first_batch) schema = first.schema def gen(): diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 238ee5330..e23b478f5 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -4847,6 +4847,81 @@ def test_stats(mem_db: DBConnection): assert stats_with_index["fragment_stats"]["num_fragments"] == 1 +@pytest.mark.parametrize("iterable_type", [iter, tuple]) +@pytest.mark.parametrize("empty", [False, True]) +@pytest.mark.parametrize("with_schema", [False, True]) +def test_create_table_recordbatch_iterable( + mem_db: DBConnection, iterable_type, empty, with_schema +): + schema = pa.schema([pa.field("id", pa.int64())]) + batches = [] if empty else [pa.RecordBatch.from_pydict({"id": [1]}, schema)] + data = iterable_type(batches) + if empty and not with_schema: + with pytest.raises(ValueError, match="empty iterable without a schema"): + mem_db.create_table("test", data=data) + return + + table = mem_db.create_table( + "test", data=data, schema=schema if with_schema else None + ) + assert table.schema == schema + assert table.to_arrow().to_pylist() == ([] if empty else [{"id": 1}]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("iterable_type", [iter, tuple]) +@pytest.mark.parametrize("empty", [False, True]) +@pytest.mark.parametrize("with_schema", [False, True]) +async def test_create_table_recordbatch_iterable_async( + mem_db_async: AsyncConnection, iterable_type, empty, with_schema +): + schema = pa.schema([pa.field("id", pa.int64())]) + batches = [] if empty else [pa.RecordBatch.from_pydict({"id": [1]}, schema)] + data = iterable_type(batches) + if empty and not with_schema: + with pytest.raises(ValueError, match="empty iterable without a schema"): + await mem_db_async.create_table("test", data=data) + return + + table = await mem_db_async.create_table( + "test", data=data, schema=schema if with_schema else None + ) + assert await table.schema() == schema + assert (await table.to_arrow()).to_pylist() == ([] if empty else [{"id": 1}]) + + +@pytest.mark.parametrize("iterable_type", [iter, tuple]) +@pytest.mark.parametrize("on_bad_vectors", ["error", "drop", "fill", "null"]) +def test_add_empty_recordbatch_iterable( + mem_db: DBConnection, iterable_type, on_bad_vectors +): + table = mem_db.create_table("test", [{"id": 1}]) + if on_bad_vectors == "error": + # The default scannable path still requires an input schema. This + # fix does not change its validation policy for schema-less input. + with pytest.raises(ValueError, match="empty iterator"): + table.add(iterable_type([]), on_bad_vectors=on_bad_vectors) + else: + # Sanitization has the existing table schema available. + table.add(iterable_type([]), on_bad_vectors=on_bad_vectors) + assert table.to_arrow().to_pylist() == [{"id": 1}] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("iterable_type", [iter, tuple]) +@pytest.mark.parametrize("on_bad_vectors", ["error", "drop", "fill", "null"]) +async def test_add_empty_recordbatch_iterable_async( + mem_db_async: AsyncConnection, iterable_type, on_bad_vectors +): + table = await mem_db_async.create_table("test", [{"id": 1}]) + if on_bad_vectors == "error": + with pytest.raises(ValueError, match="empty iterator"): + await table.add(iterable_type([]), on_bad_vectors=on_bad_vectors) + else: + await table.add(iterable_type([]), on_bad_vectors=on_bad_vectors) + assert (await table.to_arrow()).to_pylist() == [{"id": 1}] + + def test_create_table_empty_list_with_schema(mem_db: DBConnection): """Test creating table with empty list data and schema