From 0c3a1c92c862d1f4d251d55eacafabe225277722 Mon Sep 17 00:00:00 2001
From: Madan Kumar <winklemad@outlook.com>
Date: Thu, 22 Oct 2026 22:40:45 +0530
Subject: [PATCH] fix(python): don't crash inferring schema for an all-null
 vector column (#4034)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fixes #4033

`create_table(data=...)` without an explicit schema raised `IndexError:
index out of bounds` when a `vector`/`embedding`-named list column was
all-null. `_modal_list_size` infers the dimension with
`pc.mode(pc.list_value_length(arr))[0]`, and pyarrow's `mode` skips
nulls, so an all-null column produces a zero-length result and the `[0]`
subscript blows up.

The sibling `_infer_vector_dim` already guards the identical
`pc.mode(...)` with a `len == 0` check and returns `None`;
`_modal_list_size` just didn't. This makes `_modal_list_size` return
`Optional[int]` and return `None` when every value is null, and the call
site then leaves the column as a variable-length list instead of
inventing a fixed size — the dimension is genuinely unknowable when
there are no non-null rows.

Added a regression test that creates a table from an all-null
`list<float32>` vector column and asserts it succeeds, the column stays
a `list<float32>`, and the rows are queryable. It fails on `main` with
the `IndexError` and passes with this change.
---
 python/python/lancedb/table.py    | 17 ++++++++++++++---
 python/python/tests/test_table.py | 24 ++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py
index 006ef8cb9..ff866d4db 100644
--- a/python/python/lancedb/table.py
+++ b/python/python/lancedb/table.py
@@ -5335,6 +5335,12 @@ def _infer_target_schema(
             # Use the most common length of the list as the dimensions
             dim = _modal_list_size(peeked.column(i))
 
+            if dim is None:
+                # Every list entry is null (e.g. embeddings not computed yet),
+                # so the dimension is unknowable. Leave the column as a
+                # variable-length list instead of guessing a fixed size.
+                continue
+
             # Determine target type based on value type
             if pa.types.is_floating(field.type.value_type):
                 target_type = pa.list_(pa.float32(), dim)
@@ -5374,9 +5380,14 @@ def _infer_target_schema(
     return schema, reader
 
 
-def _modal_list_size(arr: Union[pa.ListArray, pa.ChunkedArray]) -> int:
-    # Use the most common length of the list as the dimensions
-    return pc.mode(pc.list_value_length(arr))[0].as_py()["mode"]
+def _modal_list_size(arr: Union[pa.ListArray, pa.ChunkedArray]) -> Optional[int]:
+    # Use the most common length of the list as the dimensions. pyarrow's
+    # `mode` skips nulls, so an all-null column yields an empty result; return
+    # None in that case so the caller can keep a variable-length list.
+    modes = pc.mode(pc.list_value_length(arr))
+    if len(modes) == 0:
+        return None
+    return modes[0].as_py()["mode"]
 
 
 def _infer_vector_dim(arr: Union[pa.Array, pa.ChunkedArray]) -> Optional[int]:
diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py
index 7a4f5981c..827f2ef58 100644
--- a/python/python/tests/test_table.py
+++ b/python/python/tests/test_table.py
@@ -280,6 +280,30 @@ def test_create_table_infers_large_int_vectors(mem_db: DBConnection):
     assert vector_column.to_pylist() == [[0.0, 300.0]]
 
 
+def test_create_table_all_null_vector_column(mem_db: DBConnection):
+    # A vector column can legitimately arrive with every value null, e.g. when
+    # the embeddings have not been computed yet. The dimension is unknowable in
+    # that case, so schema inference must not crash and should leave the column
+    # as a variable-length list rather than a fixed-size one.
+    data = pa.table(
+        {
+            "id": pa.array([1, 2]),
+            "vector": pa.array([None, None], type=pa.list_(pa.float32())),
+        }
+    )
+
+    table = mem_db.create_table("all_null_vector", data=data)
+
+    vector_field = table.schema.field("vector")
+    assert pa.types.is_list(vector_field.type)
+    assert vector_field.type.value_type == pa.float32()
+
+    result = table.to_arrow()
+    assert result.num_rows == 2
+    assert result.column("vector").to_pylist() == [None, None]
+    assert table.count_rows() == 2
+
+
 @pytest.mark.asyncio
 async def test_create_table_async_infers_large_int_vectors(
     mem_db_async: AsyncConnection,