mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-18 12:08:35 +00:00
test(python): cover float16 table creation from Arrow data (#3785)
## Summary - exercise float16 sanitization through the reported direct Arrow-data table creation path - assert that the inferred fixed-size vector schema remains float16 - retain end-to-end index creation and vector search coverage ## Root cause and fix PyArrow 16 does not provide an is_nan kernel for half-float arrays, so passing float16 vector values directly to that kernel raises ArrowNotImplementedError. LanceDB's sanitizer already carries the compatibility fix from #837: it casts float16 values to float32 only for NaN detection while preserving the stored vector type. The existing end-to-end regression created an empty schema-defined table and added data afterward. This change aligns that regression with the issue reproduction by creating a table directly from a FixedSizeList<float16> Arrow table and verifying the persisted schema. ## Validation - uv run --extra tests pytest python/tests/test_table.py::test_create_f16_table_from_arrow_data -q - direct 1,000-row by 128-dimension float16 Arrow-table reproduction - PyArrow 16.1 half-float is_nan kernel reproduction - uvx ruff@0.15.20 format --check python/python/tests/test_table.py - uvx ruff@0.15.20 check . Fixes #835 <!-- lance-gatekeeper-fix:v1 agent=dd0a32a959f691f49de958d4333fb29d generation=1 --> --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
This commit is contained in:
committed by
GitHub
parent
dbc3687c7b
commit
dd5cb4d805
@@ -2759,15 +2759,40 @@ def test_create_with_embedding_function(mem_db: DBConnection):
|
||||
assert actual == expected
|
||||
|
||||
|
||||
def test_create_f16_table_from_arrow_data(mem_db: DBConnection):
|
||||
dimension = 32
|
||||
num_rows = 512
|
||||
values = pa.array(
|
||||
np.random.default_rng(42)
|
||||
.standard_normal(num_rows * dimension)
|
||||
.astype(np.float16)
|
||||
)
|
||||
df = pa.table(
|
||||
{
|
||||
"text": [f"s-{i}" for i in range(num_rows)],
|
||||
"vector": pa.FixedSizeListArray.from_arrays(values, dimension),
|
||||
}
|
||||
)
|
||||
table = mem_db.create_table("f16_tbl", data=df)
|
||||
assert table.schema.field("vector").type == pa.list_(pa.float16(), dimension)
|
||||
table.create_index(num_partitions=2, num_sub_vectors=2)
|
||||
|
||||
query = df["vector"][2].as_py()
|
||||
expected = table.search(query).limit(2).to_arrow()
|
||||
|
||||
assert "s-2" in expected["text"].to_pylist()
|
||||
|
||||
|
||||
def test_create_f16_table(mem_db: DBConnection):
|
||||
class MyTable(LanceModel):
|
||||
text: str
|
||||
vector: Vector(32, value_type=pa.float16())
|
||||
|
||||
rng = np.random.default_rng(42)
|
||||
df = pa.table(
|
||||
{
|
||||
"text": [f"s-{i}" for i in range(512)],
|
||||
"vector": [np.random.randn(32).astype(np.float16) for _ in range(512)],
|
||||
"vector": [rng.standard_normal(32).astype(np.float16) for _ in range(512)],
|
||||
}
|
||||
)
|
||||
table = mem_db.create_table(
|
||||
|
||||
Reference in New Issue
Block a user