fix: support LargeList label list indexes (#3529)

## Summary

This PR extends nested-field regression coverage across Rust
local/remote, Python sync/async, and Node so canonical escaped paths
stay consistent across scalar, vector, and FTS index lifecycle behavior.

It also aligns LanceDB's LabelList type gate with Lance by accepting
`LargeList<primitive>` columns while keeping `List<Struct<...>>`
unsupported until Lance defines stable membership semantics for struct
labels.

Part of #3406.
This commit is contained in:
Xuanwo
2026-06-10 23:53:56 +08:00
committed by GitHub
parent 9c12fb6437
commit 566b67a634
6 changed files with 638 additions and 20 deletions
+96 -3
View File
@@ -113,8 +113,14 @@ async def test_create_nested_scalar_index_lists_canonical_paths(db_async):
pa.field("user.id", pa.int32()),
]
)
mixed_case_metadata_type = pa.struct([pa.field("userId", pa.int32())])
escaped_metadata_type = pa.struct([pa.field("user-id", pa.int32())])
literal_type = pa.struct([pa.field("a.b", pa.int32())])
data = pa.Table.from_arrays(
[
pa.array([1, 2, 3], type=pa.int32()),
pa.array([1, 2, 3], type=pa.int32()),
pa.array([1, 2, 3], type=pa.int32()),
pa.array([1, 2, 3], type=pa.int32()),
pa.array(
[
@@ -124,25 +130,67 @@ async def test_create_nested_scalar_index_lists_canonical_paths(db_async):
],
type=metadata_type,
),
pa.array(
[{"userId": 10}, {"userId": 20}, {"userId": 30}],
type=mixed_case_metadata_type,
),
pa.array(
[{"user-id": 10}, {"user-id": 20}, {"user-id": 30}],
type=escaped_metadata_type,
),
pa.array(
[{"a.b": 10}, {"a.b": 20}, {"a.b": 30}],
type=literal_type,
),
],
names=[
"rowId",
"row-id",
"userId",
"user_id",
"metadata",
"MetaData",
"meta-data",
"literal",
],
names=["user_id", "metadata"],
)
table = await db_async.create_table("nested_scalar_index", data)
await table.create_index("user_id", config=BTree(), name="top_user_id_idx")
await table.create_index("rowId", config=BTree(), name="row_id_idx")
await table.create_index("`row-id`", config=BTree(), name="row_dash_id_idx")
await table.create_index("userId", config=BTree(), name="top_user_id_idx")
await table.create_index("user_id", config=BTree(), name="top_snake_user_id_idx")
await table.create_index(
"metadata.user_id", config=BTree(), name="nested_user_id_idx"
)
await table.create_index(
"metadata.`user.id`", config=BTree(), name="escaped_user_id_idx"
)
await table.create_index(
"MetaData.userId", config=BTree(), name="mixed_case_metadata_user_id_idx"
)
await table.create_index(
"`meta-data`.`user-id`", config=BTree(), name="escaped_names_idx"
)
await table.create_index("literal.`a.b`", config=BTree(), name="literal_dot_idx")
columns_by_name = {
index.name: index.columns for index in await table.list_indices()
}
assert columns_by_name["top_user_id_idx"] == ["user_id"]
assert columns_by_name["row_id_idx"] == ["rowId"]
assert columns_by_name["row_dash_id_idx"] == ["`row-id`"]
assert columns_by_name["top_user_id_idx"] == ["userId"]
assert columns_by_name["top_snake_user_id_idx"] == ["user_id"]
assert columns_by_name["nested_user_id_idx"] == ["metadata.user_id"]
assert columns_by_name["escaped_user_id_idx"] == ["metadata.`user.id`"]
assert columns_by_name["mixed_case_metadata_user_id_idx"] == ["MetaData.userId"]
assert columns_by_name["escaped_names_idx"] == ["`meta-data`.`user-id`"]
assert columns_by_name["literal_dot_idx"] == ["literal.`a.b`"]
for index_name in columns_by_name:
stats = await table.index_stats(index_name)
assert stats is not None
assert stats.num_indexed_rows == 3
@pytest.mark.asyncio
@@ -189,6 +237,51 @@ async def test_create_label_list_index(some_table: AsyncTable):
await some_table.create_index("tags", config=LabelList())
indices = await some_table.list_indices()
assert str(indices) == '[Index(LabelList, columns=["tags"], name="tags_idx")]'
plan = await some_table.query().where("array_has(tags, 'tag0')").explain_plan()
assert "ScalarIndexQuery" in plan
@pytest.mark.asyncio
async def test_create_large_list_label_list_index(db_async):
data = pa.Table.from_pydict(
{"tags": [[f"tag{i % 2}", "shared"] for i in range(16)]},
schema=pa.schema([pa.field("tags", pa.large_list(pa.string()))]),
)
table = await db_async.create_table("large_list_label_list_index", data)
await table.create_index("tags", config=LabelList())
indices = await table.list_indices()
assert str(indices) == '[Index(LabelList, columns=["tags"], name="tags_idx")]'
plan = await table.query().where("array_has(tags, 'shared')").explain_plan()
assert "ScalarIndexQuery" in plan
@pytest.mark.asyncio
async def test_create_label_list_index_rejects_list_struct(db_async):
item_type = pa.struct(
[
pa.field("tag", pa.string()),
pa.field(
"metadata",
pa.struct([pa.field("userId", pa.string())]),
),
]
)
data = pa.Table.from_pylist(
[
{
"items": [
{"tag": "tag0", "metadata": {"userId": "user0"}},
{"tag": "shared", "metadata": {"userId": "user1"}},
]
}
],
schema=pa.schema([pa.field("items", pa.list_(item_type))]),
)
table = await db_async.create_table("list_struct_label_list_index", data)
with pytest.raises(Exception, match="LabelList index cannot be created"):
await table.create_index("items", config=LabelList())
@pytest.mark.asyncio