mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-13 14:16:54 +00:00
fix(python): reject embedding source updates without vectors
Fixes #4761 `Table.update` does not compute embeddings, so changing an embedding source column could leave the previous embedding in place. Source-column updates now raise `ValueError` unless every associated vector column is supplied in the same call. The guard reads Python embedding metadata and native `lancedb::column_definitions` without loading embedding models. It uses Lance's field-path parser for quoted targets and covers synchronous and asynchronous local and remote tables. Supplying every associated vector column, vector-only updates, and unrelated updates remain supported. This is a breaking change for source-only updates. The API documentation and Update rows guide explain the requirement and the `merge_insert` alternative for regenerating embeddings. Validation at the reviewed head covered 44 targeted public update cases and the full Python suite (3107 passed, 162 skipped, 70 deselected). All applicable exact-head checks and workflows completed successfully.
This commit is contained in:
lancedb-gatefixer[bot]
@@ -2834,6 +2834,260 @@ def test_update(mem_db: DBConnection):
|
||||
assert np.allclose(v, np.array([[1.2, 1.9], [1.1, 1.1]]))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("async_api", [False, True])
|
||||
@pytest.mark.parametrize("use_sql", [False, True])
|
||||
@pytest.mark.parametrize("where", [None, "id = 1", col("id") == 1])
|
||||
@pytest.mark.parametrize("source_target", ["text", "`text`"])
|
||||
async def test_update_embedding_source_requires_vector(
|
||||
tmp_path, async_api, use_sql, where, source_target
|
||||
):
|
||||
func = MockTextEmbeddingFunction.create()
|
||||
|
||||
class Doc(LanceModel):
|
||||
id: int
|
||||
text: str = func.SourceField()
|
||||
vector: Vector(func.ndims()) = func.VectorField()
|
||||
|
||||
data = [{"id": 1, "text": "hello world"}, {"id": 2, "text": "goodbye world"}]
|
||||
if async_api:
|
||||
db = await lancedb.connect_async(tmp_path)
|
||||
table = await db.create_table("docs", data, schema=Doc)
|
||||
before = await table.to_arrow()
|
||||
version = await table.version()
|
||||
else:
|
||||
db = lancedb.connect(tmp_path)
|
||||
table = db.create_table("docs", data, schema=Doc)
|
||||
before = table.to_arrow()
|
||||
version = table.version
|
||||
|
||||
# Rejection must happen before any rows or the table version change.
|
||||
updates = {source_target: "text || ' updated'" if use_sql else "updated text"}
|
||||
with pytest.raises(
|
||||
ValueError, match="source column 'text'.*vector column 'vector'"
|
||||
):
|
||||
if async_api:
|
||||
if use_sql:
|
||||
await table.update(where=where, updates_sql=updates)
|
||||
else:
|
||||
await table.update(updates, where=where)
|
||||
elif use_sql:
|
||||
table.update(where=where, values_sql=updates)
|
||||
else:
|
||||
table.update(where=where, values=updates)
|
||||
|
||||
if async_api:
|
||||
assert (await table.to_arrow()).equals(before)
|
||||
assert await table.version() == version
|
||||
else:
|
||||
assert table.to_arrow().equals(before)
|
||||
assert table.version == version
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("async_api", [False, True])
|
||||
@pytest.mark.parametrize("use_sql", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"source_target,vector_target,other_vector_target",
|
||||
[
|
||||
("text", "vector", "other_vector"),
|
||||
("`text`", "vector", "`other_vector`"),
|
||||
("text", "`vector`", "other_vector"),
|
||||
("`text`", "`vector`", "`other_vector`"),
|
||||
],
|
||||
)
|
||||
async def test_update_embedding_source_with_multiple_vectors(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
async_api,
|
||||
use_sql,
|
||||
source_target,
|
||||
vector_target,
|
||||
other_vector_target,
|
||||
):
|
||||
func = MockTextEmbeddingFunction.create()
|
||||
|
||||
class Doc(LanceModel):
|
||||
id: int
|
||||
text: str
|
||||
vector: Vector(func.ndims())
|
||||
other_vector: Vector(func.ndims())
|
||||
|
||||
configs = [
|
||||
EmbeddingFunctionConfig(
|
||||
source_column="text", vector_column=column, function=func
|
||||
)
|
||||
for column in ["vector", "other_vector"]
|
||||
]
|
||||
data = [{"id": 1, "text": "hello world"}, {"id": 2, "text": "goodbye world"}]
|
||||
if async_api:
|
||||
db = await lancedb.connect_async(tmp_path)
|
||||
table = await db.create_table(
|
||||
"docs", data, schema=Doc, embedding_functions=configs
|
||||
)
|
||||
else:
|
||||
db = lancedb.connect(tmp_path)
|
||||
table = db.create_table("docs", data, schema=Doc, embedding_functions=configs)
|
||||
|
||||
async def update(values):
|
||||
if async_api:
|
||||
if use_sql:
|
||||
return await table.update(updates_sql=values, where=col("id") == 1)
|
||||
return await table.update(values, where=col("id") == 1)
|
||||
if use_sql:
|
||||
return table.update(values_sql=values, where=col("id") == 1)
|
||||
return table.update(values=values, where=col("id") == 1)
|
||||
|
||||
vector = func.compute_source_embeddings(["updated text"])[0]
|
||||
updates = {source_target: "updated text", vector_target: vector}
|
||||
if use_sql:
|
||||
updates = {
|
||||
key: table_module.value_to_sql(value) for key, value in updates.items()
|
||||
}
|
||||
|
||||
# Checking column dependencies must not require loading an embedding model.
|
||||
monkeypatch.delitem(EmbeddingFunctionRegistry.get_instance()._functions, "test")
|
||||
with pytest.raises(ValueError, match="vector column 'other_vector'"):
|
||||
await update(updates)
|
||||
|
||||
updates[other_vector_target] = updates[vector_target]
|
||||
result = await update(updates)
|
||||
assert result.rows_updated == 1
|
||||
actual = await table.to_arrow() if async_api else table.to_arrow()
|
||||
rows = actual.sort_by("id").to_pylist()
|
||||
assert rows[0]["text"] == "updated text"
|
||||
assert np.allclose(rows[0]["vector"], vector)
|
||||
assert np.allclose(rows[0]["other_vector"], vector)
|
||||
assert rows[1]["text"] == "goodbye world"
|
||||
|
||||
# Vector-only and unrelated updates are still allowed on embedding tables.
|
||||
assert (await update({vector_target: updates[vector_target]})).rows_updated == 1
|
||||
assert (await update({"id": "3" if use_sql else 3})).rows_updated == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("async_api", [False, True])
|
||||
@pytest.mark.parametrize("source_target", ["text", "`text`"])
|
||||
async def test_update_embedding_source_requires_native_vector_metadata(
|
||||
tmp_path, async_api, source_target
|
||||
):
|
||||
vector_type = pa.list_(pa.float32(), 2)
|
||||
column_definitions = [
|
||||
{"kind": "Physical"},
|
||||
{"kind": "Physical"},
|
||||
{
|
||||
"kind": {
|
||||
"Embedding": {
|
||||
"source_column": "text",
|
||||
"dest_column": "vector",
|
||||
"embedding_name": "native-explicit",
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": {
|
||||
"Embedding": {
|
||||
"source_column": "text",
|
||||
"dest_column": None,
|
||||
"embedding_name": "native-default",
|
||||
}
|
||||
}
|
||||
},
|
||||
]
|
||||
schema = pa.schema(
|
||||
[
|
||||
pa.field("id", pa.int64()),
|
||||
pa.field("text", pa.string()),
|
||||
pa.field("vector", vector_type),
|
||||
pa.field("text_embedding", vector_type),
|
||||
],
|
||||
metadata={
|
||||
b"lancedb::column_definitions": json.dumps(column_definitions).encode()
|
||||
},
|
||||
)
|
||||
data = pa.Table.from_pylist(
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"text": "hello world",
|
||||
"vector": [1.0, 0.0],
|
||||
"text_embedding": [0.0, 1.0],
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"text": "goodbye world",
|
||||
"vector": [0.5, 0.5],
|
||||
"text_embedding": [0.25, 0.75],
|
||||
},
|
||||
],
|
||||
schema=schema,
|
||||
)
|
||||
|
||||
if async_api:
|
||||
db = await lancedb.connect_async(tmp_path)
|
||||
table = await db.create_table("docs", data)
|
||||
table_schema = await table.schema()
|
||||
before = await table.to_arrow()
|
||||
version = await table.version()
|
||||
else:
|
||||
db = lancedb.connect(tmp_path)
|
||||
table = db.create_table("docs", data)
|
||||
table_schema = table.schema
|
||||
before = table.to_arrow()
|
||||
version = table.version
|
||||
|
||||
metadata = table_schema.metadata or {}
|
||||
assert b"lancedb::column_definitions" in metadata
|
||||
assert b"embedding_functions" not in metadata
|
||||
|
||||
async def update(values):
|
||||
if async_api:
|
||||
return await table.update(values, where="id = 1")
|
||||
return table.update(values=values, where="id = 1")
|
||||
|
||||
with pytest.raises(
|
||||
ValueError, match="source column 'text'.*vector column 'vector'"
|
||||
):
|
||||
await update({source_target: "updated text"})
|
||||
|
||||
if async_api:
|
||||
assert (await table.to_arrow()).equals(before)
|
||||
assert await table.version() == version
|
||||
else:
|
||||
assert table.to_arrow().equals(before)
|
||||
assert table.version == version
|
||||
|
||||
with pytest.raises(
|
||||
ValueError, match="source column 'text'.*vector column 'text_embedding'"
|
||||
):
|
||||
await update({source_target: "updated text", "vector": [2.0, 3.0]})
|
||||
|
||||
if async_api:
|
||||
assert (await table.to_arrow()).equals(before)
|
||||
assert await table.version() == version
|
||||
else:
|
||||
assert table.to_arrow().equals(before)
|
||||
assert table.version == version
|
||||
|
||||
result = await update(
|
||||
{
|
||||
source_target: "updated text",
|
||||
"vector": [2.0, 3.0],
|
||||
"text_embedding": [4.0, 5.0],
|
||||
}
|
||||
)
|
||||
assert result.rows_updated == 1
|
||||
assert (await update({"vector": [6.0, 7.0]})).rows_updated == 1
|
||||
assert (await update({"id": 99})).rows_updated == 1
|
||||
|
||||
actual = await table.to_arrow() if async_api else table.to_arrow()
|
||||
rows = {row["id"]: row for row in actual.to_pylist()}
|
||||
assert rows[99]["text"] == "updated text"
|
||||
assert np.allclose(rows[99]["vector"], [6.0, 7.0])
|
||||
assert np.allclose(rows[99]["text_embedding"], [4.0, 5.0])
|
||||
assert rows[2]["text"] == "goodbye world"
|
||||
|
||||
|
||||
def test_update_expr_filter_literals(mem_db: DBConnection):
|
||||
values = ["5", "4.66e-84", "it's"]
|
||||
table = mem_db.create_table(
|
||||
|
||||
Reference in New Issue
Block a user