mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-04 12:38:38 +00:00
fix(node): read Python embedding metadata (#3836)
## Summary - normalize Python snake_case and TypeScript camelCase embedding metadata - use the normalized metadata for schema validation and embedding lookup - cover appending through `Table.add()` with a Python-authored schema fixture ## Root cause Python writes embedding source and vector column names as `source_column` and `vector_column`, but the TypeScript SDK only read `sourceColumn` and `vectorColumn`. The missing source name reached the add path as `undefined`, preventing JavaScript rows from being embedded and appended. ## Validation - `pnpm lint` - `pnpm test __test__/embedding.test.ts __test__/arrow.test.ts __test__/registry.test.ts --runInBand` (201 passed, 1 skipped) - `pnpm build` - `pnpm run docs` Fixes #1289 <!-- lance-gatekeeper-fix:v1 agent=b71c18a5e33d26f4d138972e91d34e66 generation=1 --> --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Co-authored-by: Xuanwo <github@xuanwo.io>
This commit is contained in:
committed by
GitHub
parent
ead4d27bfc
commit
2deccf21cf
@@ -187,6 +187,58 @@ describe("embedding functions", () => {
|
||||
const vector0 = JSON.parse(JSON.stringify(arr[0].vector));
|
||||
expect(vector0).toEqual([1, 2, 3]);
|
||||
});
|
||||
it("should append multiple Python embeddings with the same alias", async () => {
|
||||
@register("python-mock")
|
||||
// biome-ignore lint/correctness/noUnusedVariables: the decorator registers this class
|
||||
class MockEmbeddingFunction extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 3;
|
||||
}
|
||||
embeddingDataType(): Float {
|
||||
return new Float32();
|
||||
}
|
||||
async computeQueryEmbeddings(_data: string) {
|
||||
return [1, 2, 3];
|
||||
}
|
||||
async computeSourceEmbeddings(data: string[]) {
|
||||
return data.map((value) =>
|
||||
value === "hello world" ? [1, 2, 3] : [4, 5, 6],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const metadata = new Map([
|
||||
[
|
||||
"embedding_functions",
|
||||
'[{"source_column":"text1","vector_column":"vector1","name":"python-mock","model":{}},{"source_column":"text2","vector_column":"vector2","name":"python-mock","model":{}}]',
|
||||
],
|
||||
]);
|
||||
const schema = new Schema(
|
||||
[
|
||||
new Field("text1", new Utf8(), true),
|
||||
new Field("text2", new Utf8(), true),
|
||||
new Field(
|
||||
"vector1",
|
||||
new FixedSizeList(3, new Field("item", new Float32(), true)),
|
||||
true,
|
||||
),
|
||||
new Field(
|
||||
"vector2",
|
||||
new FixedSizeList(3, new Field("item", new Float32(), true)),
|
||||
true,
|
||||
),
|
||||
],
|
||||
metadata,
|
||||
);
|
||||
|
||||
const db = await connect(tmpDir.name);
|
||||
const table = await db.createEmptyTable("test", schema);
|
||||
await table.add([{ text1: "hello world", text2: "goodbye world" }]);
|
||||
|
||||
const rows = await table.query().toArray();
|
||||
expect(JSON.parse(JSON.stringify(rows[0].vector1))).toEqual([1, 2, 3]);
|
||||
expect(JSON.parse(JSON.stringify(rows[0].vector2))).toEqual([4, 5, 6]);
|
||||
});
|
||||
|
||||
it("should append generated vectors to a non-nullable schema", async () => {
|
||||
@register("non_nullable_schema_test")
|
||||
|
||||
Reference in New Issue
Block a user