fix(node): cover non-nullable embedding schema append (#3835)

## Summary

- Add an issue-specific regression for appending generated embeddings to
an empty table with a non-nullable vector field.
- Verify the custom embedding function produces the declared Float64
vectors and both appended rows are readable.

## Root cause

In v0.4.19, records without a vector value were materialized against the
explicit schema before embeddings were inserted. Apache Arrow inferred
the generated batch vector field as nullable while the table retained
the user-provided non-nullable field, then rejected the mismatched
schemas.

The current conversion path excludes the generated field from the
initial record conversion and realigns the completed batch to the stored
schema after embedding, but the reported empty-table append sequence
lacked permanent regression coverage.

## Validation

- `pnpm exec biome format --write __test__/embedding.test.ts`
- `pnpm lint-ci`
- `pnpm test -- --runInBand __test__/embedding.test.ts` (12 passed, 1
skipped integration test)
- `pnpm build`
- `pnpm run docs`

Fixes #1281

<!-- lance-gatekeeper-fix:v1 agent=6b7270aeb92e6b6c6f5b45022fa83f6a
generation=1 -->

---------

Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
This commit is contained in:
lancedb-gatefixer[bot]
2026-08-07 17:32:39 +08:00
committed by GitHub
parent 607e556927
commit 2ba7407dc3
+60
View File
@@ -11,8 +11,11 @@ import {
Float16,
Float32,
Float64,
Int32,
Schema,
Utf8,
fromDataToBuffer,
tableFromIPC,
} from "../lancedb/arrow";
import { EmbeddingFunction, LanceSchema } from "../lancedb/embedding";
import { getRegistry, register } from "../lancedb/embedding/registry";
@@ -184,6 +187,63 @@ describe("embedding functions", () => {
const vector0 = JSON.parse(JSON.stringify(arr[0].vector));
expect(vector0).toEqual([1, 2, 3]);
});
it("should append generated vectors to a non-nullable schema", async () => {
@register("non_nullable_schema_test")
class MockEmbeddingFunction extends EmbeddingFunction<string> {
ndims() {
return 3;
}
embeddingDataType(): Float {
return new Float64();
}
async computeSourceEmbeddings(data: string[]) {
return data.map(() => [1, 2, 3]);
}
}
const schema = new Schema([
new Field("id", new Int32()),
new Field("text", new Utf8()),
new Field("type", new Utf8()),
new Field(
"vector",
new FixedSizeList(3, new Field("item", new Float64())),
),
]);
const func = new MockEmbeddingFunction();
const db = await connect(tmpDir.name);
const table = await db.createEmptyTable("test_non_nullable", schema, {
embeddingFunction: {
function: func,
sourceColumn: "text",
},
});
const data = [
{ id: 1, text: "Carrot", type: "vegetable" },
{ id: 2, text: "Apple", type: "fruit" },
];
const buffer = await fromDataToBuffer(
data,
undefined,
await table.schema(),
);
const generatedTable = tableFromIPC(buffer);
const vectorField = generatedTable.schema.fields.find(
(field) => field.name === "vector",
);
expect(vectorField?.nullable).toBe(false);
await table.add(data);
const rows = await table.query().toArray();
expect(rows).toHaveLength(2);
for (const row of rows) {
expect([...row.vector]).toEqual([1, 2, 3]);
}
});
it("should error when appending to a table with an unregistered embedding function", async () => {
@register("mock")
class MockEmbeddingFunction extends EmbeddingFunction<string> {