mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-19 02:02:27 +00:00
chore: merge main into blob concurrency repair
Preserve descriptor-based blob sizing and version pinning together with shared request permits and finite cursor ranges. Adapt probe-only regression tests to the upstream probe helper.
This commit is contained in:
+1415
-1456
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,39 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
// Every Apache Arrow release the package accepts as a peer dependency, as
|
||||
// separately installed copies so the tests can feed objects built by one
|
||||
// Arrow version into a LanceDB running on another. When a new Arrow major
|
||||
// is released, add the alias to `package.json` and extend this list; the
|
||||
// test files pick the change up from here.
|
||||
import * as arrow15 from "apache-arrow-15";
|
||||
import * as arrow16 from "apache-arrow-16";
|
||||
import * as arrow17 from "apache-arrow-17";
|
||||
import * as arrow18 from "apache-arrow-18";
|
||||
import * as arrow19 from "apache-arrow-19";
|
||||
import * as arrow20 from "apache-arrow-20";
|
||||
import * as arrow21 from "apache-arrow-21";
|
||||
|
||||
export { arrow15, arrow16, arrow17, arrow18, arrow19, arrow20, arrow21 };
|
||||
|
||||
export type ApacheArrow =
|
||||
| typeof arrow15
|
||||
| typeof arrow16
|
||||
| typeof arrow17
|
||||
| typeof arrow18
|
||||
| typeof arrow19
|
||||
| typeof arrow20
|
||||
| typeof arrow21;
|
||||
|
||||
export const arrowVersions: ApacheArrow[] = [
|
||||
arrow15,
|
||||
arrow16,
|
||||
arrow17,
|
||||
arrow18,
|
||||
arrow19,
|
||||
arrow20,
|
||||
arrow21,
|
||||
];
|
||||
|
||||
/** The newest supported Arrow release, for tests that only need one. */
|
||||
export const latestArrow = arrow21;
|
||||
@@ -3,13 +3,10 @@
|
||||
|
||||
import * as apiArrow from "apache-arrow";
|
||||
|
||||
import * as arrow15 from "apache-arrow-15";
|
||||
import * as arrow16 from "apache-arrow-16";
|
||||
import * as arrow17 from "apache-arrow-17";
|
||||
import * as arrow18 from "apache-arrow-18";
|
||||
|
||||
import * as tmp from "tmp";
|
||||
|
||||
import { arrowVersions } from "./arrow_versions";
|
||||
|
||||
import { connect } from "../lancedb";
|
||||
import {
|
||||
EmbeddingFunction,
|
||||
@@ -18,7 +15,7 @@ import {
|
||||
} from "../lancedb/embedding";
|
||||
import { getRegistry, register } from "../lancedb/embedding/registry";
|
||||
|
||||
describe.each([arrow15, arrow16, arrow17, arrow18])("LanceSchema", (arrow) => {
|
||||
describe.each(arrowVersions)("LanceSchema", (arrow) => {
|
||||
test("should preserve input order", async () => {
|
||||
const schema = LanceSchema({
|
||||
id: new arrow.Int32(),
|
||||
@@ -29,7 +26,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])("LanceSchema", (arrow) => {
|
||||
});
|
||||
});
|
||||
|
||||
describe.each([arrow15, arrow16, arrow17, arrow18])("Registry", (arrow) => {
|
||||
describe.each(arrowVersions)("Registry", (arrow) => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
beforeEach(() => {
|
||||
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
@@ -240,7 +237,7 @@ describe("Registry.setVar", () => {
|
||||
return data.map(() => [1, 2, 3]);
|
||||
}
|
||||
embeddingDataType() {
|
||||
return new arrow18.Float32() as apiArrow.Float;
|
||||
return new apiArrow.Float32();
|
||||
}
|
||||
protected getSensitiveKeys() {
|
||||
return ["secretKey"];
|
||||
|
||||
@@ -2,7 +2,13 @@
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import * as arrow from "../lancedb/arrow";
|
||||
import { sanitizeField, sanitizeMap, sanitizeType } from "../lancedb/sanitize";
|
||||
import {
|
||||
sanitizeField,
|
||||
sanitizeMap,
|
||||
sanitizeSchema,
|
||||
sanitizeType,
|
||||
} from "../lancedb/sanitize";
|
||||
import { arrow21 } from "./arrow_versions";
|
||||
|
||||
describe("sanitize", function () {
|
||||
describe("sanitizeType function", function () {
|
||||
@@ -59,6 +65,8 @@ describe("sanitize", function () {
|
||||
["durationmicrosecond", new arrow.DurationMicrosecond()],
|
||||
["durationmillisecond", new arrow.DurationMillisecond()],
|
||||
["durationsecond", new arrow.DurationSecond()],
|
||||
["largebinary", new arrow.LargeBinary()],
|
||||
["largeutf8", new arrow.LargeUtf8()],
|
||||
] as const;
|
||||
|
||||
it.each(allTypeNameTestCases)(
|
||||
@@ -89,6 +97,110 @@ describe("sanitize", function () {
|
||||
"Unrecognized type name in schema: invalid_type",
|
||||
);
|
||||
});
|
||||
|
||||
// These types only exist in Arrow 20 (IntervalMonthDayNano) and 21 (the
|
||||
// rest), so the objects come from a second copy of the newest release. The
|
||||
// library itself may be running on an older Arrow (see the CI matrix), in
|
||||
// which case sanitizing them has to fail with a clear message instead.
|
||||
const hostArrow = arrow as unknown as Record<string, unknown>;
|
||||
const hostSupportsNewerTypes = typeof hostArrow.Utf8View === "function";
|
||||
const whenSupported = hostSupportsNewerTypes ? describe : describe.skip;
|
||||
const whenUnsupported = hostSupportsNewerTypes ? describe.skip : describe;
|
||||
|
||||
const newerTypeTestCases = [
|
||||
["Utf8View", () => new arrow21.Utf8View()],
|
||||
["BinaryView", () => new arrow21.BinaryView()],
|
||||
["IntervalMonthDayNano", () => new arrow21.IntervalMonthDayNano()],
|
||||
[
|
||||
"LargeList",
|
||||
() =>
|
||||
new arrow21.LargeList(new arrow21.Field("item", new arrow21.Utf8())),
|
||||
],
|
||||
] as const;
|
||||
|
||||
whenSupported("types added in Arrow 20 and 21", function () {
|
||||
it.each(newerTypeTestCases)(
|
||||
"should sanitize a %s type from another copy of Arrow",
|
||||
function (name, makeForeign) {
|
||||
const foreign = makeForeign();
|
||||
const result = sanitizeType(foreign);
|
||||
// Interval subclasses share `Type.Interval`, so the sanitized type
|
||||
// is the generic `Interval` carrying the same unit, as for the
|
||||
// other units.
|
||||
const expected = name === "IntervalMonthDayNano" ? "Interval" : name;
|
||||
expect(result).toBeInstanceOf(hostArrow[expected] as () => unknown);
|
||||
expect(result.typeId).toBe(foreign.typeId);
|
||||
expect(result.toString()).toBe(foreign.toString());
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
["binaryview", "BinaryView"],
|
||||
["utf8view", "Utf8View"],
|
||||
["intervalmonthdaynano", "Interval"],
|
||||
])('should map type name "%s" to %s', function (name, expected) {
|
||||
expect(sanitizeType(name)).toBeInstanceOf(
|
||||
hostArrow[expected] as () => unknown,
|
||||
);
|
||||
});
|
||||
|
||||
it("should sanitize the child field of a LargeList", function () {
|
||||
const foreign = new arrow21.LargeList(
|
||||
new arrow21.Field("item", new arrow21.Int32(), false),
|
||||
);
|
||||
const result = sanitizeType(foreign) as arrow.List;
|
||||
expect(result.children[0]).toBeInstanceOf(arrow.Field);
|
||||
expect(result.children[0].name).toBe("item");
|
||||
expect(result.children[0].nullable).toBe(false);
|
||||
const child = result.children[0].type;
|
||||
expect(arrow.DataType.isInt(child)).toBe(true);
|
||||
expect((child as arrow.Int).bitWidth).toBe(32);
|
||||
expect((child as arrow.Int).isSigned).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
whenUnsupported("types newer than the installed Arrow", function () {
|
||||
it.each(
|
||||
newerTypeTestCases.filter(([name]) => name !== "IntervalMonthDayNano"),
|
||||
)(
|
||||
"should explain that %s needs a newer apache-arrow",
|
||||
function (name, makeForeign) {
|
||||
expect(() => sanitizeType(makeForeign())).toThrow(
|
||||
`The Arrow type ${name} requires apache-arrow`,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("should sanitize IntervalMonthDayNano to a generic Interval", () => {
|
||||
// The subclass shares `Type.Interval`, which every supported Arrow
|
||||
// release has; only the unit is new.
|
||||
const result = sanitizeType(new arrow21.IntervalMonthDayNano());
|
||||
expect(result).toBeInstanceOf(arrow.Interval);
|
||||
expect((result as arrow.Interval).unit).toBe(
|
||||
arrow21.IntervalUnit.MONTH_DAY_NANO,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it("should rebuild a schema from a second copy of the same Arrow version", function () {
|
||||
// Arrow 21 makes `instanceof Schema` succeed across copies of the library
|
||||
// but not `instanceof <DataType>`; the schema must still be rebuilt.
|
||||
const foreign = new arrow21.Schema([
|
||||
new arrow21.Field("x", new arrow21.Float32(), true),
|
||||
]);
|
||||
// Only Arrow 21+ overrides `Symbol.hasInstance` this way.
|
||||
const crossCopyInstanceof = Object.getOwnPropertySymbols(
|
||||
arrow.Schema,
|
||||
).includes(Symbol.hasInstance);
|
||||
expect(foreign instanceof arrow.Schema).toBe(crossCopyInstanceof);
|
||||
const result = sanitizeSchema(foreign);
|
||||
expect(result).not.toBe(foreign);
|
||||
const type = result.fields[0].type;
|
||||
expect(type).toBeInstanceOf(arrow.DataType);
|
||||
expect(arrow.DataType.isFloat(type)).toBe(true);
|
||||
expect((type as arrow.Float).precision).toBe(arrow.Precision.SINGLE);
|
||||
expect(() => arrow.makeBuilder({ type })).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("sanitizeField function", function () {
|
||||
|
||||
+313
-12
@@ -2,15 +2,11 @@
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import * as fs from "fs";
|
||||
import { getEventListeners } from "node:events";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import * as path from "path";
|
||||
import * as tmp from "tmp";
|
||||
|
||||
import * as arrow15 from "apache-arrow-15";
|
||||
import * as arrow16 from "apache-arrow-16";
|
||||
import * as arrow17 from "apache-arrow-17";
|
||||
import * as arrow18 from "apache-arrow-18";
|
||||
|
||||
import {
|
||||
AutoQuery,
|
||||
Connection,
|
||||
@@ -55,19 +51,21 @@ import {
|
||||
instanceOfFullTextQuery,
|
||||
} from "../lancedb/query";
|
||||
import { LocalTable } from "../lancedb/table";
|
||||
import {
|
||||
type ApacheArrow,
|
||||
arrow18,
|
||||
arrowVersions,
|
||||
latestArrow,
|
||||
} from "./arrow_versions";
|
||||
|
||||
describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
describe.each(arrowVersions)(
|
||||
"Given a table",
|
||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||
(arrow: any) => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
let table: Table;
|
||||
|
||||
const schema:
|
||||
| import("apache-arrow-15").Schema
|
||||
| import("apache-arrow-16").Schema
|
||||
| import("apache-arrow-17").Schema
|
||||
| import("apache-arrow-18").Schema = new arrow.Schema([
|
||||
const schema: InstanceType<ApacheArrow["Schema"]> = new arrow.Schema([
|
||||
new arrow.Field("id", new arrow.Float64(), true),
|
||||
]);
|
||||
|
||||
@@ -92,6 +90,38 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
await expect(table.countRows()).resolves.toBe(3);
|
||||
});
|
||||
|
||||
it("creates and adds Arrow tables with large string and binary columns", async () => {
|
||||
const data = new arrow.Table({
|
||||
text: arrow.vectorFromArray(["alpha", "beta"], new arrow.LargeUtf8()),
|
||||
bytes: arrow.vectorFromArray(
|
||||
[Buffer.from("one"), Buffer.from("two")],
|
||||
new arrow.LargeBinary(),
|
||||
),
|
||||
});
|
||||
const conn = await connect(tmpDir.name);
|
||||
const largeTable = await conn.createTable("large_columns", data);
|
||||
await largeTable.add(data);
|
||||
|
||||
const rows = await largeTable.query().toArray();
|
||||
expect(rows.map((row) => row.text)).toEqual([
|
||||
"alpha",
|
||||
"beta",
|
||||
"alpha",
|
||||
"beta",
|
||||
]);
|
||||
expect(rows.map((row) => Buffer.from(row.bytes).toString())).toEqual([
|
||||
"one",
|
||||
"two",
|
||||
"one",
|
||||
"two",
|
||||
]);
|
||||
const fields = (await largeTable.schema()).fields;
|
||||
expect(fields.map((field) => field.type.typeId)).toEqual([
|
||||
arrow.Type.LargeUtf8,
|
||||
arrow.Type.LargeBinary,
|
||||
]);
|
||||
});
|
||||
|
||||
it("should support a foreign Float64 vector schema end to end", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const schema = new arrow.Schema([
|
||||
@@ -709,6 +739,114 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
);
|
||||
|
||||
// https://github.com/lancedb/lancedb/issues/1963
|
||||
// These types were added in Arrow 21, so they are built with a second copy of
|
||||
// that release rather than the per-version matrix above, and reading them back
|
||||
// needs the library's own Arrow to know them too.
|
||||
const hostArrowSupportsViewTypes =
|
||||
typeof (arrow as unknown as Record<string, unknown>).Utf8View === "function";
|
||||
(hostArrowSupportsViewTypes ? it : it.skip)(
|
||||
"creates and adds tables with LargeList, Utf8View and BinaryView columns",
|
||||
async () => {
|
||||
const tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
const db = await connect(tmpDir.name);
|
||||
const makeData = () =>
|
||||
new latestArrow.Table({
|
||||
id: latestArrow.vectorFromArray([1, 2], new latestArrow.Int32()),
|
||||
tags: latestArrow.vectorFromArray(
|
||||
[["a", "b"], ["c"]],
|
||||
new latestArrow.LargeList(
|
||||
new latestArrow.Field("item", new latestArrow.Utf8(), true),
|
||||
),
|
||||
),
|
||||
text: latestArrow.vectorFromArray(
|
||||
["short", "a string that is longer than the twelve inline bytes"],
|
||||
new latestArrow.Utf8View(),
|
||||
),
|
||||
bytes: latestArrow.vectorFromArray(
|
||||
[new Uint8Array([1, 2, 3]), new Uint8Array(20).fill(7)],
|
||||
new latestArrow.BinaryView(),
|
||||
),
|
||||
});
|
||||
const table = await db.createTable("views", makeData());
|
||||
await table.add(makeData());
|
||||
|
||||
const schema = await table.schema();
|
||||
// Take the enum from the Arrow 21 copy: the host Arrow may predate
|
||||
// `Type.LargeList`, and this file still has to type-check there.
|
||||
expect(schema.fields.map((f) => f.type.typeId)).toEqual([
|
||||
latestArrow.Type.Int,
|
||||
latestArrow.Type.LargeList,
|
||||
// Lance stores the view types as their offset-based equivalents.
|
||||
latestArrow.Type.Utf8,
|
||||
latestArrow.Type.Binary,
|
||||
]);
|
||||
const rows = await table.query().toArrow();
|
||||
expect(rows.numRows).toBe(4);
|
||||
const row = rows.get(1)!.toJSON();
|
||||
expect(row.id).toBe(2);
|
||||
expect(row.tags.toJSON()).toEqual(["c"]);
|
||||
expect(row.text).toBe(
|
||||
"a string that is longer than the twelve inline bytes",
|
||||
);
|
||||
expect(Array.from(row.bytes)).toEqual(Array(20).fill(7));
|
||||
|
||||
// Arrow JS 21 serializes a *sliced* view column with a truncated views
|
||||
// buffer; the library rebuilds view columns as Utf8/Binary so this works.
|
||||
const sliced = makeData().slice(1, 2);
|
||||
expect(sliced.getChild("text")!.data[0].offset).toBe(1);
|
||||
const slicedTable = await db.createTable("views_sliced", sliced);
|
||||
await slicedTable.add(sliced);
|
||||
const slicedRows = await slicedTable.query().toArrow();
|
||||
expect(slicedRows.numRows).toBe(2);
|
||||
for (const i of [0, 1]) {
|
||||
const r = slicedRows.get(i)!.toJSON();
|
||||
expect(r.id).toBe(2);
|
||||
expect(r.text).toBe(
|
||||
"a string that is longer than the twelve inline bytes",
|
||||
);
|
||||
expect(Array.from(r.bytes)).toEqual(Array(20).fill(7));
|
||||
}
|
||||
|
||||
// Regression: a field with an integer-like name. JavaScript enumerates
|
||||
// such keys first, so a key-ordered rebuild would pair "1" with the text
|
||||
// column's values. The batches are built explicitly to pin field order.
|
||||
const numericSchema = new latestArrow.Schema([
|
||||
new latestArrow.Field("text", new latestArrow.Utf8View(), true),
|
||||
new latestArrow.Field("1", new latestArrow.Utf8View(), true),
|
||||
new latestArrow.Field("n", new latestArrow.Int32(), true),
|
||||
]);
|
||||
const column = (
|
||||
values: unknown[],
|
||||
type: InstanceType<typeof latestArrow.DataType>,
|
||||
) => latestArrow.vectorFromArray(values, type).data[0];
|
||||
const numericBatch = new latestArrow.RecordBatch(
|
||||
numericSchema,
|
||||
latestArrow.makeData({
|
||||
type: new latestArrow.Struct(numericSchema.fields),
|
||||
length: 3,
|
||||
nullCount: 0,
|
||||
children: [
|
||||
column(["t0", "t1", "t2"], new latestArrow.Utf8View()),
|
||||
column(["one0", "one1", "one2"], new latestArrow.Utf8View()),
|
||||
column([0, 1, 2], new latestArrow.Int32()),
|
||||
],
|
||||
}),
|
||||
);
|
||||
const numericData = new latestArrow.Table(numericSchema, [numericBatch]);
|
||||
const numericTable = await db.createTable("views_numeric", numericData);
|
||||
await numericTable.add(numericData.slice(1));
|
||||
const numericRows = (await numericTable.query().toArrow()).toArray();
|
||||
expect(numericRows.map((r) => r.toJSON())).toEqual([
|
||||
{ text: "t0", "1": "one0", n: 0 },
|
||||
{ text: "t1", "1": "one1", n: 1 },
|
||||
{ text: "t2", "1": "one2", n: 2 },
|
||||
{ text: "t1", "1": "one1", n: 1 },
|
||||
{ text: "t2", "1": "one2", n: 2 },
|
||||
]);
|
||||
tmpDir.removeCallback();
|
||||
},
|
||||
);
|
||||
|
||||
it("should query documents with LangChain PDF metadata", async () => {
|
||||
const tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
try {
|
||||
@@ -2601,6 +2739,27 @@ describe("when dealing with blob columns", () => {
|
||||
expect(Buffer.from(await files[1]!.read()).toString()).toBe("beta");
|
||||
});
|
||||
|
||||
it("adds a LargeBinary Arrow column to a blob table", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const schema = new Schema([new Field("id", new Int64()), blob("image")]);
|
||||
const table = await db.createTable(
|
||||
"blobs",
|
||||
[{ id: 1n, image: Buffer.from("alpha") }],
|
||||
{ schema },
|
||||
);
|
||||
const payload = Buffer.from("beta");
|
||||
const data = new arrow18.Table({
|
||||
id: arrow18.vectorFromArray([2n], new arrow18.Int64()),
|
||||
image: arrow18.vectorFromArray([payload], new arrow18.LargeBinary()),
|
||||
});
|
||||
await table.add(data);
|
||||
|
||||
const rows = await table.query().withRowId().toArray();
|
||||
const rowId = rows.find((row) => row.id === 2n)!._rowid as bigint;
|
||||
const [actual] = await table.fetchBlobs("image", [rowId]);
|
||||
expect(actual).toEqual(payload);
|
||||
});
|
||||
|
||||
it("reads a half-open range", async () => {
|
||||
const { table, rowIds } = await openBlobTable();
|
||||
const files = await table.fetchBlobFiles("image", rowIds);
|
||||
@@ -2625,6 +2784,148 @@ describe("when dealing with blob columns", () => {
|
||||
).rejects.toThrow(/exceeds blob size/);
|
||||
});
|
||||
|
||||
it("reads at most maxBytes and advances the cursor", async () => {
|
||||
const { table, rowIds } = await openBlobTable();
|
||||
const [handle] = await table.fetchBlobFiles("image", rowIds);
|
||||
expect((await handle!.read(2n)).toString()).toBe("al");
|
||||
expect(await handle!.tell()).toBe(2n);
|
||||
expect((await handle!.read(2n)).toString()).toBe("ph");
|
||||
expect((await handle!.read(10n)).toString()).toBe("a");
|
||||
expect(await handle!.tell()).toBe(5n);
|
||||
expect(await handle!.read(1n)).toEqual(Buffer.alloc(0));
|
||||
});
|
||||
|
||||
it("seeks the cursor", async () => {
|
||||
const { table, rowIds, alpha } = await openBlobTable();
|
||||
const [handle] = await table.fetchBlobFiles("image", rowIds);
|
||||
await handle!.seek(3n);
|
||||
expect(await handle!.tell()).toBe(3n);
|
||||
expect((await handle!.read()).toString()).toBe("ha");
|
||||
await handle!.seek(0n);
|
||||
expect(await handle!.read()).toEqual(alpha);
|
||||
});
|
||||
|
||||
it("rejects a negative seek position", async () => {
|
||||
const { table, rowIds } = await openBlobTable();
|
||||
const [handle] = await table.fetchBlobFiles("image", rowIds);
|
||||
await expect(handle!.seek(-1n)).rejects.toThrow(/cannot be negative/);
|
||||
});
|
||||
|
||||
it("readRanges returns one buffer per range in order", async () => {
|
||||
const { table, rowIds } = await openBlobTable();
|
||||
const [handle] = await table.fetchBlobFiles("image", rowIds);
|
||||
const buffers = await handle!.readRanges([
|
||||
{ start: 3n, end: 5n },
|
||||
{ start: 0n, end: 2n },
|
||||
{ start: 3n, end: 5n },
|
||||
]);
|
||||
expect(buffers.map((b) => b.toString())).toEqual(["ha", "al", "ha"]);
|
||||
expect(await handle!.tell()).toBe(0n);
|
||||
});
|
||||
|
||||
it("fails when a readRanges end is past the blob size", async () => {
|
||||
const { table, rowIds, alpha } = await openBlobTable();
|
||||
const [handle] = await table.fetchBlobFiles("image", rowIds);
|
||||
await expect(
|
||||
handle!.readRanges([
|
||||
{ start: 0n, end: 1n },
|
||||
{ start: 0n, end: BigInt(alpha.length + 1) },
|
||||
]),
|
||||
).rejects.toThrow(/exceeds blob size/);
|
||||
});
|
||||
|
||||
it("fails reads after close and closes once", async () => {
|
||||
const { table, rowIds } = await openBlobTable();
|
||||
const [handle] = await table.fetchBlobFiles("image", rowIds);
|
||||
expect(handle!.isClosed()).toBe(false);
|
||||
await handle!.close();
|
||||
expect(handle!.isClosed()).toBe(true);
|
||||
await expect(handle!.close()).resolves.toBeUndefined();
|
||||
await expect(handle!.read()).rejects.toThrow(/already closed/);
|
||||
await expect(handle!.read(1n)).rejects.toThrow(/already closed/);
|
||||
await expect(handle!.readRange(0n, 1n)).rejects.toThrow(/already closed/);
|
||||
await expect(handle!.readRanges([{ start: 0n, end: 1n }])).rejects.toThrow(
|
||||
/already closed/,
|
||||
);
|
||||
await expect(handle!.seek(0n)).rejects.toThrow(/already closed/);
|
||||
await expect(handle!.tell()).rejects.toThrow(/already closed/);
|
||||
expect(handle!.size()).toBe(5n);
|
||||
});
|
||||
|
||||
it("rejects reads with an already aborted signal", async () => {
|
||||
const { table, rowIds, alpha } = await openBlobTable();
|
||||
const [handle] = await table.fetchBlobFiles("image", rowIds);
|
||||
const signal = AbortSignal.abort();
|
||||
const abortError = { name: "AbortError" };
|
||||
await expect(handle!.read(undefined, { signal })).rejects.toMatchObject(
|
||||
abortError,
|
||||
);
|
||||
await expect(handle!.read(2n, { signal })).rejects.toMatchObject(
|
||||
abortError,
|
||||
);
|
||||
await expect(handle!.readRange(0n, 1n, { signal })).rejects.toMatchObject(
|
||||
abortError,
|
||||
);
|
||||
await expect(
|
||||
handle!.readRanges([{ start: 0n, end: 1n }], { signal }),
|
||||
).rejects.toMatchObject(abortError);
|
||||
await expect(
|
||||
table.fetchBlobs("image", rowIds, { signal }),
|
||||
).rejects.toMatchObject(abortError);
|
||||
await expect(
|
||||
table.fetchBlobFiles("image", rowIds, { signal }),
|
||||
).rejects.toMatchObject(abortError);
|
||||
expect(await handle!.read()).toEqual(alpha);
|
||||
});
|
||||
|
||||
it("rejects with the abort reason", async () => {
|
||||
const { table, rowIds } = await openBlobTable();
|
||||
const [handle] = await table.fetchBlobFiles("image", rowIds);
|
||||
const reason = new Error("caller gave up");
|
||||
await expect(
|
||||
handle!.readRange(0n, 1n, { signal: AbortSignal.abort(reason) }),
|
||||
).rejects.toBe(reason);
|
||||
});
|
||||
|
||||
it("reads with a signal and removes its abort listener", async () => {
|
||||
const { table, rowIds, alpha, beta } = await openBlobTable();
|
||||
const controller = new AbortController();
|
||||
const { signal } = controller;
|
||||
const [handle] = await table.fetchBlobFiles("image", rowIds, { signal });
|
||||
expect((await handle!.readRange(0n, 2n, { signal })).toString()).toBe("al");
|
||||
expect(await table.fetchBlobs("image", rowIds, { signal })).toEqual([
|
||||
alpha,
|
||||
beta,
|
||||
null,
|
||||
]);
|
||||
expect(getEventListeners(signal, "abort")).toHaveLength(0);
|
||||
controller.abort();
|
||||
expect(await handle!.read()).toEqual(alpha);
|
||||
});
|
||||
|
||||
it("cancels an in-flight read", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int64(), true),
|
||||
blob("payload", { inlineSizeThreshold: 0, dedicatedSizeThreshold: 1 }),
|
||||
]);
|
||||
const payload = Buffer.alloc(32 * 1024 * 1024, 7);
|
||||
const table = await db.createTable("large_blobs", [{ id: 1n, payload }], {
|
||||
schema,
|
||||
});
|
||||
const [row] = await table.query().withRowId().toArray();
|
||||
const [handle] = await table.fetchBlobFiles("payload", [
|
||||
row._rowid as bigint,
|
||||
]);
|
||||
const controller = new AbortController();
|
||||
const pending = handle!.readRange(0n, handle!.size(), {
|
||||
signal: controller.signal,
|
||||
});
|
||||
controller.abort();
|
||||
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
|
||||
expect(await handle!.readRange(0n, 4n)).toEqual(payload.subarray(0, 4));
|
||||
});
|
||||
|
||||
it("rejects fetchBlobs on a non-blob column", async () => {
|
||||
const { table, rowIds } = await openBlobTable();
|
||||
await expect(table.fetchBlobs("id", rowIds)).rejects.toThrow(/blob/i);
|
||||
@@ -3007,7 +3308,7 @@ it("passes cleanupOlderThan to the native binding as an absolute timestamp", asy
|
||||
expect(optimize).toHaveBeenCalledWith(cutoff.getTime(), true);
|
||||
});
|
||||
|
||||
describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
describe.each(arrowVersions)(
|
||||
"when optimizing a dataset",
|
||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||
(arrow: any) => {
|
||||
|
||||
Reference in New Issue
Block a user