mirror of
https://github.com/lancedb/lancedb.git
synced 2026-01-07 12:22:59 +00:00
feat: reconfigure typescript linter / formatter for nodejs (#1042)
The eslint rules specify some formatting requirements that are rather strict and conflict with vscode's default formatter. I was unable to get auto-formatting to setup correctly. Also, eslint has quite recently [given up on formatting](https://eslint.org/blog/2023/10/deprecating-formatting-rules/) and recommends using a 3rd party formatter. This PR adds prettier as the formatter. It restores the eslint rules to their defaults. This does mean we now have the "no explicit any" check back on. I know that rule is pedantic but it did help me catch a few corner cases in type testing that weren't covered in the current code. Leaving in draft as this is dependent on other PRs.
This commit is contained in:
Weston Pace
GitHub
@@ -12,7 +12,12 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
import { convertToTable, fromTableToBuffer, makeArrowTable, makeEmptyTable } from '../dist/arrow'
|
||||
import {
|
||||
convertToTable,
|
||||
fromTableToBuffer,
|
||||
makeArrowTable,
|
||||
makeEmptyTable,
|
||||
} from "../dist/arrow";
|
||||
import {
|
||||
Field,
|
||||
FixedSizeList,
|
||||
@@ -32,291 +37,368 @@ import {
|
||||
Dictionary,
|
||||
Int64,
|
||||
Float,
|
||||
Precision
|
||||
} from 'apache-arrow'
|
||||
import { type EmbeddingFunction } from '../dist/embedding/embedding_function'
|
||||
Precision,
|
||||
} from "apache-arrow";
|
||||
import { type EmbeddingFunction } from "../dist/embedding/embedding_function";
|
||||
|
||||
function sampleRecords (): Array<Record<string, any>> {
|
||||
function sampleRecords(): Array<Record<string, any>> {
|
||||
return [
|
||||
{
|
||||
binary: Buffer.alloc(5),
|
||||
boolean: false,
|
||||
number: 7,
|
||||
string: 'hello',
|
||||
string: "hello",
|
||||
struct: { x: 0, y: 0 },
|
||||
list: ['anime', 'action', 'comedy']
|
||||
}
|
||||
]
|
||||
list: ["anime", "action", "comedy"],
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
// Helper method to verify various ways to create a table
|
||||
async function checkTableCreation (tableCreationMethod: (records: any, recordsReversed: any, schema: Schema) => Promise<Table>, infersTypes: boolean): Promise<void> {
|
||||
const records = sampleRecords()
|
||||
const recordsReversed = [{
|
||||
list: ['anime', 'action', 'comedy'],
|
||||
struct: { x: 0, y: 0 },
|
||||
string: 'hello',
|
||||
number: 7,
|
||||
boolean: false,
|
||||
binary: Buffer.alloc(5)
|
||||
}]
|
||||
async function checkTableCreation(
|
||||
tableCreationMethod: (
|
||||
records: any,
|
||||
recordsReversed: any,
|
||||
schema: Schema,
|
||||
) => Promise<Table>,
|
||||
infersTypes: boolean,
|
||||
): Promise<void> {
|
||||
const records = sampleRecords();
|
||||
const recordsReversed = [
|
||||
{
|
||||
list: ["anime", "action", "comedy"],
|
||||
struct: { x: 0, y: 0 },
|
||||
string: "hello",
|
||||
number: 7,
|
||||
boolean: false,
|
||||
binary: Buffer.alloc(5),
|
||||
},
|
||||
];
|
||||
const schema = new Schema([
|
||||
new Field('binary', new Binary(), false),
|
||||
new Field('boolean', new Bool(), false),
|
||||
new Field('number', new Float64(), false),
|
||||
new Field('string', new Utf8(), false),
|
||||
new Field('struct', new Struct([
|
||||
new Field('x', new Float64(), false),
|
||||
new Field('y', new Float64(), false)
|
||||
])),
|
||||
new Field('list', new List(new Field('item', new Utf8(), false)), false)
|
||||
])
|
||||
new Field("binary", new Binary(), false),
|
||||
new Field("boolean", new Bool(), false),
|
||||
new Field("number", new Float64(), false),
|
||||
new Field("string", new Utf8(), false),
|
||||
new Field(
|
||||
"struct",
|
||||
new Struct([
|
||||
new Field("x", new Float64(), false),
|
||||
new Field("y", new Float64(), false),
|
||||
]),
|
||||
),
|
||||
new Field("list", new List(new Field("item", new Utf8(), false)), false),
|
||||
]);
|
||||
|
||||
const table = await tableCreationMethod(records, recordsReversed, schema)
|
||||
const table = await tableCreationMethod(records, recordsReversed, schema);
|
||||
schema.fields.forEach((field, idx) => {
|
||||
const actualField = table.schema.fields[idx]
|
||||
const actualField = table.schema.fields[idx];
|
||||
// Type inference always assumes nullable=true
|
||||
if (infersTypes) {
|
||||
expect(actualField.nullable).toBe(true)
|
||||
expect(actualField.nullable).toBe(true);
|
||||
} else {
|
||||
expect(actualField.nullable).toBe(false)
|
||||
expect(actualField.nullable).toBe(false);
|
||||
}
|
||||
expect(table.getChild(field.name)?.type.toString()).toEqual(field.type.toString())
|
||||
expect(table.getChildAt(idx)?.type.toString()).toEqual(field.type.toString())
|
||||
})
|
||||
expect(table.getChild(field.name)?.type.toString()).toEqual(
|
||||
field.type.toString(),
|
||||
);
|
||||
expect(table.getChildAt(idx)?.type.toString()).toEqual(
|
||||
field.type.toString(),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
describe('The function makeArrowTable', function () {
|
||||
it('will use data types from a provided schema instead of inference', async function () {
|
||||
describe("The function makeArrowTable", function () {
|
||||
it("will use data types from a provided schema instead of inference", async function () {
|
||||
const schema = new Schema([
|
||||
new Field('a', new Int32()),
|
||||
new Field('b', new Float32()),
|
||||
new Field('c', new FixedSizeList(3, new Field('item', new Float16()))),
|
||||
new Field('d', new Int64())
|
||||
])
|
||||
new Field("a", new Int32()),
|
||||
new Field("b", new Float32()),
|
||||
new Field("c", new FixedSizeList(3, new Field("item", new Float16()))),
|
||||
new Field("d", new Int64()),
|
||||
]);
|
||||
const table = makeArrowTable(
|
||||
[
|
||||
{ a: 1, b: 2, c: [1, 2, 3], d: 9 },
|
||||
{ a: 4, b: 5, c: [4, 5, 6], d: 10 },
|
||||
{ a: 7, b: 8, c: [7, 8, 9], d: null }
|
||||
{ a: 7, b: 8, c: [7, 8, 9], d: null },
|
||||
],
|
||||
{ schema }
|
||||
)
|
||||
{ schema },
|
||||
);
|
||||
|
||||
const buf = await fromTableToBuffer(table)
|
||||
expect(buf.byteLength).toBeGreaterThan(0)
|
||||
const buf = await fromTableToBuffer(table);
|
||||
expect(buf.byteLength).toBeGreaterThan(0);
|
||||
|
||||
const actual = tableFromIPC(buf)
|
||||
expect(actual.numRows).toBe(3)
|
||||
const actualSchema = actual.schema
|
||||
expect(actualSchema).toEqual(schema)
|
||||
})
|
||||
const actual = tableFromIPC(buf);
|
||||
expect(actual.numRows).toBe(3);
|
||||
const actualSchema = actual.schema;
|
||||
expect(actualSchema).toEqual(schema);
|
||||
});
|
||||
|
||||
it('will assume the column `vector` is FixedSizeList<Float32> by default', async function () {
|
||||
it("will assume the column `vector` is FixedSizeList<Float32> by default", async function () {
|
||||
const schema = new Schema([
|
||||
new Field('a', new Float(Precision.DOUBLE), true),
|
||||
new Field('b', new Float(Precision.DOUBLE), true),
|
||||
new Field("a", new Float(Precision.DOUBLE), true),
|
||||
new Field("b", new Float(Precision.DOUBLE), true),
|
||||
new Field(
|
||||
'vector',
|
||||
new FixedSizeList(3, new Field('item', new Float(Precision.SINGLE), true)),
|
||||
true
|
||||
)
|
||||
])
|
||||
"vector",
|
||||
new FixedSizeList(
|
||||
3,
|
||||
new Field("item", new Float(Precision.SINGLE), true),
|
||||
),
|
||||
true,
|
||||
),
|
||||
]);
|
||||
const table = makeArrowTable([
|
||||
{ a: 1, b: 2, vector: [1, 2, 3] },
|
||||
{ a: 4, b: 5, vector: [4, 5, 6] },
|
||||
{ a: 7, b: 8, vector: [7, 8, 9] }
|
||||
])
|
||||
{ a: 7, b: 8, vector: [7, 8, 9] },
|
||||
]);
|
||||
|
||||
const buf = await fromTableToBuffer(table)
|
||||
expect(buf.byteLength).toBeGreaterThan(0)
|
||||
const buf = await fromTableToBuffer(table);
|
||||
expect(buf.byteLength).toBeGreaterThan(0);
|
||||
|
||||
const actual = tableFromIPC(buf)
|
||||
expect(actual.numRows).toBe(3)
|
||||
const actualSchema = actual.schema
|
||||
expect(actualSchema).toEqual(schema)
|
||||
})
|
||||
const actual = tableFromIPC(buf);
|
||||
expect(actual.numRows).toBe(3);
|
||||
const actualSchema = actual.schema;
|
||||
expect(actualSchema).toEqual(schema);
|
||||
});
|
||||
|
||||
it('can support multiple vector columns', async function () {
|
||||
it("can support multiple vector columns", async function () {
|
||||
const schema = new Schema([
|
||||
new Field('a', new Float(Precision.DOUBLE), true),
|
||||
new Field('b', new Float(Precision.DOUBLE), true),
|
||||
new Field('vec1', new FixedSizeList(3, new Field('item', new Float16(), true)), true),
|
||||
new Field('vec2', new FixedSizeList(3, new Field('item', new Float16(), true)), true)
|
||||
])
|
||||
new Field("a", new Float(Precision.DOUBLE), true),
|
||||
new Field("b", new Float(Precision.DOUBLE), true),
|
||||
new Field(
|
||||
"vec1",
|
||||
new FixedSizeList(3, new Field("item", new Float16(), true)),
|
||||
true,
|
||||
),
|
||||
new Field(
|
||||
"vec2",
|
||||
new FixedSizeList(3, new Field("item", new Float16(), true)),
|
||||
true,
|
||||
),
|
||||
]);
|
||||
const table = makeArrowTable(
|
||||
[
|
||||
{ a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
|
||||
{ a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
|
||||
{ a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] }
|
||||
{ a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] },
|
||||
],
|
||||
{
|
||||
vectorColumns: {
|
||||
vec1: { type: new Float16() },
|
||||
vec2: { type: new Float16() }
|
||||
}
|
||||
}
|
||||
)
|
||||
vec2: { type: new Float16() },
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const buf = await fromTableToBuffer(table)
|
||||
expect(buf.byteLength).toBeGreaterThan(0)
|
||||
const buf = await fromTableToBuffer(table);
|
||||
expect(buf.byteLength).toBeGreaterThan(0);
|
||||
|
||||
const actual = tableFromIPC(buf)
|
||||
expect(actual.numRows).toBe(3)
|
||||
const actualSchema = actual.schema
|
||||
expect(actualSchema).toEqual(schema)
|
||||
})
|
||||
const actual = tableFromIPC(buf);
|
||||
expect(actual.numRows).toBe(3);
|
||||
const actualSchema = actual.schema;
|
||||
expect(actualSchema).toEqual(schema);
|
||||
});
|
||||
|
||||
it('will allow different vector column types', async function () {
|
||||
const table = makeArrowTable(
|
||||
[
|
||||
{ fp16: [1], fp32: [1], fp64: [1] }
|
||||
],
|
||||
{
|
||||
vectorColumns: {
|
||||
fp16: { type: new Float16() },
|
||||
fp32: { type: new Float32() },
|
||||
fp64: { type: new Float64() }
|
||||
}
|
||||
}
|
||||
)
|
||||
it("will allow different vector column types", async function () {
|
||||
const table = makeArrowTable([{ fp16: [1], fp32: [1], fp64: [1] }], {
|
||||
vectorColumns: {
|
||||
fp16: { type: new Float16() },
|
||||
fp32: { type: new Float32() },
|
||||
fp64: { type: new Float64() },
|
||||
},
|
||||
});
|
||||
|
||||
expect(table.getChild('fp16')?.type.children[0].type.toString()).toEqual(new Float16().toString())
|
||||
expect(table.getChild('fp32')?.type.children[0].type.toString()).toEqual(new Float32().toString())
|
||||
expect(table.getChild('fp64')?.type.children[0].type.toString()).toEqual(new Float64().toString())
|
||||
})
|
||||
expect(table.getChild("fp16")?.type.children[0].type.toString()).toEqual(
|
||||
new Float16().toString(),
|
||||
);
|
||||
expect(table.getChild("fp32")?.type.children[0].type.toString()).toEqual(
|
||||
new Float32().toString(),
|
||||
);
|
||||
expect(table.getChild("fp64")?.type.children[0].type.toString()).toEqual(
|
||||
new Float64().toString(),
|
||||
);
|
||||
});
|
||||
|
||||
it('will use dictionary encoded strings if asked', async function () {
|
||||
const table = makeArrowTable([{ str: 'hello' }])
|
||||
expect(DataType.isUtf8(table.getChild('str')?.type)).toBe(true)
|
||||
it("will use dictionary encoded strings if asked", async function () {
|
||||
const table = makeArrowTable([{ str: "hello" }]);
|
||||
expect(DataType.isUtf8(table.getChild("str")?.type)).toBe(true);
|
||||
|
||||
const tableWithDict = makeArrowTable([{ str: 'hello' }], { dictionaryEncodeStrings: true })
|
||||
expect(DataType.isDictionary(tableWithDict.getChild('str')?.type)).toBe(true)
|
||||
const tableWithDict = makeArrowTable([{ str: "hello" }], {
|
||||
dictionaryEncodeStrings: true,
|
||||
});
|
||||
expect(DataType.isDictionary(tableWithDict.getChild("str")?.type)).toBe(
|
||||
true,
|
||||
);
|
||||
|
||||
const schema = new Schema([
|
||||
new Field('str', new Dictionary(new Utf8(), new Int32()))
|
||||
])
|
||||
new Field("str", new Dictionary(new Utf8(), new Int32())),
|
||||
]);
|
||||
|
||||
const tableWithDict2 = makeArrowTable([{ str: 'hello' }], { schema })
|
||||
expect(DataType.isDictionary(tableWithDict2.getChild('str')?.type)).toBe(true)
|
||||
})
|
||||
const tableWithDict2 = makeArrowTable([{ str: "hello" }], { schema });
|
||||
expect(DataType.isDictionary(tableWithDict2.getChild("str")?.type)).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('will infer data types correctly', async function () {
|
||||
await checkTableCreation(async (records) => makeArrowTable(records), true)
|
||||
})
|
||||
it("will infer data types correctly", async function () {
|
||||
await checkTableCreation(async (records) => makeArrowTable(records), true);
|
||||
});
|
||||
|
||||
it('will allow a schema to be provided', async function () {
|
||||
await checkTableCreation(async (records, _, schema) => makeArrowTable(records, { schema }), false)
|
||||
})
|
||||
it("will allow a schema to be provided", async function () {
|
||||
await checkTableCreation(
|
||||
async (records, _, schema) => makeArrowTable(records, { schema }),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('will use the field order of any provided schema', async function () {
|
||||
await checkTableCreation(async (_, recordsReversed, schema) => makeArrowTable(recordsReversed, { schema }), false)
|
||||
})
|
||||
it("will use the field order of any provided schema", async function () {
|
||||
await checkTableCreation(
|
||||
async (_, recordsReversed, schema) =>
|
||||
makeArrowTable(recordsReversed, { schema }),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('will make an empty table', async function () {
|
||||
await checkTableCreation(async (_, __, schema) => makeArrowTable([], { schema }), false)
|
||||
})
|
||||
})
|
||||
it("will make an empty table", async function () {
|
||||
await checkTableCreation(
|
||||
async (_, __, schema) => makeArrowTable([], { schema }),
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
class DummyEmbedding implements EmbeddingFunction<string> {
|
||||
public readonly sourceColumn = 'string'
|
||||
public readonly embeddingDimension = 2
|
||||
public readonly embeddingDataType = new Float16()
|
||||
public readonly sourceColumn = "string";
|
||||
public readonly embeddingDimension = 2;
|
||||
public readonly embeddingDataType = new Float16();
|
||||
|
||||
async embed (data: string[]): Promise<number[][]> {
|
||||
return data.map(
|
||||
() => [0.0, 0.0]
|
||||
)
|
||||
async embed(data: string[]): Promise<number[][]> {
|
||||
return data.map(() => [0.0, 0.0]);
|
||||
}
|
||||
}
|
||||
|
||||
class DummyEmbeddingWithNoDimension implements EmbeddingFunction<string> {
|
||||
public readonly sourceColumn = 'string'
|
||||
public readonly sourceColumn = "string";
|
||||
|
||||
async embed (data: string[]): Promise<number[][]> {
|
||||
return data.map(
|
||||
() => [0.0, 0.0]
|
||||
)
|
||||
async embed(data: string[]): Promise<number[][]> {
|
||||
return data.map(() => [0.0, 0.0]);
|
||||
}
|
||||
}
|
||||
|
||||
describe('convertToTable', function () {
|
||||
it('will infer data types correctly', async function () {
|
||||
await checkTableCreation(async (records) => await convertToTable(records), true)
|
||||
})
|
||||
describe("convertToTable", function () {
|
||||
it("will infer data types correctly", async function () {
|
||||
await checkTableCreation(
|
||||
async (records) => await convertToTable(records),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('will allow a schema to be provided', async function () {
|
||||
await checkTableCreation(async (records, _, schema) => await convertToTable(records, undefined, { schema }), false)
|
||||
})
|
||||
it("will allow a schema to be provided", async function () {
|
||||
await checkTableCreation(
|
||||
async (records, _, schema) =>
|
||||
await convertToTable(records, undefined, { schema }),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('will use the field order of any provided schema', async function () {
|
||||
await checkTableCreation(async (_, recordsReversed, schema) => await convertToTable(recordsReversed, undefined, { schema }), false)
|
||||
})
|
||||
it("will use the field order of any provided schema", async function () {
|
||||
await checkTableCreation(
|
||||
async (_, recordsReversed, schema) =>
|
||||
await convertToTable(recordsReversed, undefined, { schema }),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('will make an empty table', async function () {
|
||||
await checkTableCreation(async (_, __, schema) => await convertToTable([], undefined, { schema }), false)
|
||||
})
|
||||
it("will make an empty table", async function () {
|
||||
await checkTableCreation(
|
||||
async (_, __, schema) => await convertToTable([], undefined, { schema }),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('will apply embeddings', async function () {
|
||||
const records = sampleRecords()
|
||||
const table = await convertToTable(records, new DummyEmbedding())
|
||||
expect(DataType.isFixedSizeList(table.getChild('vector')?.type)).toBe(true)
|
||||
expect(table.getChild('vector')?.type.children[0].type.toString()).toEqual(new Float16().toString())
|
||||
})
|
||||
it("will apply embeddings", async function () {
|
||||
const records = sampleRecords();
|
||||
const table = await convertToTable(records, new DummyEmbedding());
|
||||
expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe(true);
|
||||
expect(table.getChild("vector")?.type.children[0].type.toString()).toEqual(
|
||||
new Float16().toString(),
|
||||
);
|
||||
});
|
||||
|
||||
it('will fail if missing the embedding source column', async function () {
|
||||
await expect(convertToTable([{ id: 1 }], new DummyEmbedding())).rejects.toThrow("'string' was not present")
|
||||
})
|
||||
it("will fail if missing the embedding source column", async function () {
|
||||
await expect(
|
||||
convertToTable([{ id: 1 }], new DummyEmbedding()),
|
||||
).rejects.toThrow("'string' was not present");
|
||||
});
|
||||
|
||||
it('use embeddingDimension if embedding missing from table', async function () {
|
||||
const schema = new Schema([
|
||||
new Field('string', new Utf8(), false)
|
||||
])
|
||||
it("use embeddingDimension if embedding missing from table", async function () {
|
||||
const schema = new Schema([new Field("string", new Utf8(), false)]);
|
||||
// Simulate getting an empty Arrow table (minus embedding) from some other source
|
||||
// In other words, we aren't starting with records
|
||||
const table = makeEmptyTable(schema)
|
||||
const table = makeEmptyTable(schema);
|
||||
|
||||
// If the embedding specifies the dimension we are fine
|
||||
await fromTableToBuffer(table, new DummyEmbedding())
|
||||
await fromTableToBuffer(table, new DummyEmbedding());
|
||||
|
||||
// We can also supply a schema and should be ok
|
||||
const schemaWithEmbedding = new Schema([
|
||||
new Field('string', new Utf8(), false),
|
||||
new Field('vector', new FixedSizeList(2, new Field('item', new Float16(), false)), false)
|
||||
])
|
||||
await fromTableToBuffer(table, new DummyEmbeddingWithNoDimension(), schemaWithEmbedding)
|
||||
new Field("string", new Utf8(), false),
|
||||
new Field(
|
||||
"vector",
|
||||
new FixedSizeList(2, new Field("item", new Float16(), false)),
|
||||
false,
|
||||
),
|
||||
]);
|
||||
await fromTableToBuffer(
|
||||
table,
|
||||
new DummyEmbeddingWithNoDimension(),
|
||||
schemaWithEmbedding,
|
||||
);
|
||||
|
||||
// Otherwise we will get an error
|
||||
await expect(fromTableToBuffer(table, new DummyEmbeddingWithNoDimension())).rejects.toThrow('does not specify `embeddingDimension`')
|
||||
})
|
||||
await expect(
|
||||
fromTableToBuffer(table, new DummyEmbeddingWithNoDimension()),
|
||||
).rejects.toThrow("does not specify `embeddingDimension`");
|
||||
});
|
||||
|
||||
it('will apply embeddings to an empty table', async function () {
|
||||
it("will apply embeddings to an empty table", async function () {
|
||||
const schema = new Schema([
|
||||
new Field('string', new Utf8(), false),
|
||||
new Field('vector', new FixedSizeList(2, new Field('item', new Float16(), false)), false)
|
||||
])
|
||||
const table = await convertToTable([], new DummyEmbedding(), { schema })
|
||||
expect(DataType.isFixedSizeList(table.getChild('vector')?.type)).toBe(true)
|
||||
expect(table.getChild('vector')?.type.children[0].type.toString()).toEqual(new Float16().toString())
|
||||
})
|
||||
new Field("string", new Utf8(), false),
|
||||
new Field(
|
||||
"vector",
|
||||
new FixedSizeList(2, new Field("item", new Float16(), false)),
|
||||
false,
|
||||
),
|
||||
]);
|
||||
const table = await convertToTable([], new DummyEmbedding(), { schema });
|
||||
expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe(true);
|
||||
expect(table.getChild("vector")?.type.children[0].type.toString()).toEqual(
|
||||
new Float16().toString(),
|
||||
);
|
||||
});
|
||||
|
||||
it('will complain if embeddings present but schema missing embedding column', async function () {
|
||||
const schema = new Schema([
|
||||
new Field('string', new Utf8(), false)
|
||||
])
|
||||
await expect(convertToTable([], new DummyEmbedding(), { schema })).rejects.toThrow('column vector was missing')
|
||||
})
|
||||
it("will complain if embeddings present but schema missing embedding column", async function () {
|
||||
const schema = new Schema([new Field("string", new Utf8(), false)]);
|
||||
await expect(
|
||||
convertToTable([], new DummyEmbedding(), { schema }),
|
||||
).rejects.toThrow("column vector was missing");
|
||||
});
|
||||
|
||||
it('will provide a nice error if run twice', async function () {
|
||||
const records = sampleRecords()
|
||||
const table = await convertToTable(records, new DummyEmbedding())
|
||||
it("will provide a nice error if run twice", async function () {
|
||||
const records = sampleRecords();
|
||||
const table = await convertToTable(records, new DummyEmbedding());
|
||||
// fromTableToBuffer will try and apply the embeddings again
|
||||
await expect(fromTableToBuffer(table, new DummyEmbedding())).rejects.toThrow('already existed')
|
||||
})
|
||||
})
|
||||
await expect(
|
||||
fromTableToBuffer(table, new DummyEmbedding()),
|
||||
).rejects.toThrow("already existed");
|
||||
});
|
||||
});
|
||||
|
||||
describe('makeEmptyTable', function () {
|
||||
it('will make an empty table', async function () {
|
||||
await checkTableCreation(async (_, __, schema) => makeEmptyTable(schema), false)
|
||||
})
|
||||
})
|
||||
describe("makeEmptyTable", function () {
|
||||
it("will make an empty table", async function () {
|
||||
await checkTableCreation(
|
||||
async (_, __, schema) => makeEmptyTable(schema),
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -17,55 +17,58 @@ import * as tmp from "tmp";
|
||||
import { Connection, connect } from "../dist/index.js";
|
||||
|
||||
describe("when connecting", () => {
|
||||
|
||||
let tmpDir: tmp.DirResult;
|
||||
beforeEach(() => tmpDir = tmp.dirSync({ unsafeCleanup: true }));
|
||||
beforeEach(() => (tmpDir = tmp.dirSync({ unsafeCleanup: true })));
|
||||
afterEach(() => tmpDir.removeCallback());
|
||||
|
||||
it("should connect", async() => {
|
||||
it("should connect", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
expect(db.display()).toBe(`NativeDatabase(uri=${tmpDir.name}, read_consistency_interval=None)`);
|
||||
})
|
||||
expect(db.display()).toBe(
|
||||
`NativeDatabase(uri=${tmpDir.name}, read_consistency_interval=None)`,
|
||||
);
|
||||
});
|
||||
|
||||
it("should allow read consistency interval to be specified", async() => {
|
||||
const db = await connect(tmpDir.name, { readConsistencyInterval: 5});
|
||||
expect(db.display()).toBe(`NativeDatabase(uri=${tmpDir.name}, read_consistency_interval=5s)`);
|
||||
})
|
||||
it("should allow read consistency interval to be specified", async () => {
|
||||
const db = await connect(tmpDir.name, { readConsistencyInterval: 5 });
|
||||
expect(db.display()).toBe(
|
||||
`NativeDatabase(uri=${tmpDir.name}, read_consistency_interval=5s)`,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("given a connection", () => {
|
||||
|
||||
let tmpDir: tmp.DirResult
|
||||
let db: Connection
|
||||
let tmpDir: tmp.DirResult;
|
||||
let db: Connection;
|
||||
beforeEach(async () => {
|
||||
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
db = await connect(tmpDir.name)
|
||||
db = await connect(tmpDir.name);
|
||||
});
|
||||
afterEach(() => tmpDir.removeCallback());
|
||||
|
||||
it("should raise an error if opening a non-existent table", async() => {
|
||||
it("should raise an error if opening a non-existent table", async () => {
|
||||
await expect(db.openTable("non-existent")).rejects.toThrow("was not found");
|
||||
})
|
||||
});
|
||||
|
||||
it("should raise an error if any operation is tried after it is closed", async() => {
|
||||
it("should raise an error if any operation is tried after it is closed", async () => {
|
||||
expect(db.isOpen()).toBe(true);
|
||||
await db.close();
|
||||
expect(db.isOpen()).toBe(false);
|
||||
await expect(db.tableNames()).rejects.toThrow("Connection is closed");
|
||||
})
|
||||
});
|
||||
|
||||
it("should fail if creating table twice, unless overwrite is true", async() => {
|
||||
it("should fail if creating table twice, unless overwrite is true", async () => {
|
||||
let tbl = await db.createTable("test", [{ id: 1 }, { id: 2 }]);
|
||||
await expect(tbl.countRows()).resolves.toBe(2);
|
||||
await expect(db.createTable("test", [{ id: 1 }, { id: 2 }])).rejects.toThrow();
|
||||
await expect(
|
||||
db.createTable("test", [{ id: 1 }, { id: 2 }]),
|
||||
).rejects.toThrow();
|
||||
tbl = await db.createTable("test", [{ id: 3 }], { mode: "overwrite" });
|
||||
await expect(tbl.countRows()).resolves.toBe(1);
|
||||
})
|
||||
});
|
||||
|
||||
it("should list tables", async() => {
|
||||
it("should list tables", async () => {
|
||||
await db.createTable("test2", [{ id: 1 }, { id: 2 }]);
|
||||
await db.createTable("test1", [{ id: 1 }, { id: 2 }]);
|
||||
expect(await db.tableNames()).toEqual(["test1", "test2"]);
|
||||
})
|
||||
|
||||
});
|
||||
});
|
||||
|
||||
@@ -17,15 +17,21 @@ import * as path from "path";
|
||||
import * as tmp from "tmp";
|
||||
|
||||
import { Table, connect } from "../dist";
|
||||
import { Schema, Field, Float32, Int32, FixedSizeList, Int64, Float64 } from "apache-arrow";
|
||||
import {
|
||||
Schema,
|
||||
Field,
|
||||
Float32,
|
||||
Int32,
|
||||
FixedSizeList,
|
||||
Int64,
|
||||
Float64,
|
||||
} from "apache-arrow";
|
||||
import { makeArrowTable } from "../dist/arrow";
|
||||
|
||||
describe("Given a table", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
let table: Table;
|
||||
const schema = new Schema([
|
||||
new Field("id", new Float64(), true),
|
||||
]);
|
||||
const schema = new Schema([new Field("id", new Float64(), true)]);
|
||||
beforeEach(async () => {
|
||||
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
const conn = await connect(tmpDir.name);
|
||||
@@ -34,31 +40,32 @@ describe("Given a table", () => {
|
||||
afterEach(() => tmpDir.removeCallback());
|
||||
|
||||
it("be displayable", async () => {
|
||||
expect(table.display()).toMatch(/NativeTable\(some_table, uri=.*, read_consistency_interval=None\)/);
|
||||
table.close()
|
||||
expect(table.display()).toBe("ClosedTable(some_table)")
|
||||
})
|
||||
expect(table.display()).toMatch(
|
||||
/NativeTable\(some_table, uri=.*, read_consistency_interval=None\)/,
|
||||
);
|
||||
table.close();
|
||||
expect(table.display()).toBe("ClosedTable(some_table)");
|
||||
});
|
||||
|
||||
it("should let me add data", async () => {
|
||||
await table.add([{ id: 1 }, { id: 2 }]);
|
||||
await table.add([{ id: 1 }]);
|
||||
await expect(table.countRows()).resolves.toBe(3);
|
||||
})
|
||||
});
|
||||
|
||||
it("should overwrite data if asked", async () => {
|
||||
await table.add([{ id: 1 }, { id: 2 }]);
|
||||
await table.add([{ id: 1 }], { mode: "overwrite" });
|
||||
await expect(table.countRows()).resolves.toBe(1);
|
||||
})
|
||||
});
|
||||
|
||||
it("should let me close the table", async () => {
|
||||
expect(table.isOpen()).toBe(true);
|
||||
table.close();
|
||||
expect(table.isOpen()).toBe(false);
|
||||
expect(table.countRows()).rejects.toThrow("Table some_table is closed");
|
||||
})
|
||||
|
||||
})
|
||||
});
|
||||
});
|
||||
|
||||
describe("Test creating index", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
@@ -85,7 +92,7 @@ describe("Test creating index", () => {
|
||||
})),
|
||||
{
|
||||
schema,
|
||||
}
|
||||
},
|
||||
);
|
||||
const tbl = await db.createTable("test", data);
|
||||
await tbl.createIndex().build();
|
||||
@@ -113,10 +120,10 @@ describe("Test creating index", () => {
|
||||
makeArrowTable([
|
||||
{ id: 1, val: 2 },
|
||||
{ id: 2, val: 3 },
|
||||
])
|
||||
]),
|
||||
);
|
||||
await expect(tbl.createIndex().build()).rejects.toThrow(
|
||||
"No vector column found"
|
||||
"No vector column found",
|
||||
);
|
||||
|
||||
await tbl.createIndex("val").build();
|
||||
@@ -135,7 +142,7 @@ describe("Test creating index", () => {
|
||||
new Field("vec", new FixedSizeList(32, new Field("item", new Float32()))),
|
||||
new Field(
|
||||
"vec2",
|
||||
new FixedSizeList(64, new Field("item", new Float32()))
|
||||
new FixedSizeList(64, new Field("item", new Float32())),
|
||||
),
|
||||
]);
|
||||
const tbl = await db.createTable(
|
||||
@@ -152,13 +159,13 @@ describe("Test creating index", () => {
|
||||
.fill(1)
|
||||
.map(() => Math.random()),
|
||||
})),
|
||||
{ schema }
|
||||
)
|
||||
{ schema },
|
||||
),
|
||||
);
|
||||
|
||||
// Only build index over v1
|
||||
await expect(tbl.createIndex().build()).rejects.toThrow(
|
||||
/.*More than one vector columns found.*/
|
||||
/.*More than one vector columns found.*/,
|
||||
);
|
||||
tbl
|
||||
.createIndex("vec")
|
||||
@@ -170,7 +177,7 @@ describe("Test creating index", () => {
|
||||
.nearestTo(
|
||||
Array(32)
|
||||
.fill(1)
|
||||
.map(() => Math.random())
|
||||
.map(() => Math.random()),
|
||||
)
|
||||
.limit(2)
|
||||
.toArrow();
|
||||
@@ -183,10 +190,10 @@ describe("Test creating index", () => {
|
||||
Array(64)
|
||||
.fill(1)
|
||||
.map(() => Math.random()),
|
||||
"vec"
|
||||
"vec",
|
||||
)
|
||||
.limit(2)
|
||||
.toArrow()
|
||||
.toArrow(),
|
||||
).rejects.toThrow(/.*does not match the dimension.*/);
|
||||
|
||||
const query64 = Array(64)
|
||||
@@ -211,7 +218,7 @@ describe("Test creating index", () => {
|
||||
})),
|
||||
{
|
||||
schema,
|
||||
}
|
||||
},
|
||||
);
|
||||
const tbl = await db.createTable("test", data);
|
||||
await tbl.createIndex("id").build();
|
||||
@@ -224,7 +231,6 @@ describe("Test creating index", () => {
|
||||
});
|
||||
|
||||
describe("Read consistency interval", () => {
|
||||
|
||||
let tmpDir: tmp.DirResult;
|
||||
beforeEach(() => {
|
||||
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
@@ -237,7 +243,9 @@ describe("Read consistency interval", () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const table = await db.createTable("my_table", [{ id: 1 }]);
|
||||
|
||||
const db2 = await connect(tmpDir.name, { readConsistencyInterval: interval });
|
||||
const db2 = await connect(tmpDir.name, {
|
||||
readConsistencyInterval: interval,
|
||||
});
|
||||
const table2 = await db2.openTable("my_table");
|
||||
expect(await table2.countRows()).toEqual(await table.countRows());
|
||||
|
||||
@@ -253,77 +261,89 @@ describe("Read consistency interval", () => {
|
||||
} else {
|
||||
// interval == 0.1
|
||||
expect(await table2.countRows()).toEqual(1);
|
||||
await new Promise(r => setTimeout(r, 100));
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
expect(await table2.countRows()).toEqual(2);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe('schema evolution', function () {
|
||||
|
||||
describe("schema evolution", function () {
|
||||
let tmpDir: tmp.DirResult;
|
||||
beforeEach(() => {
|
||||
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
});
|
||||
afterEach(() => {
|
||||
tmpDir.removeCallback();
|
||||
})
|
||||
});
|
||||
|
||||
// Create a new sample table
|
||||
it('can add a new column to the schema', async function () {
|
||||
const con = await connect(tmpDir.name)
|
||||
const table = await con.createTable('vectors', [
|
||||
{ id: 1n, vector: [0.1, 0.2] }
|
||||
])
|
||||
it("can add a new column to the schema", async function () {
|
||||
const con = await connect(tmpDir.name);
|
||||
const table = await con.createTable("vectors", [
|
||||
{ id: 1n, vector: [0.1, 0.2] },
|
||||
]);
|
||||
|
||||
await table.addColumns([{ name: 'price', valueSql: 'cast(10.0 as float)' }])
|
||||
await table.addColumns([
|
||||
{ name: "price", valueSql: "cast(10.0 as float)" },
|
||||
]);
|
||||
|
||||
const expectedSchema = new Schema([
|
||||
new Field('id', new Int64(), true),
|
||||
new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true)), true),
|
||||
new Field('price', new Float32(), false)
|
||||
])
|
||||
expect(await table.schema()).toEqual(expectedSchema)
|
||||
new Field("id", new Int64(), true),
|
||||
new Field(
|
||||
"vector",
|
||||
new FixedSizeList(2, new Field("item", new Float32(), true)),
|
||||
true,
|
||||
),
|
||||
new Field("price", new Float32(), false),
|
||||
]);
|
||||
expect(await table.schema()).toEqual(expectedSchema);
|
||||
});
|
||||
|
||||
it('can alter the columns in the schema', async function () {
|
||||
const con = await connect(tmpDir.name)
|
||||
it("can alter the columns in the schema", async function () {
|
||||
const con = await connect(tmpDir.name);
|
||||
const schema = new Schema([
|
||||
new Field('id', new Int64(), true),
|
||||
new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true)), true),
|
||||
new Field('price', new Float64(), false)
|
||||
])
|
||||
const table = await con.createTable('vectors', [
|
||||
{ id: 1n, vector: [0.1, 0.2] }
|
||||
])
|
||||
new Field("id", new Int64(), true),
|
||||
new Field(
|
||||
"vector",
|
||||
new FixedSizeList(2, new Field("item", new Float32(), true)),
|
||||
true,
|
||||
),
|
||||
new Field("price", new Float64(), false),
|
||||
]);
|
||||
const table = await con.createTable("vectors", [
|
||||
{ id: 1n, vector: [0.1, 0.2] },
|
||||
]);
|
||||
// Can create a non-nullable column only through addColumns at the moment.
|
||||
await table.addColumns([{ name: 'price', valueSql: 'cast(10.0 as double)' }])
|
||||
expect(await table.schema()).toEqual(schema)
|
||||
await table.addColumns([
|
||||
{ name: "price", valueSql: "cast(10.0 as double)" },
|
||||
]);
|
||||
expect(await table.schema()).toEqual(schema);
|
||||
|
||||
await table.alterColumns([
|
||||
{ path: 'id', rename: 'new_id' },
|
||||
{ path: 'price', nullable: true }
|
||||
])
|
||||
{ path: "id", rename: "new_id" },
|
||||
{ path: "price", nullable: true },
|
||||
]);
|
||||
|
||||
const expectedSchema = new Schema([
|
||||
new Field('new_id', new Int64(), true),
|
||||
new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true)), true),
|
||||
new Field('price', new Float64(), true)
|
||||
])
|
||||
expect(await table.schema()).toEqual(expectedSchema)
|
||||
new Field("new_id", new Int64(), true),
|
||||
new Field(
|
||||
"vector",
|
||||
new FixedSizeList(2, new Field("item", new Float32(), true)),
|
||||
true,
|
||||
),
|
||||
new Field("price", new Float64(), true),
|
||||
]);
|
||||
expect(await table.schema()).toEqual(expectedSchema);
|
||||
});
|
||||
|
||||
it('can drop a column from the schema', async function () {
|
||||
const con = await connect(tmpDir.name)
|
||||
const table = await con.createTable('vectors', [
|
||||
{ id: 1n, vector: [0.1, 0.2] }
|
||||
])
|
||||
await table.dropColumns(['vector'])
|
||||
it("can drop a column from the schema", async function () {
|
||||
const con = await connect(tmpDir.name);
|
||||
const table = await con.createTable("vectors", [
|
||||
{ id: 1n, vector: [0.1, 0.2] },
|
||||
]);
|
||||
await table.dropColumns(["vector"]);
|
||||
|
||||
const expectedSchema = new Schema([
|
||||
new Field('id', new Int64(), true)
|
||||
])
|
||||
expect(await table.schema()).toEqual(expectedSchema)
|
||||
const expectedSchema = new Schema([new Field("id", new Int64(), true)]);
|
||||
expect(await table.schema()).toEqual(expectedSchema);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,15 +1,10 @@
|
||||
{
|
||||
"extends": "../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "./dist/spec",
|
||||
"module": "commonjs",
|
||||
"target": "es2022",
|
||||
"types": [
|
||||
"jest",
|
||||
"node"
|
||||
]
|
||||
},
|
||||
"include": [
|
||||
"**/*",
|
||||
]
|
||||
"extends": "../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "./dist/spec",
|
||||
"module": "commonjs",
|
||||
"target": "es2022",
|
||||
"types": ["jest", "node"]
|
||||
},
|
||||
"include": ["**/*"]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user