rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-05-26 08:20:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

add openai embedding function to nodejs client (#107)

Browse Source 
- openai is an optional dependency for lancedb
- added an example to show how to use it
This commit is contained in:
gsilvestrin
2023-06-01 10:25:00 -07:00
committed by GitHub
parent 99cbda8b07
commit 3e14b357e7
12 changed files with 683 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
node/src/arrow.ts
View File

@@ -24,7 +24,7 @@ import {
} from 'apache-arrow'
import { type EmbeddingFunction } from './index'
export function convertToTable<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Table {
export async function convertToTable<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Table> {
if (data.length === 0) {
throw new Error('At least one record needs to be provided')
}
@@ -51,7 +51,7 @@ export function convertToTable<T> (data: Array<Record<string, unknown>>, embeddi
}
if (columnsKey === embeddings?.sourceColumn) {
const vectors = embeddings.embed(values as T[])
const vectors = await embeddings.embed(values as T[])
const listBuilder = newVectorListBuilder()
vectors.map(v => listBuilder.append(v))
records.vector = listBuilder.finish().toVector()
@@ -79,7 +79,7 @@ function newVectorListBuilder (): ListBuilder<Float32, any> {
}
export async function fromRecordsToBuffer<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Buffer> {
const table = convertToTable(data, embeddings)
const table = await convertToTable(data, embeddings)
const writer = RecordBatchFileWriter.writeAll(table)
return Buffer.from(await writer.toUint8Array())
}