lancedb/docs/web-tests/ts/multimodal.test.ts

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
// --8<-- [start:multimodal_imports]
import * as arrow from "apache-arrow";
import { Buffer } from "node:buffer";
import * as lancedb from "@lancedb/lancedb";
// --8<-- [end:multimodal_imports]
import { withTempDirectory } from "./util.ts";

test("multimodal snippets (async)", async () => {
  await withTempDirectory(async (databaseDir) => {
    const db = await lancedb.connect(databaseDir);

    // --8<-- [start:create_dummy_data]
    const createDummyImage = (color: string): Uint8Array => {
      const pngHeader = Uint8Array.from([137, 80, 78, 71, 13, 10, 26, 10]);
      return Buffer.concat([Buffer.from(pngHeader), Buffer.from(color, "utf8")]);
    };

    const data = [
      {
        id: 1,
        filename: "red_square.png",
        vector: Array.from({ length: 128 }, (_, i) => (i % 16) / 16),
        image_blob: createDummyImage("red"),
        label: "red",
      },
      {
        id: 2,
        filename: "blue_square.png",
        vector: Array.from({ length: 128 }, (_, i) => ((i + 8) % 16) / 16),
        image_blob: createDummyImage("blue"),
        label: "blue",
      },
    ];
    // --8<-- [end:create_dummy_data]

    // --8<-- [start:define_schema]
    const schema = new arrow.Schema([
      new arrow.Field("id", new arrow.Int32()),
      new arrow.Field("filename", new arrow.Utf8()),
      new arrow.Field(
        "vector",
        new arrow.FixedSizeList(
          128,
          new arrow.Field("item", new arrow.Float32(), true),
        ),
      ),
      new arrow.Field("image_blob", new arrow.Binary()),
      new arrow.Field("label", new arrow.Utf8()),
    ]);
    // --8<-- [end:define_schema]

    // --8<-- [start:ingest_data]
    const multimodalData = lancedb.makeArrowTable(data, { schema });
    const tbl = await db.createTable("images", multimodalData, {
      mode: "overwrite",
    });
    // --8<-- [end:ingest_data]
    expect(await tbl.countRows()).toBe(2);

    // --8<-- [start:search_data]
    const queryVector = Array.from({ length: 128 }, (_, i) => (i % 16) / 16);
    const results = await tbl.search(queryVector).limit(1).toArray();
    // --8<-- [end:search_data]

    // --8<-- [start:process_results]
    for (const row of results) {
      const imageBytes = row.image_blob as Uint8Array;
      console.log(
        `Retrieved image: ${row.filename}, Byte length: ${imageBytes.length}`,
      );
    }
    // --8<-- [end:process_results]
    expect(results).toHaveLength(1);

    // --8<-- [start:blob_api_schema]
    const blobSchema = new arrow.Schema([
      new arrow.Field("id", new arrow.Int64()),
      new arrow.Field(
        "video",
        new arrow.LargeBinary(),
        true,
        new Map([["lance-encoding:blob", "true"]]),
      ),
    ]);
    // --8<-- [end:blob_api_schema]

    // --8<-- [start:blob_api_ingest]
    const blobData = lancedb.makeArrowTable(
      [
        { id: 1, video: Buffer.from("fake_video_bytes_1") },
        { id: 2, video: Buffer.from("fake_video_bytes_2") },
      ],
      { schema: blobSchema },
    );
    const blobTable = await db.createTable("videos", blobData, {
      mode: "overwrite",
    });
    // --8<-- [end:blob_api_ingest]
    expect(await blobTable.countRows()).toBe(2);
  });
});