diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 17fe3a7b4..ffd2162c6 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -11,6 +11,7 @@ on: - rust-toolchain.toml - deny.toml - rust/** + - docs/web-tests/blob-v2/rs/** - nodejs/Cargo.toml - python/Cargo.toml - .github/workflows/rust.yml @@ -222,6 +223,11 @@ jobs: run: CARGO_ARGS="--profile ci" make -C ./lancedb feature-tests - name: Run examples run: cargo run --profile ci --all-features --example simple --locked + - name: Run blob v2 docs example + # Python and TypeScript blob docs tests install preview packages. + # Those builds are not guaranteed to remain available for more than + # 6 months, so they stay a manual run. This example uses the workspace crate. + run: cargo run --profile ci --all-features -p lancedb-blob-v2-docs --example multimodal --locked remote: timeout-minutes: 30 diff --git a/Cargo.lock b/Cargo.lock index 03f4b0864..bbd41c0b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5796,6 +5796,18 @@ dependencies = [ "webbrowser", ] +[[package]] +name = "lancedb-blob-v2-docs" +version = "0.0.0" +dependencies = [ + "arrow-array", + "arrow-schema", + "futures-util", + "lancedb", + "tempfile", + "tokio", +] + [[package]] name = "lancedb-nodejs" version = "0.41.0-beta.14" diff --git a/Cargo.toml b/Cargo.toml index 870ef5d07..3027ee065 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,10 @@ [workspace] -members = ["rust/lancedb", "nodejs", "python"] +members = [ + "rust/lancedb", + "nodejs", + "python", + "docs/web-tests/blob-v2/rs", +] resolver = "2" [workspace.package] diff --git a/docs/web-tests/blob-v2/README.md b/docs/web-tests/blob-v2/README.md new file mode 100644 index 000000000..c0f75cdc6 --- /dev/null +++ b/docs/web-tests/blob-v2/README.md @@ -0,0 +1,55 @@ +# Blob v2 documentation tests + +`docs/web-tests/ts` pins `@lancedb/lancedb` 0.31.0 and `docs/web-tests/rs` pins `lancedb` 0.31.0. Those releases do not include `blob()`. Run the Blob v2 examples here. + +## Python + +`lancedb==0.41.0b10` from the LanceDB package index, without pylance. PyPI `lancedb` 0.40.0 does not provide `lancedb.blob()`. + +```shell +python3.13 -m venv .venv +.venv/bin/pip install -r docs/web-tests/blob-v2/py/requirements.txt +.venv/bin/python -m pytest docs/web-tests/blob-v2/py/test_multimodal.py +``` + +`test_blob_pylance.py` checks custom thresholds and `to_pandas()` in `lazy` and `bytes` mode. Those calls import pylance. Use a second virtualenv. `test_multimodal.py` fails if `lance` can be imported. Snippets from this file are written to `docs/web/snippets/blob_pylance.mdx`. + +This install is for the optional tests only. The public guide does not tell users to bypass the dependency resolver. PyPI `pylance` 13.0.0 requires `lance-namespace<0.12`, and `lancedb==0.41.0b10` requires `lance-namespace>=0.13`, so `lancedb[pylance]` does not resolve. The three tests passed with pylance installed after the base requirements, without its own dependency pin. + +```shell +python3.13 -m venv .venv-pylance +.venv-pylance/bin/pip install -r docs/web-tests/blob-v2/py/requirements.txt +.venv-pylance/bin/pip install --no-deps pylance==13.0.0 +.venv-pylance/bin/python -m pytest docs/web-tests/blob-v2/py/test_blob_pylance.py +``` + +## TypeScript + +`@lancedb/lancedb@0.41.0-beta.8`. Node.js 22.6 or newer. `npm test` passes `--experimental-strip-types`, which Node added in 22.6.0. + +```shell +cd docs/web-tests/blob-v2/ts +npm ci +npm test +``` + +## Rust + +crates.io `lancedb` 0.40.0 does not include `blob()`. This example builds the workspace crate, `lancedb` 0.41.0-beta.10. Pass `--features remote` so this checkout compiles. The example uses a local database. The Rust workflow runs this example. It is not a default feature, so a workspace build without `remote` does not turn that feature on for the rest of the workspace. + +```shell +cargo run -p lancedb-blob-v2-docs --features remote --example multimodal +``` + +## CI + +Python and TypeScript above install preview packages. Pre-release builds are not guaranteed to stay available for more than 6 months, so those runs are not required PR checks. Run them when the guide's pins change. The Rust example is the in-repo check. + +Regenerate snippets from the repository root: + +```shell +python3 docs/web-tests/mdx_snippets_gen.py \ + -s docs/web-tests/py -s docs/web-tests/ts -s docs/web-tests/rs \ + -s docs/web-tests/blob-v2/py -s docs/web-tests/blob-v2/ts -s docs/web-tests/blob-v2/rs \ + -o docs/web/snippets +``` diff --git a/docs/web-tests/blob-v2/py/pytest.ini b/docs/web-tests/blob-v2/py/pytest.ini new file mode 100644 index 000000000..17c9e9cb4 --- /dev/null +++ b/docs/web-tests/blob-v2/py/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +asyncio_mode = auto +testpaths = test_multimodal.py diff --git a/docs/web-tests/blob-v2/py/requirements.txt b/docs/web-tests/blob-v2/py/requirements.txt new file mode 100644 index 000000000..c90c8d622 --- /dev/null +++ b/docs/web-tests/blob-v2/py/requirements.txt @@ -0,0 +1,8 @@ +--extra-index-url https://pypi.fury.io/lancedb/ + +lancedb==0.41.0b10 +numpy==2.2.6 +pandas==2.3.3 +pillow==12.1.1 +pytest==8.4.2 +pytest-asyncio==1.2.0 diff --git a/docs/web-tests/blob-v2/py/test_blob_pylance.py b/docs/web-tests/blob-v2/py/test_blob_pylance.py new file mode 100644 index 000000000..8f66088a7 --- /dev/null +++ b/docs/web-tests/blob-v2/py/test_blob_pylance.py @@ -0,0 +1,154 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""Blob reads that import pylance. + +Install pylance==13.0.0 with --no-deps after lancedb==0.41.0b10. +See docs/web-tests/blob-v2/README.md. +""" + +import lance +import lancedb +import numpy as np +import pyarrow as pa +import pytest + + +def test_blob_api_to_pandas(tmp_path): + db = lancedb.connect(tmp_path) + schema = pa.schema( + [ + pa.field("id", pa.int64()), + lancedb.blob("video"), + ] + ) + tbl = db.create_table( + "videos", + data=[ + {"id": 1, "video": b"fake_video_bytes_1"}, + {"id": 2, "video": b"fake_video_bytes_2"}, + ], + schema=schema, + mode="overwrite", + ) + + # --8<-- [start:blob_api_to_pandas] + df_lazy = tbl.to_pandas() + + df_bytes = tbl.to_pandas(blob_mode="bytes") + + df_desc = tbl.to_pandas(blob_mode="descriptions") + + arrow_bytes = tbl.to_arrow(blob_mode="bytes") + + df_typed = tbl.to_pandas(split_blocks=True, self_destruct=True) + # --8<-- [end:blob_api_to_pandas] + + assert len(df_lazy) == 2 + assert all(isinstance(value, lance.BlobFile) for value in df_lazy["video"]) + assert isinstance(df_bytes["video"].iloc[0], bytes) + assert df_bytes["video"].tolist() == [ + b"fake_video_bytes_1", + b"fake_video_bytes_2", + ] + assert arrow_bytes["video"].to_pylist() == [ + b"fake_video_bytes_1", + b"fake_video_bytes_2", + ] + assert "_rowid" not in df_bytes.columns + assert len(df_desc) == 2 + assert set(df_desc["video"].iloc[0]) == { + "kind", + "position", + "size", + "blob_id", + "blob_uri", + } + assert len(df_typed) == 2 + + +@pytest.mark.asyncio +async def test_query_to_pandas_kwargs(tmp_path): + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("vector", pa.list_(pa.float32(), 128)), + lancedb.blob("video"), + ] + ) + data = [ + { + "id": i, + "vector": np.random.rand(128).astype(np.float32), + "video": f"fake_video_bytes_{i}".encode(), + } + for i in range(10) + ] + + db = lancedb.connect(tmp_path / "sync") + tbl = db.create_table("search_demo", data=data, schema=schema, mode="overwrite") + + async_db = await lancedb.connect_async(str(tmp_path / "async")) + tbl_async = await async_db.create_table( + "search_demo", data=data, schema=schema, mode="overwrite" + ) + + query_vector = np.random.rand(128).astype(np.float32) + + # --8<-- [start:query_to_pandas_kwargs] + df_lazy = ( + tbl.search().where("id = 1").select(["id", "video"]).to_pandas(blob_mode="lazy") + ) + + df_bytes = await ( + tbl_async.query() + .where("id = 1") + .select(["id", "video"]) + .to_pandas(blob_mode="bytes") + ) + + df_vec = ( + tbl.search(query_vector) + .limit(2) + .select(["id", "video"]) + .to_pandas(blob_mode="bytes", split_blocks=True, self_destruct=True) + ) + # --8<-- [end:query_to_pandas_kwargs] + + assert len(df_lazy) == 1 + assert "_rowid" not in df_bytes.columns + assert len(df_bytes) == 1 + assert df_bytes["video"].iloc[0] == b"fake_video_bytes_1" + assert len(df_vec) == 2 + assert isinstance(df_vec["video"].iloc[0], bytes) + + +def test_append_keeps_table_blob_thresholds(tmp_path): + db = lancedb.connect(tmp_path) + schema = pa.schema( + [ + pa.field("id", pa.int64()), + lancedb.blob("video", inline_size_threshold=64 * 1024), + ] + ) + tbl = db.create_table("videos", schema=schema) + other = pa.schema( + [ + pa.field("id", pa.int64()), + lancedb.blob("video", inline_size_threshold=8 * 1024), + ] + ) + payload = b"x" * (20 * 1024) + tbl.add( + pa.table( + [pa.array([1], type=pa.int64()), lance.blob_array([payload])], + schema=other, + ) + ) + + stored = tbl.schema.field("video").metadata[ + b"lance-encoding:blob-inline-size-threshold" + ] + assert stored == b"65536" + descriptor = tbl.search().select(["video"]).to_arrow(blob_mode="descriptions") + assert descriptor.column("video")[0].as_py()["kind"] == 0 diff --git a/docs/web-tests/blob-v2/py/test_multimodal.py b/docs/web-tests/blob-v2/py/test_multimodal.py new file mode 100644 index 000000000..bf14ba57b --- /dev/null +++ b/docs/web-tests/blob-v2/py/test_multimodal.py @@ -0,0 +1,204 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import importlib.util + +import lancedb +import pyarrow as pa + +if importlib.util.find_spec("lance") is not None: + raise RuntimeError( + "Blob v2 documentation tests must run without pylance. " + "Install docs/web-tests/blob-v2/py/requirements.txt into a fresh virtualenv." + ) + +INLINE_MAX = 64 * 1024 +DEDICATED_MIN = 4 * 1024 * 1024 + + +def test_blob_api_definition(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + # --8<-- [start:blob_api_setup] + import lancedb + + db = lancedb.connect("./blob_db") + # --8<-- [end:blob_api_setup] + assert db.list_tables().tables == [] + + # --8<-- [start:blob_api_schema] + import lancedb + import pyarrow as pa + + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("label", pa.string()), + lancedb.blob("image"), + ] + ) + # --8<-- [end:blob_api_schema] + image = schema.field("image").type + assert image.extension_name == "lance.blob.v2" + assert type(image).__module__ == "lancedb.schema" + + # --8<-- [start:blob_api_ingest] + import io + from PIL import Image + + def jpeg_with_make(color, make): + image = Image.new("RGB", (8, 8), color=color) + exif = Image.Exif() + exif[0x010F] = make + buf = io.BytesIO() + image.save(buf, format="JPEG", exif=exif) + return buf.getvalue() + + tbl = db.create_table( + "photos", + data=[ + {"id": 1, "label": "red", "image": jpeg_with_make("red", "TestCam")}, + {"id": 2, "label": "blue", "image": jpeg_with_make("blue", "OtherCam")}, + ], + schema=schema, + mode="overwrite", + ) + # --8<-- [end:blob_api_ingest] + assert len(tbl) == 2 + + # --8<-- [start:blob_api_bytes] + rows = ( + tbl.search().where("id = 1").select(["id", "image"]).to_list(blob_mode="bytes") + ) + # --8<-- [end:blob_api_bytes] + assert len(rows) == 1 + assert rows[0]["id"] == 1 + assert "_rowid" not in rows[0] + assert rows[0]["image"].startswith(b"\xff\xd8") + + # --8<-- [start:blob_api_files] + from PIL import ExifTags + + rows = tbl.search().where("id = 1").select(["label"]).with_row_id(True).to_list() + with tbl.fetch_blob_files("image", [rows[0]["_rowid"]])[0] as image_file: + make = Image.open(image_file).getexif().get(ExifTags.Base.Make) + print(rows[0]["label"], make) + # --8<-- [end:blob_api_files] + assert rows[0]["label"] == "red" + assert make == "TestCam" + + # --8<-- [start:blob_api_fetch] + rows = tbl.search().select(["id"]).with_row_id(True).to_list() + row_ids = [row["_rowid"] for row in rows] + payloads = tbl.fetch_blobs("image", row_ids) + # --8<-- [end:blob_api_fetch] + assert all(payload.startswith(b"\xff\xd8") for payload in payloads.to_pylist()) + + # --8<-- [start:blob_api_ranges] + slices = tbl.fetch_blob_ranges("image", [(row_ids[0], 0, 2)]) + + with tbl.fetch_blob_files("image", [row_ids[0]])[0] as image_file: + parts = image_file.read_ranges([(0, 2)]) + # --8<-- [end:blob_api_ranges] + assert slices.to_pylist() == [b"\xff\xd8"] + assert parts == [b"\xff\xd8"] + + +def test_eager_query_returns_blob_bytes_without_row_id(tmp_path): + db = lancedb.connect(tmp_path) + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + payload = b"inline-bytes" + tbl = db.create_table( + "photos", + data=[{"id": 1, "image": payload}], + schema=schema, + ) + + rows = ( + tbl.search().where("id = 1").select(["id", "image"]).to_list(blob_mode="bytes") + ) + arrow_bytes = ( + tbl.search().where("id = 1").select(["id", "image"]).to_arrow(blob_mode="bytes") + ) + described = tbl.to_pandas(blob_mode="descriptions") + + assert rows == [{"id": 1, "image": payload}] + assert "_rowid" not in rows[0] + assert arrow_bytes.column("image").to_pylist() == [payload] + assert "_rowid" not in arrow_bytes.column_names + assert described["image"].iloc[0]["kind"] == 0 + + +def test_deferred_fetch_reads_blob_by_row_id(tmp_path): + db = lancedb.connect(tmp_path) + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + payload = b"deferred-bytes" + tbl = db.create_table( + "photos", + data=[{"id": 1, "image": payload}], + schema=schema, + ) + + described = ( + tbl.search() + .where("id = 1") + .select(["id", "image"]) + .to_list(blob_mode="descriptions") + ) + assert described[0]["image"]["kind"] == 0 + assert "_rowid" not in described[0] + + found = tbl.search().where("id = 1").select(["id"]).with_row_id(True).to_list() + row_id = found[0]["_rowid"] + fetched = tbl.fetch_blobs("image", [row_id]).to_pylist() + with tbl.fetch_blob_files("image", [row_id])[0] as handle: + streamed = handle.read() + + assert fetched == [payload] + assert streamed == payload + + +def _marked(size, marker): + payload = bytearray(size) + payload[0] = marker + payload[-1] = marker + return bytes(payload) + + +def test_storage_tiers_round_trip(tmp_path): + cases = [ + (1, _marked(INLINE_MAX, 11), 0), + (2, _marked(INLINE_MAX + 1, 22), 1), + (3, _marked(DEDICATED_MIN, 33), 1), + (4, _marked(DEDICATED_MIN + 1, 44), 2), + ] + db = lancedb.connect(tmp_path) + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + tbl = db.create_table( + "tiers", + data=[{"id": row_id, "image": payload} for row_id, payload, _kind in cases], + schema=schema, + ) + + described = { + row["id"]: row["image"] + for row in tbl.search() + .select(["id", "image"]) + .to_list(blob_mode="descriptions") + } + located = { + row["id"]: row["_rowid"] + for row in tbl.search().select(["id"]).with_row_id(True).to_list() + } + fetched = { + row_id: tbl.fetch_blobs("image", [located[row_id]]).to_pylist()[0] + for row_id, _payload, _kind in cases + } + + for row_id, payload, kind in cases: + descriptor = described[row_id] + assert descriptor["kind"] == kind + assert descriptor["size"] == len(payload) + assert fetched[row_id] == payload + with tbl.fetch_blob_files("image", [located[row_id]])[0] as handle: + assert handle.read_range(0, 1) == payload[:1] + assert handle.read_range(len(payload) - 1, 1) == payload[-1:] diff --git a/docs/web-tests/blob-v2/rs/Cargo.toml b/docs/web-tests/blob-v2/rs/Cargo.toml new file mode 100644 index 000000000..f96fde230 --- /dev/null +++ b/docs/web-tests/blob-v2/rs/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "lancedb-blob-v2-docs" +version = "0.0.0" +edition.workspace = true +publish = false +license.workspace = true + +[features] +remote = ["lancedb/remote"] + +[dependencies] +arrow-array = { workspace = true } +arrow-schema = { workspace = true } +futures-util = "0.3" +lancedb = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } + +[[example]] +name = "multimodal" +path = "multimodal.rs" diff --git a/docs/web-tests/blob-v2/rs/multimodal.rs b/docs/web-tests/blob-v2/rs/multimodal.rs new file mode 100644 index 000000000..257bc01c1 --- /dev/null +++ b/docs/web-tests/blob-v2/rs/multimodal.rs @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:blob_api_imports] +use std::sync::Arc; + +use arrow_array::{ + Array, Int64Array, LargeBinaryArray, RecordBatch, RecordBatchIterator, StringArray, UInt64Array, +}; +use arrow_schema::{DataType, Field, Schema}; +use futures_util::TryStreamExt; +use lancedb::blob::{BlobRangeRequest, blob}; +use lancedb::connect; +use lancedb::database::CreateTableMode; +use lancedb::query::{ExecutableQuery, QueryBase}; +// --8<-- [end:blob_api_imports] + +use arrow_array::{StructArray, UInt8Array}; + +const INLINE_MAX: usize = 64 * 1024; +const DEDICATED_MIN: usize = 4 * 1024 * 1024; + +#[tokio::main] +async fn main() { + tutorial().await; + deferred_fetch().await; + storage_tiers().await; +} + +async fn tutorial() { + let temp_dir = tempfile::tempdir().unwrap(); + let original_dir = std::env::current_dir().unwrap(); + std::env::set_current_dir(temp_dir.path()).unwrap(); + + // --8<-- [start:blob_api_setup] + let db = connect("./blob_db").execute().await.unwrap(); + // --8<-- [end:blob_api_setup] + + // --8<-- [start:blob_api_schema] + let blob_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("label", DataType::Utf8, true), + blob("image", true), + ])); + // --8<-- [end:blob_api_schema] + + // --8<-- [start:blob_api_ingest] + let blob_table = db + .create_empty_table("photos", blob_schema) + .mode(CreateTableMode::Overwrite) + .execute() + .await + .unwrap(); + + let input_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("label", DataType::Utf8, true), + Field::new("image", DataType::LargeBinary, true), + ])); + let input_batch = RecordBatch::try_new( + input_schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1_i64, 2_i64])), + Arc::new(StringArray::from(vec![Some("door"), Some("window")])), + Arc::new(LargeBinaryArray::from_iter_values([ + b"inline_image_bytes".as_slice(), + b"second_image_bytes".as_slice(), + ])), + ], + ) + .unwrap(); + let input_reader: Box = Box::new( + RecordBatchIterator::new(vec![Ok(input_batch)].into_iter(), input_schema), + ); + blob_table.add(input_reader).execute().await.unwrap(); + // --8<-- [end:blob_api_ingest] + assert_eq!(blob_table.count_rows(None).await.unwrap(), 2); + + // --8<-- [start:blob_api_files] + let batches = blob_table + .query() + .with_row_id() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let row_ids = batches[0] + .column_by_name("_rowid") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec(); + + let handles = blob_table + .fetch_blob_files("image", &row_ids[..1]) + .await + .unwrap(); + let header = handles[0].as_ref().unwrap().read_range(0..6).await.unwrap(); + handles[0].as_ref().unwrap().close().await.unwrap(); + // --8<-- [end:blob_api_files] + assert_eq!(header.as_ref(), b"inline"); + + // --8<-- [start:blob_api_fetch] + let payloads = blob_table.fetch_blobs("image", &row_ids).await.unwrap(); + // --8<-- [end:blob_api_fetch] + assert_eq!(payloads.len(), 2); + + // --8<-- [start:blob_api_ranges] + let slices = blob_table + .fetch_blob_ranges( + "image", + [ + BlobRangeRequest::new(row_ids[0], 0, 6), + BlobRangeRequest::new(row_ids[1], 0, 6), + BlobRangeRequest::new(row_ids[1], 7, 5), + ], + ) + .await + .unwrap(); + + let second_handles = blob_table + .fetch_blob_files("image", &row_ids[1..2]) + .await + .unwrap(); + let second_image = second_handles[0].as_ref().unwrap(); + let parts = second_image.read_ranges(&[0..6, 7..12]).await.unwrap(); + second_image.close().await.unwrap(); + // --8<-- [end:blob_api_ranges] + assert_eq!(slices.value(0), b"inline"); + assert_eq!(slices.value(1), b"second"); + assert_eq!(slices.value(2), b"image"); + assert_eq!(parts[0].as_ref(), b"second"); + assert_eq!(parts[1].as_ref(), b"image"); + + std::env::set_current_dir(original_dir).unwrap(); +} + +async fn deferred_fetch() { + let temp_dir = tempfile::tempdir().unwrap(); + let db = connect(temp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + blob("image", true), + ])); + let table = db + .create_empty_table("photos", schema) + .execute() + .await + .unwrap(); + let input_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("image", DataType::LargeBinary, true), + ])); + let batch = RecordBatch::try_new( + input_schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1_i64])), + Arc::new(LargeBinaryArray::from_iter_values([ + b"deferred-bytes".as_slice() + ])), + ], + ) + .unwrap(); + let reader: Box = Box::new( + RecordBatchIterator::new(vec![Ok(batch)].into_iter(), input_schema), + ); + table.add(reader).execute().await.unwrap(); + + let described = table + .query() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(descriptor_kind(&described[0], 0), 0); + assert!(described[0].column_by_name("_rowid").is_none()); + + let located = table + .query() + .with_row_id() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let row_id = located[0] + .column_by_name("_rowid") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + let payloads = table.fetch_blobs("image", &[row_id]).await.unwrap(); + let handles = table.fetch_blob_files("image", &[row_id]).await.unwrap(); + let streamed = handles[0] + .as_ref() + .unwrap() + .read_range(0..payloads.value(0).len() as u64) + .await + .unwrap(); + handles[0].as_ref().unwrap().close().await.unwrap(); + + assert_eq!(payloads.value(0), b"deferred-bytes"); + assert_eq!(streamed.as_ref(), b"deferred-bytes"); +} + +async fn storage_tiers() { + let cases = [ + (1_i64, marked(INLINE_MAX, 11), 0_u8), + (2_i64, marked(INLINE_MAX + 1, 22), 1_u8), + (3_i64, marked(DEDICATED_MIN, 33), 1_u8), + (4_i64, marked(DEDICATED_MIN + 1, 44), 2_u8), + ]; + let temp_dir = tempfile::tempdir().unwrap(); + let db = connect(temp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + blob("image", true), + ])); + let table = db + .create_empty_table("tiers", schema) + .execute() + .await + .unwrap(); + let input_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("image", DataType::LargeBinary, true), + ])); + let batch = RecordBatch::try_new( + input_schema.clone(), + vec![ + Arc::new(Int64Array::from_iter_values(cases.iter().map(|row| row.0))), + Arc::new(LargeBinaryArray::from_iter_values( + cases.iter().map(|row| row.1.as_slice()), + )), + ], + ) + .unwrap(); + let reader: Box = Box::new( + RecordBatchIterator::new(vec![Ok(batch)].into_iter(), input_schema), + ); + table.add(reader).execute().await.unwrap(); + + let described = table + .query() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let located = table + .query() + .with_row_id() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let mut kind_by_id = std::collections::HashMap::new(); + for batch in &described { + let ids = batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for row in 0..batch.num_rows() { + kind_by_id.insert(ids.value(row), descriptor_kind(batch, row)); + } + } + let mut row_id_by_id = std::collections::HashMap::new(); + for batch in &located { + let ids = batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let row_ids = batch + .column_by_name("_rowid") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for row in 0..batch.num_rows() { + row_id_by_id.insert(ids.value(row), row_ids.value(row)); + } + } + + for (id, payload, kind) in &cases { + assert_eq!(kind_by_id[id], *kind); + let row_id = row_id_by_id[id]; + let fetched = table.fetch_blobs("image", &[row_id]).await.unwrap(); + assert_eq!(fetched.value(0), payload.as_slice()); + let handles = table.fetch_blob_files("image", &[row_id]).await.unwrap(); + let handle = handles[0].as_ref().unwrap(); + let head = handle.read_range(0..1).await.unwrap(); + let tail = handle + .read_range((payload.len() - 1) as u64..payload.len() as u64) + .await + .unwrap(); + handle.close().await.unwrap(); + assert_eq!(head.as_ref(), &payload[..1]); + assert_eq!(tail.as_ref(), &payload[payload.len() - 1..]); + } +} + +fn marked(size: usize, marker: u8) -> Vec { + let mut payload = vec![0_u8; size]; + payload[0] = marker; + payload[size - 1] = marker; + payload +} + +fn descriptor_kind(batch: &RecordBatch, row: usize) -> u8 { + let image = batch + .column_by_name("image") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap_or_else(|| panic!("image column was {}", batch.schema())); + image + .column_by_name("kind") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(row) +} diff --git a/docs/web-tests/blob-v2/ts/multimodal.test.ts b/docs/web-tests/blob-v2/ts/multimodal.test.ts new file mode 100644 index 000000000..b253ee9b2 --- /dev/null +++ b/docs/web-tests/blob-v2/ts/multimodal.test.ts @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { chdir, cwd } from "node:process"; +import { test } from "node:test"; +import * as arrow from "apache-arrow"; +import { Buffer } from "node:buffer"; +import * as lancedb from "@lancedb/lancedb"; + +const INLINE_MAX = 64 * 1024; +const DEDICATED_MIN = 4 * 1024 * 1024; + +function marked(size: number, marker: number): Buffer { + const payload = Buffer.alloc(size); + payload[0] = marker; + payload[payload.length - 1] = marker; + return payload; +} + +function kindOf(value: unknown): number { + assert.equal(typeof value, "object"); + assert.ok(value !== null); + const kind = (value as { kind?: unknown }).kind; + assert.equal(typeof kind, "number"); + return kind; +} + +test("blob column tutorial", async () => { + const directory = mkdtempSync(join(tmpdir(), "blob-v2-")); + const previous = cwd(); + chdir(directory); + try { + // --8<-- [start:blob_api_setup] + const db = await lancedb.connect("./blob_db"); + // --8<-- [end:blob_api_setup] + + // --8<-- [start:blob_api_schema] + const blobSchema = new arrow.Schema([ + new arrow.Field("id", new arrow.Int64()), + new arrow.Field("label", new arrow.Utf8()), + lancedb.blob("image"), + ]); + // --8<-- [end:blob_api_schema] + + // --8<-- [start:blob_api_ingest] + const blobTable = await db.createEmptyTable("photos", blobSchema, { + mode: "overwrite", + }); + + await blobTable.add([ + { id: 1n, label: "door", image: Buffer.from("inline_image_bytes") }, + { id: 2n, label: "window", image: Buffer.from("second_image_bytes") }, + ]); + // --8<-- [end:blob_api_ingest] + assert.equal(await blobTable.countRows(), 2); + + // --8<-- [start:blob_api_files] + const rows = await blobTable + .query() + .select(["id", "label"]) + .withRowId() + .toArray(); + const rowIds = rows.map((row) => row._rowid as bigint); + + const [image] = await blobTable.fetchBlobFiles("image", [rowIds[0]!]); + const header = await image!.readRange(0n, 6n); + await image!.close(); + // --8<-- [end:blob_api_files] + assert.equal(header.toString("utf8"), "inline"); + + // --8<-- [start:blob_api_fetch] + const payloads = await blobTable.fetchBlobs("image", rowIds); + // --8<-- [end:blob_api_fetch] + assert.equal(payloads.length, 2); + + // --8<-- [start:blob_api_ranges] + const [secondImage] = await blobTable.fetchBlobFiles("image", [rowIds[1]!]); + const parts = await secondImage!.readRanges([ + { start: 0n, end: 6n }, + { start: 7n, end: 12n }, + ]); + await secondImage!.close(); + // --8<-- [end:blob_api_ranges] + assert.deepEqual( + parts.map((part) => part.toString("utf8")), + ["second", "image"], + ); + + // --8<-- [start:blob_api_bytes_mode] + const withBytes = await blobTable + .query() + .select(["id", "image"]) + .toArray({ blobMode: "bytes" }); + const firstImage: Uint8Array | null = withBytes[0].image; + // --8<-- [end:blob_api_bytes_mode] + assert.equal(Buffer.from(firstImage!).toString("utf8"), "inline_image_bytes"); + assert.equal(Object.hasOwn(withBytes[0], "_rowid"), false); + } finally { + chdir(previous); + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("eager query returns blob bytes without a row id", async () => { + const directory = mkdtempSync(join(tmpdir(), "blob-v2-eager-")); + try { + const db = await lancedb.connect(directory); + const schema = new arrow.Schema([ + new arrow.Field("id", new arrow.Int64()), + lancedb.blob("image"), + ]); + const table = await db.createEmptyTable("photos", schema); + const payload = Buffer.from("inline-bytes"); + await table.add([{ id: 1n, image: payload }]); + + const rows = await table + .query() + .where("id = 1") + .select(["id", "image"]) + .toArray({ blobMode: "bytes" }); + + assert.equal(rows.length, 1); + assert.equal(Object.hasOwn(rows[0], "_rowid"), false); + assert.equal(Buffer.from(rows[0].image).toString("utf8"), "inline-bytes"); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("deferred fetch reads a blob by row id", async () => { + const directory = mkdtempSync(join(tmpdir(), "blob-v2-deferred-")); + try { + const db = await lancedb.connect(directory); + const schema = new arrow.Schema([ + new arrow.Field("id", new arrow.Int64()), + lancedb.blob("image"), + ]); + const table = await db.createEmptyTable("photos", schema); + await table.add([{ id: 1n, image: Buffer.from("deferred-bytes") }]); + + const described = await table + .query() + .where("id = 1") + .select(["id", "image"]) + .toArray(); + assert.equal(kindOf(described[0].image), 0); + assert.equal(Object.hasOwn(described[0], "_rowid"), false); + + const found = await table.query().where("id = 1").select(["id"]).withRowId().toArray(); + const rowId = found[0]._rowid as bigint; + const payloads = await table.fetchBlobs("image", [rowId]); + const [handle] = await table.fetchBlobFiles("image", [rowId]); + const streamed = await handle!.readRange(0n, handle!.size()); + await handle!.close(); + + assert.equal(Buffer.from(payloads[0]).toString("utf8"), "deferred-bytes"); + assert.equal(Buffer.from(streamed).toString("utf8"), "deferred-bytes"); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("storage tiers round-trip", async () => { + const cases = [ + { id: 1n, image: marked(INLINE_MAX, 11), kind: 0 }, + { id: 2n, image: marked(INLINE_MAX + 1, 22), kind: 1 }, + { id: 3n, image: marked(DEDICATED_MIN, 33), kind: 1 }, + { id: 4n, image: marked(DEDICATED_MIN + 1, 44), kind: 2 }, + ]; + const directory = mkdtempSync(join(tmpdir(), "blob-v2-tiers-")); + try { + const db = await lancedb.connect(directory); + const schema = new arrow.Schema([ + new arrow.Field("id", new arrow.Int64()), + lancedb.blob("image"), + ]); + const table = await db.createEmptyTable("tiers", schema); + await table.add(cases.map(({ id, image }) => ({ id, image }))); + + const described = await table.query().select(["id", "image"]).toArray(); + const located = await table.query().select(["id"]).withRowId().toArray(); + const rowIdByValue = new Map(located.map((row) => [row.id as bigint, row._rowid as bigint])); + + for (const item of cases) { + const descriptor = described.find((row) => row.id === item.id); + assert.ok(descriptor); + assert.equal(kindOf(descriptor.image), item.kind); + const rowId = rowIdByValue.get(item.id); + assert.ok(rowId !== undefined); + const [payload] = await table.fetchBlobs("image", [rowId]); + assert.deepEqual(Buffer.from(payload), item.image); + const [handle] = await table.fetchBlobFiles("image", [rowId]); + const streamed = await handle!.read(); + await handle!.close(); + assert.deepEqual(streamed, item.image); + } + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/docs/web-tests/blob-v2/ts/package-lock.json b/docs/web-tests/blob-v2/ts/package-lock.json new file mode 100644 index 000000000..9426f2b18 --- /dev/null +++ b/docs/web-tests/blob-v2/ts/package-lock.json @@ -0,0 +1,1565 @@ +{ + "name": "lancedb-blob-v2-docs", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "lancedb-blob-v2-docs", + "version": "0.0.0", + "dependencies": { + "@lancedb/lancedb": "0.41.0-beta.8", + "apache-arrow": "21.2.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@huggingface/jinja": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.3.4.tgz", + "integrity": "sha512-kFFQWJiWwvxezKQnvH1X7GjsECcMljFx+UZK9hx6P26aVHwwidJVTB0ptLfRVZQvVkOGHoMmTGvo4nT0X9hHOA==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/@huggingface/transformers": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-3.0.2.tgz", + "integrity": "sha512-lTyS81eQazMea5UCehDGFMfdcNRZyei7XQLH5X6j4AhA/18Ka0+5qPgMxUxuZLU4xkv60aY2KNz9Yzthv6WVJg==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@huggingface/jinja": "^0.3.0", + "onnxruntime-node": "1.19.2", + "onnxruntime-web": "1.21.0-dev.20241024-d9ca84ef96", + "sharp": "^0.33.5" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz", + "integrity": "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz", + "integrity": "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz", + "integrity": "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz", + "integrity": "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz", + "integrity": "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz", + "integrity": "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz", + "integrity": "sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz", + "integrity": "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz", + "integrity": "sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz", + "integrity": "sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz", + "integrity": "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.0.5" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz", + "integrity": "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz", + "integrity": "sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz", + "integrity": "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz", + "integrity": "sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz", + "integrity": "sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz", + "integrity": "sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.2.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz", + "integrity": "sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz", + "integrity": "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "license": "ISC", + "optional": true, + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@lancedb/lancedb": { + "version": "0.41.0-beta.8", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb/-/lancedb-0.41.0-beta.8.tgz", + "integrity": "sha512-W6Sw1rJgJJAT75jXom2fkNvE976iH8EO74ShytymS6sEzCj63gQXQqCYg+jPvQva/dnvYQiOeOGsdkGDFwKmlA==", + "cpu": [ + "x64", + "arm64" + ], + "license": "Apache-2.0", + "os": [ + "darwin", + "linux", + "win32" + ], + "dependencies": { + "@opentelemetry/api": "^1.9.0", + "reflect-metadata": "^0.2.2" + }, + "engines": { + "node": ">= 22" + }, + "optionalDependencies": { + "@huggingface/transformers": "3.0.2", + "@lancedb/lancedb-darwin-arm64": "0.41.0-beta.8", + "@lancedb/lancedb-linux-arm64-gnu": "0.41.0-beta.8", + "@lancedb/lancedb-linux-arm64-musl": "0.41.0-beta.8", + "@lancedb/lancedb-linux-x64-gnu": "0.41.0-beta.8", + "@lancedb/lancedb-linux-x64-musl": "0.41.0-beta.8", + "@lancedb/lancedb-win32-arm64-msvc": "0.41.0-beta.8", + "@lancedb/lancedb-win32-x64-msvc": "0.41.0-beta.8", + "openai": "4.29.2" + }, + "peerDependencies": { + "@types/node": ">=22", + "apache-arrow": ">=15.0.0 <22.0.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@lancedb/lancedb-darwin-arm64": { + "version": "0.41.0-beta.8", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-darwin-arm64/-/lancedb-darwin-arm64-0.41.0-beta.8.tgz", + "integrity": "sha512-M1o7VQmB1th9L8YnOqXlxruB1jW6Gns9rTS5H5rCzjv25CTEq/mw9uwl6jrDlLiuVSLWJ89Jj2EDUMH8ZKkEbw==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 18" + } + }, + "node_modules/@lancedb/lancedb-linux-arm64-gnu": { + "version": "0.41.0-beta.8", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-arm64-gnu/-/lancedb-linux-arm64-gnu-0.41.0-beta.8.tgz", + "integrity": "sha512-X5lJ2JWMHIrvq/IkkHPjQ3v6myv2jZKK/eayiLe/8Wasxt6LIm9Z7KJYLtXpp5y+kqncAmMRP+BaAJAY02FgvQ==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 18" + } + }, + "node_modules/@lancedb/lancedb-linux-arm64-musl": { + "version": "0.41.0-beta.8", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-arm64-musl/-/lancedb-linux-arm64-musl-0.41.0-beta.8.tgz", + "integrity": "sha512-9reTx4gKOIZlx1hNp3s+O2tMv56aF0bp+kt22+XniwL6xLhVyMlyp+cRnhReVqbvu3Ykqe1lZEQO8U3EXuxSFg==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 18" + } + }, + "node_modules/@lancedb/lancedb-linux-x64-gnu": { + "version": "0.41.0-beta.8", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-x64-gnu/-/lancedb-linux-x64-gnu-0.41.0-beta.8.tgz", + "integrity": "sha512-tu9RvxpNNVeUOIWrjmsZgufP+ebBvNES0x4D4qHpPsUJXwdR/LJ4QjON57NvuyQ9CZTUL7Rdk8xbzg+nvlmAaQ==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 18" + } + }, + "node_modules/@lancedb/lancedb-linux-x64-musl": { + "version": "0.41.0-beta.8", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-x64-musl/-/lancedb-linux-x64-musl-0.41.0-beta.8.tgz", + "integrity": "sha512-EFMuG0GaFO0BPNfY/XRo66nDKKq3ndyiq/t+P4P500cagDtFgCpZphhF8Q+BkQpQ1b+ZgVS2Uaxu5tOyo0SPhg==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 18" + } + }, + "node_modules/@lancedb/lancedb-win32-arm64-msvc": { + "version": "0.41.0-beta.8", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-win32-arm64-msvc/-/lancedb-win32-arm64-msvc-0.41.0-beta.8.tgz", + "integrity": "sha512-+vKZHO0CzZgoqqu9izXP2Ea0fBQ/PtolNz3B9JZItR5nc2w12bknA57K1sS06H74BBU9Yie2hDHNt3qzdPBEdw==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 18" + } + }, + "node_modules/@lancedb/lancedb-win32-x64-msvc": { + "version": "0.41.0-beta.8", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-win32-x64-msvc/-/lancedb-win32-x64-msvc-0.41.0-beta.8.tgz", + "integrity": "sha512-qSgWdgh9TzUiurd658ynzD4BxI/6E9yih7HFuFxvi8S+Zcp/NNiDpgrMWLQn/KfvorVJg9XBO8nhZMtg34UTGw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 18" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause", + "optional": true + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause", + "optional": true + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause", + "optional": true + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause", + "optional": true + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "optional": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause", + "optional": true + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause", + "optional": true + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause", + "optional": true + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause", + "optional": true + }, + "node_modules/@types/node": { + "version": "25.9.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.9.tgz", + "integrity": "sha512-b4e2xxj/yMeT2hNlD6x7zNFiQK2Dlmkb9CQ6v3/1G2RezKN4YS2lcEv8lzGm1mhs4f2lM4yNAMI/zUDBZBBt7w==", + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "node_modules/@types/node-fetch": { + "version": "2.6.13", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz", + "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", + "license": "MIT", + "optional": true, + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.4" + } + }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "optional": true, + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/agentkeepalive": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", + "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/apache-arrow": { + "version": "21.2.0", + "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-21.2.0.tgz", + "integrity": "sha512-Hxe6Agq26gQOM954qpzYSllJBPJl+e16U5CkfuMUhLrNba+5nKkttIVlflaovN6oaTratqMGAO8H5u/aNhmHWQ==", + "license": "Apache-2.0", + "dependencies": { + "@types/node": "^25.2.0", + "flatbuffers": "^25.1.24", + "json-with-bigint": "^3.5.3", + "tslib": "^2.6.2" + }, + "bin": { + "arrow2csv": "bin/arrow2csv.js" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT", + "optional": true + }, + "node_modules/base-64": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz", + "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==", + "optional": true + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", + "license": "BSD-3-Clause", + "optional": true, + "engines": { + "node": "*" + } + }, + "node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "license": "BlueOak-1.0.0", + "optional": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "license": "MIT", + "optional": true, + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT", + "optional": true + }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "license": "MIT", + "optional": true, + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "optional": true, + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", + "license": "BSD-3-Clause", + "optional": true, + "engines": { + "node": "*" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/digest-fetch": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz", + "integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==", + "license": "ISC", + "optional": true, + "dependencies": { + "base-64": "^0.1.0", + "md5": "^2.3.0" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "optional": true, + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "optional": true, + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "optional": true, + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0" + }, + "node_modules/form-data": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", + "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.4", + "mime-types": "^2.1.35" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", + "license": "MIT", + "optional": true + }, + "node_modules/formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "engines": { + "node": ">= 12.20" + } + }, + "node_modules/formdata-node/node_modules/web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 14" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "optional": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "optional": true, + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", + "license": "ISC", + "optional": true + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "optional": true, + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "optional": true, + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "ms": "^2.0.0" + } + }, + "node_modules/is-arrayish": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.4.tgz", + "integrity": "sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA==", + "license": "MIT", + "optional": true + }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "license": "MIT", + "optional": true + }, + "node_modules/json-with-bigint": { + "version": "3.5.12", + "resolved": "https://registry.npmjs.org/json-with-bigint/-/json-with-bigint-3.5.12.tgz", + "integrity": "sha512-uwbF/wSSuOgC7qqlq27Xp5B6a2MHVug3t0idZdTqu0JnlFvgJuH7ju+KAk/J06C7GfhoYy2gnb9wz2INqcne7w==", + "license": "MIT" + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "license": "BSD-3-Clause", + "optional": true, + "dependencies": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "optional": true, + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "optional": true, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "license": "MIT", + "optional": true, + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT", + "optional": true + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "optional": true, + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "optional": true, + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/onnxruntime-common": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.19.2.tgz", + "integrity": "sha512-a4R7wYEVFbZBlp0BfhpbFWqe4opCor3KM+5Wm22Az3NGDcQMiU2hfG/0MfnBs+1ZrlSGmlgWeMcXQkDk1UFb8Q==", + "license": "MIT", + "optional": true + }, + "node_modules/onnxruntime-node": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.19.2.tgz", + "integrity": "sha512-9eHMP/HKbbeUcqte1JYzaaRC8JPn7ojWeCeoyShO86TOR97OCyIyAIOGX3V95ErjslVhJRXY8Em/caIUc0hm1Q==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "onnxruntime-common": "1.19.2", + "tar": "^7.0.1" + } + }, + "node_modules/onnxruntime-web": { + "version": "1.21.0-dev.20241024-d9ca84ef96", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.21.0-dev.20241024-d9ca84ef96.tgz", + "integrity": "sha512-ANSQfMALvCviN3Y4tvTViKofKToV1WUb2r2VjZVCi3uUBPaK15oNJyIxhsNyEckBr/Num3JmSXlkHOD8HfVzSQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "flatbuffers": "^1.12.0", + "guid-typescript": "^1.0.9", + "long": "^5.2.3", + "onnxruntime-common": "1.20.0-dev.20241016-2b8fc5529b", + "platform": "^1.3.6", + "protobufjs": "^7.2.4" + } + }, + "node_modules/onnxruntime-web/node_modules/flatbuffers": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-1.12.0.tgz", + "integrity": "sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==", + "license": "SEE LICENSE IN LICENSE.txt", + "optional": true + }, + "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { + "version": "1.20.0-dev.20241016-2b8fc5529b", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.20.0-dev.20241016-2b8fc5529b.tgz", + "integrity": "sha512-KZK8b6zCYGZFjd4ANze0pqBnqnFTS3GIVeclQpa2qseDpXrCQJfkWBixRcrZShNhm3LpFOZ8qJYFC5/qsJK9WQ==", + "license": "MIT", + "optional": true + }, + "node_modules/openai": { + "version": "4.29.2", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.29.2.tgz", + "integrity": "sha512-cPkT6zjEcE4qU5OW/SoDDuXEsdOLrXlAORhzmaguj5xZSPlgKvLhi27sFWhLKj07Y6WKNWxcwIbzm512FzTBNQ==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "digest-fetch": "^1.3.0", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7", + "web-streams-polyfill": "^3.2.1" + }, + "bin": { + "openai": "bin/cli" + } + }, + "node_modules/openai/node_modules/@types/node": { + "version": "18.19.130", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", + "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", + "license": "MIT", + "optional": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/openai/node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "license": "MIT", + "optional": true + }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT", + "optional": true + }, + "node_modules/protobufjs": { + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "optional": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/reflect-metadata": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", + "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==", + "license": "Apache-2.0" + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "optional": true, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz", + "integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==", + "hasInstallScript": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.3", + "semver": "^7.6.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.33.5", + "@img/sharp-darwin-x64": "0.33.5", + "@img/sharp-libvips-darwin-arm64": "1.0.4", + "@img/sharp-libvips-darwin-x64": "1.0.4", + "@img/sharp-libvips-linux-arm": "1.0.5", + "@img/sharp-libvips-linux-arm64": "1.0.4", + "@img/sharp-libvips-linux-s390x": "1.0.4", + "@img/sharp-libvips-linux-x64": "1.0.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", + "@img/sharp-libvips-linuxmusl-x64": "1.0.4", + "@img/sharp-linux-arm": "0.33.5", + "@img/sharp-linux-arm64": "0.33.5", + "@img/sharp-linux-s390x": "0.33.5", + "@img/sharp-linux-x64": "0.33.5", + "@img/sharp-linuxmusl-arm64": "0.33.5", + "@img/sharp-linuxmusl-x64": "0.33.5", + "@img/sharp-wasm32": "0.33.5", + "@img/sharp-win32-ia32": "0.33.5", + "@img/sharp-win32-x64": "0.33.5" + } + }, + "node_modules/simple-swizzle": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.4.tgz", + "integrity": "sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw==", + "license": "MIT", + "optional": true, + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, + "node_modules/tar": { + "version": "7.5.22", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.22.tgz", + "integrity": "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==", + "license": "BlueOak-1.0.0", + "optional": true, + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT", + "optional": true + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "license": "MIT" + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause", + "optional": true + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "optional": true, + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "license": "BlueOak-1.0.0", + "optional": true, + "engines": { + "node": ">=18" + } + } + } +} diff --git a/docs/web-tests/blob-v2/ts/package.json b/docs/web-tests/blob-v2/ts/package.json new file mode 100644 index 000000000..f39495edd --- /dev/null +++ b/docs/web-tests/blob-v2/ts/package.json @@ -0,0 +1,16 @@ +{ + "name": "lancedb-blob-v2-docs", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "test": "node --experimental-strip-types --test multimodal.test.ts" + }, + "engines": { + "node": ">=22.6.0" + }, + "dependencies": { + "@lancedb/lancedb": "0.41.0-beta.8", + "apache-arrow": "21.2.0" + } +} diff --git a/docs/web-tests/py/test_multimodal.py b/docs/web-tests/py/test_multimodal.py index cb4a647c3..dbce74ca8 100644 --- a/docs/web-tests/py/test_multimodal.py +++ b/docs/web-tests/py/test_multimodal.py @@ -1,137 +1,134 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The LanceDB Authors +import io import pytest +import lancedb +import numpy as np +import pyarrow as pa + try: + # --8<-- [start:multimodal_imports] + import io import lancedb import numpy as np import pyarrow as pa - import io from PIL import Image + # --8<-- [end:multimodal_imports] except ImportError: pass -# --8<-- [start:multimodal_imports] -import lancedb -import pyarrow as pa -import pandas as pd -import numpy as np -import io -from PIL import Image -# --8<-- [end:multimodal_imports] def test_multimodal_ingestion(db_path_factory): - # Ensure dependencies are available pytest.importorskip("PIL") - pytest.importorskip("lancedb") - pytest.importorskip("numpy") # --8<-- [start:create_dummy_data] - # Create some dummy images def create_dummy_image(color): - img = Image.new('RGB', (100, 100), color=color) + img = Image.new("RGB", (100, 100), color=color) buf = io.BytesIO() - img.save(buf, format='PNG') + img.save(buf, format="PNG") return buf.getvalue() - # Create dataset with metadata, vectors, and image blobs data = [ { "id": 1, "filename": "red_square.png", "vector": np.random.rand(128).astype(np.float32), - "image_blob": create_dummy_image('red'), - "label": "red" + "image_blob": create_dummy_image("red"), + "label": "red", }, { "id": 2, "filename": "blue_square.png", "vector": np.random.rand(128).astype(np.float32), - "image_blob": create_dummy_image('blue'), - "label": "blue" - } + "image_blob": create_dummy_image("blue"), + "label": "blue", + }, ] # --8<-- [end:create_dummy_data] # --8<-- [start:define_schema] - # Define schema explicitly to ensure image_blob is treated as binary - schema = pa.schema([ - pa.field("id", pa.int32()), - pa.field("filename", pa.string()), - pa.field("vector", pa.list_(pa.float32(), 128)), - pa.field("image_blob", pa.binary()), # Important: Use pa.binary() for blobs - pa.field("label", pa.string()) - ]) + schema = pa.schema( + [ + pa.field("id", pa.int32()), + pa.field("filename", pa.string()), + pa.field("vector", pa.list_(pa.float32(), 128)), + pa.field("image_blob", pa.binary()), + pa.field("label", pa.string()), + ] + ) # --8<-- [end:define_schema] - db_uri = db_path_factory("multimodal_db") - db = lancedb.connect(db_uri) + db = lancedb.connect(db_path_factory("multimodal_db")) # --8<-- [start:ingest_data] tbl = db.create_table("images", data=data, schema=schema, mode="overwrite") # --8<-- [end:ingest_data] - + assert len(tbl) == 2 # --8<-- [start:search_data] - # Search for similar images query_vector = np.random.rand(128).astype(np.float32) results = tbl.search(query_vector).limit(1).to_pandas() # --8<-- [end:search_data] # --8<-- [start:process_results] - # Convert back to PIL Image for _, row in results.iterrows(): - image_bytes = row['image_blob'] + image_bytes = row["image_blob"] image = Image.open(io.BytesIO(image_bytes)) print(f"Retrieved image: {row['filename']}, Size: {image.size}") - # You can now use 'image' with other libraries or display it # --8<-- [end:process_results] - + assert len(results) == 1 -def test_blob_api_definition(db_path_factory): - # --8<-- [start:blob_api_schema] + +def test_legacy_blob_api_definition(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + # --8<-- [start:legacy_blob_api_schema] import pyarrow as pa # Define schema with Blob API metadata for lazy loading - schema = pa.schema([ - pa.field("id", pa.int64()), - pa.field( - "video", - pa.large_binary(), - metadata={"lance-encoding:blob": "true"} # Enable Blob API - ), - ]) - # --8<-- [end:blob_api_schema] + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field( + "video", + pa.large_binary(), + metadata={"lance-encoding:blob": "true"}, # Enable Blob API + ), + ] + ) + # --8<-- [end:legacy_blob_api_schema] - # --8<-- [start:blob_api_ingest] + # --8<-- [start:legacy_blob_api_ingest] import lancedb - db = lancedb.connect(db_path_factory("blob_db")) - + db = lancedb.connect("./blob_db") + # Create sample data data = [ {"id": 1, "video": b"fake_video_bytes_1"}, - {"id": 2, "video": b"fake_video_bytes_2"} + {"id": 2, "video": b"fake_video_bytes_2"}, ] - + # Create the table tbl = db.create_table("videos", data=data, schema=schema) - # --8<-- [end:blob_api_ingest] + # --8<-- [end:legacy_blob_api_ingest] assert len(tbl) == 2 -def test_blob_api_to_pandas(db_path_factory): - db = lancedb.connect(db_path_factory("blob_to_pandas_db")) - schema = pa.schema([ - pa.field("id", pa.int64()), - pa.field( - "video", - pa.large_binary(), - metadata={"lance-encoding:blob": "true"}, - ), - ]) +def test_legacy_blob_api_to_pandas(db_path_factory): + db = lancedb.connect(db_path_factory("legacy_blob_to_pandas_db")) + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field( + "video", + pa.large_binary(), + metadata={"lance-encoding:blob": "true"}, + ), + ] + ) tbl = db.create_table( "videos", data=[ @@ -142,7 +139,7 @@ def test_blob_api_to_pandas(db_path_factory): mode="overwrite", ) - # --8<-- [start:blob_api_to_pandas] + # --8<-- [start:legacy_blob_api_to_pandas] # Default: blob columns come back lazily df_lazy = tbl.to_pandas() @@ -154,7 +151,7 @@ def test_blob_api_to_pandas(db_path_factory): # Forward extra kwargs to PyArrow's to_pandas df_typed = tbl.to_pandas(split_blocks=True, self_destruct=True) - # --8<-- [end:blob_api_to_pandas] + # --8<-- [end:legacy_blob_api_to_pandas] assert len(df_lazy) == 2 assert isinstance(df_bytes["video"].iloc[0], bytes) @@ -167,16 +164,18 @@ def test_blob_api_to_pandas(db_path_factory): @pytest.mark.asyncio -async def test_query_to_pandas_kwargs(db_path_factory): - schema = pa.schema([ - pa.field("id", pa.int64()), - pa.field("vector", pa.list_(pa.float32(), 128)), - pa.field( - "video", - pa.large_binary(), - metadata={"lance-encoding:blob": "true"}, - ), - ]) +async def test_legacy_query_to_pandas_kwargs(db_path_factory): + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("vector", pa.list_(pa.float32(), 128)), + pa.field( + "video", + pa.large_binary(), + metadata={"lance-encoding:blob": "true"}, + ), + ] + ) data = [ { "id": i, @@ -186,11 +185,11 @@ async def test_query_to_pandas_kwargs(db_path_factory): for i in range(10) ] - db = lancedb.connect(db_path_factory("query_to_pandas_db")) + db = lancedb.connect(db_path_factory("legacy_query_to_pandas_db")) tbl = db.create_table("search_demo", data=data, schema=schema, mode="overwrite") async_db = await lancedb.connect_async( - str(db_path_factory("query_to_pandas_async_db")) + str(db_path_factory("legacy_query_to_pandas_async_db")) ) tbl_async = await async_db.create_table( "search_demo", data=data, schema=schema, mode="overwrite" @@ -198,7 +197,8 @@ async def test_query_to_pandas_kwargs(db_path_factory): query_vector = np.random.rand(128).astype(np.float32) - # --8<-- [start:query_to_pandas_kwargs] + # fmt: off + # --8<-- [start:legacy_query_to_pandas_kwargs] # Plain scan query: blob_mode is supported end to end df_lazy = ( tbl.search() @@ -223,7 +223,8 @@ async def test_query_to_pandas_kwargs(db_path_factory): .select(["id", "vector"]) .to_pandas(split_blocks=True, self_destruct=True) ) - # --8<-- [end:query_to_pandas_kwargs] + # --8<-- [end:legacy_query_to_pandas_kwargs] + # fmt: on assert len(df_lazy) == 1 assert len(df_bytes) == 1 diff --git a/docs/web-tests/rs/multimodal.rs b/docs/web-tests/rs/multimodal.rs index 8cae02eee..f9531d2d5 100644 --- a/docs/web-tests/rs/multimodal.rs +++ b/docs/web-tests/rs/multimodal.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use arrow_array::types::Float32Type; use arrow_array::{ - BinaryArray, FixedSizeListArray, Int32Array, Int64Array, LargeBinaryArray, RecordBatch, + Array, BinaryArray, FixedSizeListArray, Int32Array, Int64Array, LargeBinaryArray, RecordBatch, RecordBatchIterator, StringArray, }; use arrow_schema::{DataType, Field, Schema}; @@ -83,8 +83,9 @@ async fn main() { ], ) .unwrap(); - let image_reader: Box = - Box::new(RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), schema.clone())); + let image_reader: Box = Box::new( + RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), schema.clone()), + ); let table = db .create_table("images", image_reader) .mode(CreateTableMode::Overwrite) @@ -137,42 +138,45 @@ async fn main() { let search_rows: usize = results.iter().map(|batch| batch.num_rows()).sum(); assert_eq!(search_rows, 1); - // --8<-- [start:blob_api_schema] - let blob_metadata = HashMap::from([( + // --8<-- [start:legacy_blob_api_schema] + let legacy_blob_metadata = HashMap::from([( "lance-encoding:blob".to_string(), "true".to_string(), )]); - let blob_schema = Schema::new(vec![ + let legacy_blob_schema = Schema::new(vec![ Field::new("id", DataType::Int64, false), - Field::new("video", DataType::LargeBinary, true).with_metadata(blob_metadata), + Field::new("video", DataType::LargeBinary, true).with_metadata(legacy_blob_metadata), ]); - // --8<-- [end:blob_api_schema] + // --8<-- [end:legacy_blob_api_schema] - // --8<-- [start:blob_api_ingest] - let blob_rows = vec![ + // --8<-- [start:legacy_blob_api_ingest] + let legacy_blob_rows = vec![ (1_i64, b"fake_video_bytes_1".to_vec()), (2_i64, b"fake_video_bytes_2".to_vec()), ]; - let blob_schema = Arc::new(blob_schema); - let blob_batch = RecordBatch::try_new( - blob_schema.clone(), + let legacy_blob_schema = Arc::new(legacy_blob_schema); + let legacy_blob_batch = RecordBatch::try_new( + legacy_blob_schema.clone(), vec![ - Arc::new(Int64Array::from_iter_values(blob_rows.iter().map(|row| row.0))), + Arc::new(Int64Array::from_iter_values( + legacy_blob_rows.iter().map(|row| row.0), + )), Arc::new(LargeBinaryArray::from_iter_values( - blob_rows.iter().map(|row| row.1.as_slice()), + legacy_blob_rows.iter().map(|row| row.1.as_slice()), )), ], ) .unwrap(); - let blob_reader: Box = - Box::new(RecordBatchIterator::new(vec![Ok(blob_batch)].into_iter(), blob_schema)); - let blob_table = db - .create_table("videos", blob_reader) + let legacy_blob_reader: Box = Box::new( + RecordBatchIterator::new(vec![Ok(legacy_blob_batch)].into_iter(), legacy_blob_schema), + ); + let legacy_blob_table = db + .create_table("legacy_videos", legacy_blob_reader) .mode(CreateTableMode::Overwrite) .execute() .await .unwrap(); - // --8<-- [end:blob_api_ingest] - assert_eq!(blob_table.count_rows(None).await.unwrap(), 2); + // --8<-- [end:legacy_blob_api_ingest] + assert_eq!(legacy_blob_table.count_rows(None).await.unwrap(), 2); } diff --git a/docs/web-tests/ts/multimodal.test.ts b/docs/web-tests/ts/multimodal.test.ts index ac9a8201e..32672a6a3 100644 --- a/docs/web-tests/ts/multimodal.test.ts +++ b/docs/web-tests/ts/multimodal.test.ts @@ -75,8 +75,8 @@ test("multimodal snippets (async)", async () => { // --8<-- [end:process_results] expect(results).toHaveLength(1); - // --8<-- [start:blob_api_schema] - const blobSchema = new arrow.Schema([ + // --8<-- [start:legacy_blob_api_schema] + const legacyBlobSchema = new arrow.Schema([ new arrow.Field("id", new arrow.Int64()), new arrow.Field( "video", @@ -85,20 +85,24 @@ test("multimodal snippets (async)", async () => { new Map([["lance-encoding:blob", "true"]]), ), ]); - // --8<-- [end:blob_api_schema] + // --8<-- [end:legacy_blob_api_schema] - // --8<-- [start:blob_api_ingest] - const blobData = lancedb.makeArrowTable( + // --8<-- [start:legacy_blob_api_ingest] + const legacyBlobData = lancedb.makeArrowTable( [ { id: 1, video: Buffer.from("fake_video_bytes_1") }, { id: 2, video: Buffer.from("fake_video_bytes_2") }, ], - { schema: blobSchema }, + { schema: legacyBlobSchema }, ); - const blobTable = await db.createTable("videos", blobData, { - mode: "overwrite", - }); - // --8<-- [end:blob_api_ingest] - expect(await blobTable.countRows()).toBe(2); + const legacyBlobTable = await db.createTable( + "legacy_videos", + legacyBlobData, + { + mode: "overwrite", + }, + ); + // --8<-- [end:legacy_blob_api_ingest] + expect(await legacyBlobTable.countRows()).toBe(2); }); }); diff --git a/docs/web/ingestion/create-table.mdx b/docs/web/ingestion/create-table.mdx index 0921de22e..535ea83e3 100644 --- a/docs/web/ingestion/create-table.mdx +++ b/docs/web/ingestion/create-table.mdx @@ -146,7 +146,8 @@ You can define a custom Arrow schema for the table. This is useful when you want An explicit schema also determines which columns accept missing values (`null`). If later writes omit a non-nullable column, or provide actual nulls for it, ingestion fails; nullable columns can be omitted or written with null values. Without an explicit schema, Python infers list-like vector -values as fixed-size `float32` vector fields from the observed dimension. +values as fixed-size `float32` vector fields from the observed dimension. For images, audio, video, or +other binary columns whose payloads should load on demand, declare a [blob column](/management/table#multimodal-blob-v2) in the schema. For Python ingestion, malformed vectors fail by default (`on_bad_vectors="error"`). This includes vectors with incorrect sizes, NaNs, or null components, including pandas NaNs converted to Arrow diff --git a/docs/web/management/table.mdx b/docs/web/management/table.mdx index ed9883277..d817d4fa0 100644 --- a/docs/web/management/table.mdx +++ b/docs/web/management/table.mdx @@ -110,15 +110,40 @@ import { PyProcessResults as ProcessResults, TsProcessResults as TsProcessResults, RsProcessResults as RsProcessResults, + PyBlobApiSetup as BlobApiSetup, + TsBlobApiSetup as TsBlobApiSetup, + RsBlobApiImports as RsBlobApiImports, + RsBlobApiSetup as RsBlobApiSetup, PyBlobApiSchema as BlobApiSchema, TsBlobApiSchema as TsBlobApiSchema, RsBlobApiSchema as RsBlobApiSchema, PyBlobApiIngest as BlobApiIngest, + PyBlobApiBytes as BlobApiBytes, TsBlobApiIngest as TsBlobApiIngest, RsBlobApiIngest as RsBlobApiIngest, + PyBlobApiFiles as BlobApiFiles, + TsBlobApiFiles as TsBlobApiFiles, + RsBlobApiFiles as RsBlobApiFiles, + PyBlobApiFetch as BlobApiFetch, + TsBlobApiFetch as TsBlobApiFetch, + RsBlobApiFetch as RsBlobApiFetch, + PyBlobApiRanges as BlobApiRanges, + TsBlobApiRanges as TsBlobApiRanges, + RsBlobApiRanges as RsBlobApiRanges, + TsBlobApiBytesMode as TsBlobApiBytesMode, + PyLegacyBlobApiSchema as LegacyBlobApiSchema, + TsLegacyBlobApiSchema as TsLegacyBlobApiSchema, + RsLegacyBlobApiSchema as RsLegacyBlobApiSchema, + PyLegacyBlobApiIngest as LegacyBlobApiIngest, + TsLegacyBlobApiIngest as TsLegacyBlobApiIngest, + RsLegacyBlobApiIngest as RsLegacyBlobApiIngest, + PyLegacyBlobApiToPandas as LegacyBlobApiToPandas, + PyLegacyQueryToPandasKwargs as LegacyQueryToPandasKwargs, +} from '/snippets/multimodal.mdx'; +import { PyBlobApiToPandas as BlobApiToPandas, PyQueryToPandasKwargs as QueryToPandasKwargs, -} from '/snippets/multimodal.mdx'; +} from '/snippets/blob_pylance.mdx'; A table stores data in rows and columns and keeps a history of committed versions. Queries read table data, writes update it, and indexes accelerate @@ -307,138 +332,280 @@ extension types and pseudo-columns. ## Multimodal data {#multimodal} -Binary data (images, audio, video, model artifacts) lives in the table -beside the structured columns and embeddings that describe it, and is governed -by the same versioning and search. +Store images, audio, video, and documents alongside structured columns and +embeddings in the same versioned table. -Small values go in an ordinary `binary` column. Large ones go in a **blob** -column, stored out of line so a scan over the other columns stays fast. See -[Data types](/sql/data-types#blobs) for how the two differ to a reader. +Use `blob` columns for these payloads. Small values stay inline in the data +file, up to 64 KiB by default, and larger values are stored outside it. -### Store binary data {#multimodal-store-binary-data} +### Blob columns {#multimodal-blob-v2} -To store binary data, define a binary Arrow field in your schema (`pa.binary()` in Python, `Binary` in TypeScript, and `DataType::Binary` in Rust). + +Blob V2 is in early access. + -#### 1\. Setup and imports {#multimodal-1-setup-and-imports} +A `blob` column keeps metadata separate from the payload bytes, so queries can +filter rows without loading the blobs. -First, import the necessary libraries for LanceDB and Arrow in your SDK. +Most queries return descriptors, which contain metadata about the blob instead +of its bytes. You can +[read the bytes in the query results](#multimodal-blob-query-results) or +[fetch them by row ID](#multimodal-read-blobs). + +In Python, plain `to_pandas()` scans can also return `BlobFile` handles when +`pylance` is installed. + +When you write raw bytes, Lance chooses a storage layout based on the payload +size. A descriptor's `kind` tells you which one was used: + +| Storage mode | `kind` | Default placement | Storage behavior | +| --- | --- | --- | --- | +| `Inline` | `0` | Up to 64 KiB | Stored in the main data file. | +| `Packed` | `1` | Above 64 KiB, up to 4 MiB | Packed with other blobs into a shared sidecar file, up to 1 GiB per file by default. | +| `Dedicated` | `2` | Above 4 MiB | Stored in its own raw sidecar file. | +| `Managed` | `4` or `5` | Out-of-line values when managed storage is enabled | Stored in a Lance-owned object independently of the data file. | + +Managed storage must be enabled separately. See +[Managed blobs](#managed-blobs) for details. `kind=5` identifies a Managed +object using an explicitly registered base. + +Lance uses the default thresholds unless you customize them. Explicitly +configured thresholds are stored in the table schema and preserved on append. + +#### Setup {#multimodal-blob-v2-setup} + +Import the SDK and open a local database. The examples in this section use `db`. - {MultimodalImports} + {BlobApiSetup} - {TsMultimodalImports} + {TsMultimodalImports}{"\n"}{TsBlobApiSetup} - {RsMultimodalImports} + {RsBlobApiImports}{"\n"}{RsBlobApiSetup} -#### 2\. Prepare data {#multimodal-2-prepare-data} +#### Define a blob schema {#multimodal-blob-v2-schema} -For this example, we'll create some dummy in-memory images. In a real application, you would read these from files or an API. The key is to convert your data (image, audio, etc.) into a raw `bytes` object. +Declare a blob column with `lancedb.blob()` in Python and TypeScript, or +`blob()` in Rust: - {CreateDummyData} + {BlobApiSchema} - {TsCreateDummyData} + {TsBlobApiSchema} - {RsCreateDummyData} + {RsBlobApiSchema} -#### 3\. Define the schema {#multimodal-3-define-the-schema} +#### Ingest blobs {#multimodal-blob-v2-ingest} -When creating the table, it is **highly recommended** to define the schema explicitly. This ensures that your binary data is correctly interpreted as a `binary` type by Arrow/LanceDB and not as a generic string or list. +Pass raw bytes for each blob value. In Python, a blob column accepts `bytes`. +In TypeScript, it accepts a `Buffer`. The Python example creates two JPEGs +with EXIF metadata, which we'll read using a blob handle below. - {DefineSchema} + {BlobApiIngest} - {TsDefineSchema} + {TsBlobApiIngest} - {RsDefineSchema} + {RsBlobApiIngest} -#### 4\. Ingest data {#multimodal-4-ingest-data} +#### Read bytes in query results {#multimodal-blob-query-results} -Now, create the table using the data and the defined schema. +Python and TypeScript can return blob bytes in query results without +collecting `_rowid`. Rust uses [`fetch_blobs`](#multimodal-read-blobs) with +row IDs instead. + +##### Python {#multimodal-blob-python} + +`to_list(blob_mode="bytes")` returns the `image` bytes from the `photos` table +above: - {IngestData} - - - - {TsIngestData} - - - - {RsIngestData} + {BlobApiBytes} -### Retrieve and use blobs {#multimodal-retrieve-and-use-blobs} +In the Python SDK, `Table.to_pandas()`, `Table.to_arrow()`, and the query builder +methods `to_pandas()`, `to_arrow()`, and `to_list()` accept a `blob_mode` +argument: -When you search your LanceDB table, you can retrieve the binary column just like any other metadata. +| `blob_mode` | `to_pandas()` | `to_arrow()` and `to_list()` | +| --- | --- | --- | +| `"lazy"` | Default. Plain scans return seekable `BlobFile` handles. Vector, full-text, hybrid, and ordered queries return descriptor dicts. | Default. Returns descriptors carrying `_lance_row_id` so the result table can be passed to `fetch_blobs` or `fetch_blob_files`. | +| `"descriptions"` | Returns descriptor dicts with `kind`, `position`, `size`, `blob_id`, and `blob_uri`. | Returns the 5-field descriptor without `_lance_row_id`. | +| `"bytes"` | Materializes each blob as `bytes` in the DataFrame. | Materializes each blob as `large_binary` in Arrow or `bytes` in Python dicts. | + +`to_pandas(blob_mode="lazy")` and `to_pandas(blob_mode="bytes")` require +pylance. Without it they raise `ImportError`. `to_arrow()`, +`to_list(blob_mode="bytes")`, `to_pandas(blob_mode="descriptions")`, and the +fetch methods do not. + +`blob_mode="bytes"` requires a local table snapshot, so remote and in-memory +tables raise `NotImplementedError`. On a remote table, `Table.to_pandas()` and +`Table.to_arrow()` aren't supported, and a query's `to_pandas()` also raises in +the default `"lazy"` mode because it would open `BlobFile` handles. Query with +`blob_mode="descriptions"`, then read bytes with `fetch_blobs` or +`fetch_blob_files`. + +`blob_mode="bytes"` materializes top-level and struct-nested blob columns. It +raises `ValueError` when the selected columns include blobs +inside a list. Select those columns with `blob_mode="descriptions"`, or leave +them out of the projection. Extra keyword arguments on +`to_pandas()`, such as `split_blocks` and `self_destruct`, are forwarded to +PyArrow. + +###### Optional pandas examples {#multimodal-blob-pandas} + +These examples require pylance. The query example expects `tbl_async`, +`query_vector`, and a `video` column, which the `photos` table does not define. - {SearchData} - - - - {TsSearchData} - - - - {RsSearchData} + {BlobApiToPandas} -#### Convert bytes back to objects {#multimodal-convert-bytes-back-to-objects} - -Once you have the bytes back from the search result, you can decode them into the original format (for example, an image object or audio buffer). +Sync and async query builders accept the same `blob_mode` values: - {ProcessResults} + {QueryToPandasKwargs} + + + +##### TypeScript {#multimodal-blob-typescript} + +In TypeScript, `toArray()` and `toArrow()` accept a `blobMode` option: + +| `blobMode` | Result | +| --- | --- | +| `"descriptions"` | Default. Returns descriptors. | +| `"bytes"` | Returns each top-level blob as bytes, or `null` for a null blob. Blobs nested in a struct or list keep their descriptors. | + +With `"bytes"`, the rows and the bytes come from the same table version. If the +table changes while the query runs, the query runs again, up to three times: + + + + {TsBlobApiBytesMode} + + + +#### Read blobs on demand {#multimodal-read-blobs} + +Query the rows you want and pass their `_rowid` values to a fetch method. +`fetch_blob_files` opens a seekable handle and loads bytes only when you read. +The Python example opens the JPEG from the ingest step and reads its EXIF +Make tag with Pillow. TypeScript and Rust read the first bytes from the handle. + + + + {BlobApiFiles} - {TsProcessResults} + {TsBlobApiFiles} - {RsProcessResults} + {RsBlobApiFiles} +The other fetch methods load bytes directly: + +| Need | Python | TypeScript | Rust | +| --- | --- | --- | --- | +| Whole payloads for a set of rows | `fetch_blobs` | `fetchBlobs` | `fetch_blobs` | +| Byte slices across many rows in one call | `fetch_blob_ranges` | Not available. Use `fetchBlobFiles` and `readRanges`. | `fetch_blob_ranges` | + +- `fetch_blobs` returns the full bytes for each row ID in order. Null blobs + produce null entries, and empty blobs produce empty byte strings. +- `fetch_blob_files` loads bytes when you call `read`, `read_range`, or + `read_ranges`. Range reads don't move the file cursor. Null blobs produce + null handles. +- `fetch_blob_ranges` takes `(row_id, offset, length)` requests and returns one + slice per request, in order. A request list can repeat a row to read more + than one slice of that blob. + + + + {BlobApiFetch} + + + + {TsBlobApiFetch} + + + + {RsBlobApiFetch} + + + +Python and Rust read slices across rows with `fetch_blob_ranges`. On a handle, +`read_ranges` reads more than one slice of one blob. Python takes +`(offset, length)` pairs, TypeScript takes `{ start, end }` ranges with an +exclusive `end`, and Rust takes `start..end` ranges: + + + + {BlobApiRanges} + + + + {TsBlobApiRanges} + + + + {RsBlobApiRanges} + + + +#### Notes and limits {#multimodal-blob-notes} + +Blob fetches fail if any requested row ID is missing or deleted. With stable +row IDs enabled, IDs remain valid across compaction. + +A blob field can sit at the top level of the schema or inside a struct or list. +Struct-nested blob columns use dotted paths such as `clip.video` in the fetch +APIs. Blobs inside lists can be written and queried as descriptors. A table +row ID does not address one list element, so `blob_columns()`, `fetch_blobs`, +`fetch_blob_files`, and `blob_mode="bytes"` can't return those bytes. + +See the Lance [blob guide](https://lance.org/guide/blob/) for the storage format. + ### Managed blobs (Blob v2) {#managed-blobs} -Use `lancedb.blob()` for images, audio, videos, and other binary values whose -payloads should load separately from the rest of a row. Write Python `bytes` -normally. Set the table schema metadata `lance.blob.managed` to `true` to -store small values inline and larger values in managed objects under the table's `_blobs/` directory. Medium values can share a packed -object; large values use a dedicated object. +Set the table schema metadata `lance.blob.managed` to `true` on a table with +[`lancedb.blob()`](#multimodal-blob-v2) columns to store small values inline and +larger values in managed objects under the table's `_blobs/` directory. Medium +values can share a packed object, and large values use a dedicated object. The table owns these objects. You do not upload them separately, keep an object -catalog, or delete their files yourself. A descriptor records where to read the -bytes. Compaction preserves the descriptor and its payload object, so merging -table fragments does not rewrite the blob payloads. +catalog, or delete their files yourself. Compaction preserves each descriptor +and its payload object, so merging table fragments does not rewrite the blob +payloads. Use an SDK and Enterprise deployment that support Blob v2 Managed storage. @@ -484,16 +651,10 @@ assert by_id[1] == b"" assert len(by_id[2]) == 102400 ``` -`None` is a null value; `b""` is a present, empty blob. `fetch_blobs` returns -bytes in the order of the supplied row IDs. It materializes the selected -payloads, so fetch bounded batches when blobs are large. Row IDs belong to -their table; fetch them from the same table and snapshot that produced the -query result. - -Keep descriptors opaque in application code. Their representation can differ -between inline values, Managed objects, and externally referenced blobs. For -streaming and range reads, see the -[Lance blob guide](https://lance.org/guide/blob/). +`fetch_blobs` materializes the selected payloads in memory, so fetch bounded +batches when blobs are large. Row IDs belong to their table and snapshot. Keep +descriptors opaque in application code, because their representation can differ +between inline values and Managed objects. #### Derived blobs and maintenance {#managed-blobs-lifecycle} @@ -509,92 +670,190 @@ any of its contained blobs is still referenced. ### Legacy large-binary blobs {#multimodal-large-blobs-blob-api} -The schema below describes the earlier large-binary Blob API. It is distinct -from the `lancedb.blob()` schema above; changing the field's metadata alone does -not migrate an existing column to Blob v2. - -For larger files like high-resolution images or videos, Lance provides a specialized **Blob API**. By using a large-binary Arrow type (`pa.large_binary()` in Python, `LargeBinary` in TypeScript, and `DataType::LargeBinary` in Rust) and specific metadata, you enable **lazy loading** and optimized encoding. This allows you to work with massive datasets without loading all binary data into memory upfront. +Use [`lancedb.blob()`](#multimodal-blob-v2) for new tables. The sections below +cover existing tables that use the V1 large-binary Blob API +(`pa.large_binary()` in Python, `LargeBinary` in TypeScript, and +`DataType::LargeBinary` in Rust with `{"lance-encoding:blob": "true"}` field +metadata). Changing a field's metadata alone does not migrate an existing V1 +column to Blob v2. #### 1\. Define a blob schema {#multimodal-1-define-a-blob-schema} -To use the Blob API, you must mark the column with `{"lance-encoding:blob": "true"}` metadata. +V1 blob columns use a `large_binary` field marked with `{"lance-encoding:blob": "true"}` metadata: - {BlobApiSchema} + {LegacyBlobApiSchema} - {TsBlobApiSchema} + {TsLegacyBlobApiSchema} - {RsBlobApiSchema} + {RsLegacyBlobApiSchema} #### 2\. Ingest large blobs {#multimodal-2-ingest-large-blobs} -You can then ingest data normally, and Lance will handle the optimized storage. +Write raw bytes to a V1 blob column: - {BlobApiIngest} + {LegacyBlobApiIngest} - {TsBlobApiIngest} + {TsLegacyBlobApiIngest} - {RsBlobApiIngest} + {RsLegacyBlobApiIngest} - -For more advanced usage, including random access and file-like reading of blobs, see the -Lance format's [blob API documentation](https://lance.org/guide/blob/). - - #### 3\. Convert blob tables to pandas {#multimodal-3-convert-blob-tables-to-pandas} -When you call `to_pandas()` on a local LanceDB table that contains Blob API columns, the `blob_mode` argument controls how those columns materialize. This is available in the Python SDK on local tables; remote tables raise `NotImplementedError`. - -`blob_mode` accepts: - -- `"lazy"` (default): returns blob columns as lazy `BlobFile` objects without eagerly materializing their payloads. Use this when you want to stream blob bytes on demand or only inspect a subset of rows. Namespace-backed local tables also use the Lance native blob-aware pandas conversion for lazy blobs; in-memory datasets fall back to the standard PyArrow `to_pandas()` path. -- `"bytes"`: eagerly materializes each blob as `bytes`. Use this when you need the raw payload in the DataFrame, for example to decode an image or audio clip in-process. -- `"descriptions"`: returns blob descriptors (offsets, sizes, and positions) instead of the data itself. Use this when you want to plan I/O without paying the cost of loading every blob. - -`"bytes"` is not supported on in-memory tables. - -Extra keyword arguments are forwarded to the underlying PyArrow / Lance pandas conversion, so you can also pass options like `split_blocks` or `self_destruct`: +On local tables with V1 blob columns, `to_pandas()` in Python accepts `blob_mode` +(`"lazy"`, `"bytes"`, or `"descriptions"`). Remote tables raise +`NotImplementedError`, and `"bytes"` is not supported on in-memory tables. +Extra keyword arguments such as `split_blocks` or `self_destruct` are forwarded +to the PyArrow / Lance pandas conversion: - {BlobApiToPandas} + {LegacyBlobApiToPandas} -Query builders also accept `blob_mode` on their `to_pandas()` method: - -- Plain scans support `"lazy"`, `"bytes"`, and `"descriptions"` with filters, projections, aliases, `limit`, and `offset`. -- Vector, FTS, hybrid, and ordered queries can't materialize blob columns through `to_pandas()`; omit blob columns from the projection for those query shapes. -- This works on both sync and async query builders. Extra PyArrow kwargs like `split_blocks` and `self_destruct` are still forwarded. +On query builders with V1 blob columns, plain scans support `"lazy"`, `"bytes"`, +and `"descriptions"`. Vector, FTS, hybrid, and ordered queries cannot +materialize V1 blob columns through `to_pandas()`, so omit those columns from +the projection: - {QueryToPandasKwargs} + {LegacyQueryToPandasKwargs} + + + +### Store binary data {#multimodal-store-binary-data} + +Use a `binary` column when the query should return the raw bytes. Declare it as `pa.binary()` in Python, `Binary` in TypeScript, and `DataType::Binary` in Rust. + +#### 1\. Setup and imports {#multimodal-1-setup-and-imports} + +Import LanceDB and Arrow in your SDK. + + + + {MultimodalImports} + + + + {TsMultimodalImports} + + + + {RsMultimodalImports} + + + +#### 2\. Prepare data {#multimodal-2-prepare-data} + +Encode each image or file into raw bytes before writing. + + + + {CreateDummyData} + + + + {TsCreateDummyData} + + + + {RsCreateDummyData} + + + +#### 3\. Define the schema {#multimodal-3-define-the-schema} + +Declare the schema explicitly so Arrow and LanceDB store the field as `binary` instead of inferring a string or integer list. + + + + {DefineSchema} + + + + {TsDefineSchema} + + + + {RsDefineSchema} + + + +#### 4\. Ingest data {#multimodal-4-ingest-data} + +Create the table from the rows and schema. + + + + {IngestData} + + + + {TsIngestData} + + + + {RsIngestData} + + + +### Retrieve binary data {#multimodal-retrieve-and-use-blobs} + +When you search the table, select the `binary` column alongside your metadata columns. + + + + {SearchData} + + + + {TsSearchData} + + + + {RsSearchData} + + + +#### Convert bytes back to objects {#multimodal-convert-bytes-back-to-objects} + +Decode the returned bytes into an image or media buffer in your application. + + + + {ProcessResults} + + + + {TsProcessResults} + + + + {RsProcessResults} ### Other modalities {#multimodal-other-modalities} -The `pa.binary()` and `pa.large_binary()` types are universal. You can use this same pattern for other types of multimodal data: - -- **Audio:** Read `.wav` or `.mp3` files as bytes. -- **Video:** Store video transitions or full clips using the Blob API. -- **PDFs/Documents:** Store the raw file content for document search. +For PDFs, audio recordings, and other files, keep searchable fields in separate +columns so you can filter rows before fetching blob bytes. ### bfloat16 {#multimodal-bfloat16} diff --git a/docs/web/snippets/blob_pylance.mdx b/docs/web/snippets/blob_pylance.mdx new file mode 100644 index 000000000..eb0bd74d6 --- /dev/null +++ b/docs/web/snippets/blob_pylance.mdx @@ -0,0 +1,6 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const PyBlobApiToPandas = "df_lazy = tbl.to_pandas()\n\ndf_bytes = tbl.to_pandas(blob_mode=\"bytes\")\n\ndf_desc = tbl.to_pandas(blob_mode=\"descriptions\")\n\narrow_bytes = tbl.to_arrow(blob_mode=\"bytes\")\n\ndf_typed = tbl.to_pandas(split_blocks=True, self_destruct=True)\n"; + +export const PyQueryToPandasKwargs = "df_lazy = (\n tbl.search().where(\"id = 1\").select([\"id\", \"video\"]).to_pandas(blob_mode=\"lazy\")\n)\n\ndf_bytes = await (\n tbl_async.query()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"bytes\")\n)\n\ndf_vec = (\n tbl.search(query_vector)\n .limit(2)\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"bytes\", split_blocks=True, self_destruct=True)\n)\n"; + diff --git a/docs/web/snippets/multimodal.mdx b/docs/web/snippets/multimodal.mdx index f41af6629..19180dfd0 100644 --- a/docs/web/snippets/multimodal.mdx +++ b/docs/web/snippets/multimodal.mdx @@ -1,28 +1,52 @@ {/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} -export const PyBlobApiIngest = "import lancedb\n\ndb = lancedb.connect(db_path_factory(\"blob_db\"))\n \n# Create sample data\ndata = [\n {\"id\": 1, \"video\": b\"fake_video_bytes_1\"},\n {\"id\": 2, \"video\": b\"fake_video_bytes_2\"}\n]\n \n# Create the table\ntbl = db.create_table(\"videos\", data=data, schema=schema)\n"; +export const PyBlobApiBytes = "rows = (\n tbl.search().where(\"id = 1\").select([\"id\", \"image\"]).to_list(blob_mode=\"bytes\")\n)\n"; -export const PyBlobApiSchema = "import pyarrow as pa\n\n# Define schema with Blob API metadata for lazy loading\nschema = pa.schema([\n pa.field(\"id\", pa.int64()),\n pa.field(\n \"video\", \n pa.large_binary(), \n metadata={\"lance-encoding:blob\": \"true\"} # Enable Blob API\n ),\n])\n"; +export const PyBlobApiFetch = "rows = tbl.search().select([\"id\"]).with_row_id(True).to_list()\nrow_ids = [row[\"_rowid\"] for row in rows]\npayloads = tbl.fetch_blobs(\"image\", row_ids)\n"; -export const PyBlobApiToPandas = "# Default: blob columns come back lazily\ndf_lazy = tbl.to_pandas()\n\n# Materialize blob bytes eagerly\ndf_bytes = tbl.to_pandas(blob_mode=\"bytes\")\n\n# Return descriptors instead of payloads\ndf_desc = tbl.to_pandas(blob_mode=\"descriptions\")\n\n# Forward extra kwargs to PyArrow's to_pandas\ndf_typed = tbl.to_pandas(split_blocks=True, self_destruct=True)\n"; +export const PyBlobApiFiles = "from PIL import ExifTags\n\nrows = tbl.search().where(\"id = 1\").select([\"label\"]).with_row_id(True).to_list()\nwith tbl.fetch_blob_files(\"image\", [rows[0][\"_rowid\"]])[0] as image_file:\n make = Image.open(image_file).getexif().get(ExifTags.Base.Make)\nprint(rows[0][\"label\"], make)\n"; -export const PyCreateDummyData = "# Create some dummy images\ndef create_dummy_image(color):\n img = Image.new('RGB', (100, 100), color=color)\n buf = io.BytesIO()\n img.save(buf, format='PNG')\n return buf.getvalue()\n\n# Create dataset with metadata, vectors, and image blobs\ndata = [\n {\n \"id\": 1,\n \"filename\": \"red_square.png\",\n \"vector\": np.random.rand(128).astype(np.float32),\n \"image_blob\": create_dummy_image('red'),\n \"label\": \"red\"\n },\n {\n \"id\": 2,\n \"filename\": \"blue_square.png\",\n \"vector\": np.random.rand(128).astype(np.float32),\n \"image_blob\": create_dummy_image('blue'),\n \"label\": \"blue\"\n }\n]\n"; +export const PyBlobApiIngest = "import io\nfrom PIL import Image\n\ndef jpeg_with_make(color, make):\n image = Image.new(\"RGB\", (8, 8), color=color)\n exif = Image.Exif()\n exif[0x010F] = make\n buf = io.BytesIO()\n image.save(buf, format=\"JPEG\", exif=exif)\n return buf.getvalue()\n\ntbl = db.create_table(\n \"photos\",\n data=[\n {\"id\": 1, \"label\": \"red\", \"image\": jpeg_with_make(\"red\", \"TestCam\")},\n {\"id\": 2, \"label\": \"blue\", \"image\": jpeg_with_make(\"blue\", \"OtherCam\")},\n ],\n schema=schema,\n mode=\"overwrite\",\n)\n"; -export const PyDefineSchema = "# Define schema explicitly to ensure image_blob is treated as binary\nschema = pa.schema([\n pa.field(\"id\", pa.int32()),\n pa.field(\"filename\", pa.string()),\n pa.field(\"vector\", pa.list_(pa.float32(), 128)),\n pa.field(\"image_blob\", pa.binary()), # Important: Use pa.binary() for blobs\n pa.field(\"label\", pa.string())\n])\n"; +export const PyBlobApiRanges = "slices = tbl.fetch_blob_ranges(\"image\", [(row_ids[0], 0, 2)])\n\nwith tbl.fetch_blob_files(\"image\", [row_ids[0]])[0] as image_file:\n parts = image_file.read_ranges([(0, 2)])\n"; + +export const PyBlobApiSchema = "import lancedb\nimport pyarrow as pa\n\nschema = pa.schema(\n [\n pa.field(\"id\", pa.int64()),\n pa.field(\"label\", pa.string()),\n lancedb.blob(\"image\"),\n ]\n)\n"; + +export const PyBlobApiSetup = "import lancedb\n\ndb = lancedb.connect(\"./blob_db\")\n"; + +export const PyCreateDummyData = "def create_dummy_image(color):\n img = Image.new(\"RGB\", (100, 100), color=color)\n buf = io.BytesIO()\n img.save(buf, format=\"PNG\")\n return buf.getvalue()\n\ndata = [\n {\n \"id\": 1,\n \"filename\": \"red_square.png\",\n \"vector\": np.random.rand(128).astype(np.float32),\n \"image_blob\": create_dummy_image(\"red\"),\n \"label\": \"red\",\n },\n {\n \"id\": 2,\n \"filename\": \"blue_square.png\",\n \"vector\": np.random.rand(128).astype(np.float32),\n \"image_blob\": create_dummy_image(\"blue\"),\n \"label\": \"blue\",\n },\n]\n"; + +export const PyDefineSchema = "schema = pa.schema(\n [\n pa.field(\"id\", pa.int32()),\n pa.field(\"filename\", pa.string()),\n pa.field(\"vector\", pa.list_(pa.float32(), 128)),\n pa.field(\"image_blob\", pa.binary()),\n pa.field(\"label\", pa.string()),\n ]\n)\n"; export const PyIngestData = "tbl = db.create_table(\"images\", data=data, schema=schema, mode=\"overwrite\")\n"; -export const PyMultimodalImports = "import lancedb\nimport pyarrow as pa\nimport pandas as pd\nimport numpy as np\nimport io\nfrom PIL import Image\n"; +export const PyLegacyBlobApiIngest = "import lancedb\n\ndb = lancedb.connect(\"./blob_db\")\n\n# Create sample data\ndata = [\n {\"id\": 1, \"video\": b\"fake_video_bytes_1\"},\n {\"id\": 2, \"video\": b\"fake_video_bytes_2\"},\n]\n\n# Create the table\ntbl = db.create_table(\"videos\", data=data, schema=schema)\n"; -export const PyProcessResults = "# Convert back to PIL Image\nfor _, row in results.iterrows():\n image_bytes = row['image_blob']\n image = Image.open(io.BytesIO(image_bytes))\n print(f\"Retrieved image: {row['filename']}, Size: {image.size}\")\n # You can now use 'image' with other libraries or display it\n"; +export const PyLegacyBlobApiSchema = "import pyarrow as pa\n\n# Define schema with Blob API metadata for lazy loading\nschema = pa.schema(\n [\n pa.field(\"id\", pa.int64()),\n pa.field(\n \"video\",\n pa.large_binary(),\n metadata={\"lance-encoding:blob\": \"true\"}, # Enable Blob API\n ),\n ]\n)\n"; -export const PyQueryToPandasKwargs = "# Plain scan query: blob_mode is supported end to end\ndf_lazy = (\n tbl.search()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"lazy\")\n)\n\n# Same call shape works on async query builders\ndf_bytes = await (\n tbl_async.query()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"bytes\")\n)\n\n# Vector / FTS / hybrid queries can't materialize blob columns,\n# so omit them from the projection\ndf_vec = (\n tbl.search(query_vector)\n .limit(10)\n .select([\"id\", \"vector\"])\n .to_pandas(split_blocks=True, self_destruct=True)\n)\n"; +export const PyLegacyBlobApiToPandas = "# Default: blob columns come back lazily\ndf_lazy = tbl.to_pandas()\n\n# Materialize blob bytes eagerly\ndf_bytes = tbl.to_pandas(blob_mode=\"bytes\")\n\n# Return descriptors instead of payloads\ndf_desc = tbl.to_pandas(blob_mode=\"descriptions\")\n\n# Forward extra kwargs to PyArrow's to_pandas\ndf_typed = tbl.to_pandas(split_blocks=True, self_destruct=True)\n"; -export const PySearchData = "# Search for similar images\nquery_vector = np.random.rand(128).astype(np.float32)\nresults = tbl.search(query_vector).limit(1).to_pandas()\n"; +export const PyLegacyQueryToPandasKwargs = "# Plain scan query: blob_mode is supported end to end\ndf_lazy = (\n tbl.search()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"lazy\")\n)\n\n# Same call shape works on async query builders\ndf_bytes = await (\n tbl_async.query()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"bytes\")\n)\n\n# Vector / FTS / hybrid queries can't materialize blob columns,\n# so omit them from the projection\ndf_vec = (\n tbl.search(query_vector)\n .limit(10)\n .select([\"id\", \"vector\"])\n .to_pandas(split_blocks=True, self_destruct=True)\n)\n"; -export const TsBlobApiIngest = "const blobData = lancedb.makeArrowTable(\n [\n { id: 1, video: Buffer.from(\"fake_video_bytes_1\") },\n { id: 2, video: Buffer.from(\"fake_video_bytes_2\") },\n ],\n { schema: blobSchema },\n);\nconst blobTable = await db.createTable(\"videos\", blobData, {\n mode: \"overwrite\",\n});\n"; +export const PyMultimodalImports = "import io\nimport lancedb\nimport numpy as np\nimport pyarrow as pa\nfrom PIL import Image\n"; -export const TsBlobApiSchema = "const blobSchema = new arrow.Schema([\n new arrow.Field(\"id\", new arrow.Int64()),\n new arrow.Field(\n \"video\",\n new arrow.LargeBinary(),\n true,\n new Map([[\"lance-encoding:blob\", \"true\"]]),\n ),\n]);\n"; +export const PyProcessResults = "for _, row in results.iterrows():\n image_bytes = row[\"image_blob\"]\n image = Image.open(io.BytesIO(image_bytes))\n print(f\"Retrieved image: {row['filename']}, Size: {image.size}\")\n"; + +export const PySearchData = "query_vector = np.random.rand(128).astype(np.float32)\nresults = tbl.search(query_vector).limit(1).to_pandas()\n"; + +export const TsBlobApiBytesMode = "const withBytes = await blobTable\n .query()\n .select([\"id\", \"image\"])\n .toArray({ blobMode: \"bytes\" });\nconst firstImage: Uint8Array | null = withBytes[0].image;\n"; + +export const TsBlobApiFetch = "const payloads = await blobTable.fetchBlobs(\"image\", rowIds);\n"; + +export const TsBlobApiFiles = "const rows = await blobTable\n .query()\n .select([\"id\", \"label\"])\n .withRowId()\n .toArray();\nconst rowIds = rows.map((row) => row._rowid as bigint);\n\nconst [image] = await blobTable.fetchBlobFiles(\"image\", [rowIds[0]!]);\nconst header = await image!.readRange(0n, 6n);\nawait image!.close();\n"; + +export const TsBlobApiIngest = "const blobTable = await db.createEmptyTable(\"photos\", blobSchema, {\n mode: \"overwrite\",\n});\n\nawait blobTable.add([\n { id: 1n, label: \"door\", image: Buffer.from(\"inline_image_bytes\") },\n { id: 2n, label: \"window\", image: Buffer.from(\"second_image_bytes\") },\n]);\n"; + +export const TsBlobApiRanges = "const [secondImage] = await blobTable.fetchBlobFiles(\"image\", [rowIds[1]!]);\nconst parts = await secondImage!.readRanges([\n { start: 0n, end: 6n },\n { start: 7n, end: 12n },\n]);\nawait secondImage!.close();\n"; + +export const TsBlobApiSchema = "const blobSchema = new arrow.Schema([\n new arrow.Field(\"id\", new arrow.Int64()),\n new arrow.Field(\"label\", new arrow.Utf8()),\n lancedb.blob(\"image\"),\n]);\n"; + +export const TsBlobApiSetup = "const db = await lancedb.connect(\"./blob_db\");\n"; export const TsCreateDummyData = "const createDummyImage = (color: string): Uint8Array => {\n const pngHeader = Uint8Array.from([137, 80, 78, 71, 13, 10, 26, 10]);\n return Buffer.concat([Buffer.from(pngHeader), Buffer.from(color, \"utf8\")]);\n};\n\nconst data = [\n {\n id: 1,\n filename: \"red_square.png\",\n vector: Array.from({ length: 128 }, (_, i) => (i % 16) / 16),\n image_blob: createDummyImage(\"red\"),\n label: \"red\",\n },\n {\n id: 2,\n filename: \"blue_square.png\",\n vector: Array.from({ length: 128 }, (_, i) => ((i + 8) % 16) / 16),\n image_blob: createDummyImage(\"blue\"),\n label: \"blue\",\n },\n];\n"; @@ -30,23 +54,41 @@ export const TsDefineSchema = "const schema = new arrow.Schema([\n new arrow.Fi export const TsIngestData = "const multimodalData = lancedb.makeArrowTable(data, { schema });\nconst tbl = await db.createTable(\"images\", multimodalData, {\n mode: \"overwrite\",\n});\n"; +export const TsLegacyBlobApiIngest = "const legacyBlobData = lancedb.makeArrowTable(\n [\n { id: 1, video: Buffer.from(\"fake_video_bytes_1\") },\n { id: 2, video: Buffer.from(\"fake_video_bytes_2\") },\n ],\n { schema: legacyBlobSchema },\n);\nconst legacyBlobTable = await db.createTable(\n \"legacy_videos\",\n legacyBlobData,\n {\n mode: \"overwrite\",\n },\n);\n"; + +export const TsLegacyBlobApiSchema = "const legacyBlobSchema = new arrow.Schema([\n new arrow.Field(\"id\", new arrow.Int64()),\n new arrow.Field(\n \"video\",\n new arrow.LargeBinary(),\n true,\n new Map([[\"lance-encoding:blob\", \"true\"]]),\n ),\n]);\n"; + export const TsMultimodalImports = "import * as arrow from \"apache-arrow\";\nimport { Buffer } from \"node:buffer\";\nimport * as lancedb from \"@lancedb/lancedb\";\n"; export const TsProcessResults = "for (const row of results) {\n const imageBytes = row.image_blob as Uint8Array;\n console.log(\n `Retrieved image: ${row.filename}, Byte length: ${imageBytes.length}`,\n );\n}\n"; export const TsSearchData = "const queryVector = Array.from({ length: 128 }, (_, i) => (i % 16) / 16);\nconst results = await tbl.search(queryVector).limit(1).toArray();\n"; -export const RsBlobApiIngest = "let blob_rows = vec![\n (1_i64, b\"fake_video_bytes_1\".to_vec()),\n (2_i64, b\"fake_video_bytes_2\".to_vec()),\n];\n\nlet blob_schema = Arc::new(blob_schema);\nlet blob_batch = RecordBatch::try_new(\n blob_schema.clone(),\n vec![\n Arc::new(Int64Array::from_iter_values(blob_rows.iter().map(|row| row.0))),\n Arc::new(LargeBinaryArray::from_iter_values(\n blob_rows.iter().map(|row| row.1.as_slice()),\n )),\n ],\n)\n.unwrap();\nlet blob_reader: Box =\n Box::new(RecordBatchIterator::new(vec![Ok(blob_batch)].into_iter(), blob_schema));\nlet blob_table = db\n .create_table(\"videos\", blob_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; +export const RsBlobApiFetch = "let payloads = blob_table.fetch_blobs(\"image\", &row_ids).await.unwrap();\n"; -export const RsBlobApiSchema = "let blob_metadata = HashMap::from([(\n \"lance-encoding:blob\".to_string(),\n \"true\".to_string(),\n)]);\nlet blob_schema = Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"video\", DataType::LargeBinary, true).with_metadata(blob_metadata),\n]);\n"; +export const RsBlobApiFiles = "let batches = blob_table\n .query()\n .with_row_id()\n .execute()\n .await\n .unwrap()\n .try_collect::>()\n .await\n .unwrap();\nlet row_ids = batches[0]\n .column_by_name(\"_rowid\")\n .unwrap()\n .as_any()\n .downcast_ref::()\n .unwrap()\n .values()\n .to_vec();\n\nlet handles = blob_table\n .fetch_blob_files(\"image\", &row_ids[..1])\n .await\n .unwrap();\nlet header = handles[0].as_ref().unwrap().read_range(0..6).await.unwrap();\nhandles[0].as_ref().unwrap().close().await.unwrap();\n"; + +export const RsBlobApiImports = "use std::sync::Arc;\n\nuse arrow_array::{\n Array, Int64Array, LargeBinaryArray, RecordBatch, RecordBatchIterator, StringArray, UInt64Array,\n};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures_util::TryStreamExt;\nuse lancedb::blob::{BlobRangeRequest, blob};\nuse lancedb::connect;\nuse lancedb::database::CreateTableMode;\nuse lancedb::query::{ExecutableQuery, QueryBase};\n"; + +export const RsBlobApiIngest = "let blob_table = db\n .create_empty_table(\"photos\", blob_schema)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n\nlet input_schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"label\", DataType::Utf8, true),\n Field::new(\"image\", DataType::LargeBinary, true),\n]));\nlet input_batch = RecordBatch::try_new(\n input_schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1_i64, 2_i64])),\n Arc::new(StringArray::from(vec![Some(\"door\"), Some(\"window\")])),\n Arc::new(LargeBinaryArray::from_iter_values([\n b\"inline_image_bytes\".as_slice(),\n b\"second_image_bytes\".as_slice(),\n ])),\n ],\n)\n.unwrap();\nlet input_reader: Box = Box::new(\n RecordBatchIterator::new(vec![Ok(input_batch)].into_iter(), input_schema),\n);\nblob_table.add(input_reader).execute().await.unwrap();\n"; + +export const RsBlobApiRanges = "let slices = blob_table\n .fetch_blob_ranges(\n \"image\",\n [\n BlobRangeRequest::new(row_ids[0], 0, 6),\n BlobRangeRequest::new(row_ids[1], 0, 6),\n BlobRangeRequest::new(row_ids[1], 7, 5),\n ],\n )\n .await\n .unwrap();\n\nlet second_handles = blob_table\n .fetch_blob_files(\"image\", &row_ids[1..2])\n .await\n .unwrap();\nlet second_image = second_handles[0].as_ref().unwrap();\nlet parts = second_image.read_ranges(&[0..6, 7..12]).await.unwrap();\nsecond_image.close().await.unwrap();\n"; + +export const RsBlobApiSchema = "let blob_schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"label\", DataType::Utf8, true),\n blob(\"image\", true),\n]));\n"; + +export const RsBlobApiSetup = "let db = connect(\"./blob_db\").execute().await.unwrap();\n"; export const RsCreateDummyData = "let create_dummy_image = |color: u8| -> Vec {\n let mut png_like = vec![137, 80, 78, 71, 13, 10, 26, 10];\n png_like.push(color);\n png_like\n};\n\nlet data = vec![\n (\n 1_i32,\n \"red_square.png\",\n vec![0.1_f32; 128],\n create_dummy_image(1),\n \"red\",\n ),\n (\n 2_i32,\n \"blue_square.png\",\n vec![0.2_f32; 128],\n create_dummy_image(2),\n \"blue\",\n ),\n];\n"; export const RsDefineSchema = "let schema = Schema::new(vec![\n Field::new(\"id\", DataType::Int32, false),\n Field::new(\"filename\", DataType::Utf8, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 128),\n false,\n ),\n Field::new(\"image_blob\", DataType::Binary, false),\n Field::new(\"label\", DataType::Utf8, false),\n]);\n"; -export const RsIngestData = "let schema = Arc::new(schema);\nlet image_batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int32Array::from_iter_values(data.iter().map(|row| row.0))),\n Arc::new(StringArray::from_iter_values(data.iter().map(|row| row.1))),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n data.iter()\n .map(|row| Some(row.2.iter().copied().map(Some).collect::>())),\n 128,\n ),\n ),\n Arc::new(BinaryArray::from_iter_values(\n data.iter().map(|row| row.3.as_slice()),\n )),\n Arc::new(StringArray::from_iter_values(data.iter().map(|row| row.4))),\n ],\n)\n.unwrap();\nlet image_reader: Box =\n Box::new(RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), schema.clone()));\nlet table = db\n .create_table(\"images\", image_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; +export const RsIngestData = "let schema = Arc::new(schema);\nlet image_batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int32Array::from_iter_values(data.iter().map(|row| row.0))),\n Arc::new(StringArray::from_iter_values(data.iter().map(|row| row.1))),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n data.iter()\n .map(|row| Some(row.2.iter().copied().map(Some).collect::>())),\n 128,\n ),\n ),\n Arc::new(BinaryArray::from_iter_values(\n data.iter().map(|row| row.3.as_slice()),\n )),\n Arc::new(StringArray::from_iter_values(data.iter().map(|row| row.4))),\n ],\n)\n.unwrap();\nlet image_reader: Box = Box::new(\n RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), schema.clone()),\n);\nlet table = db\n .create_table(\"images\", image_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; -export const RsMultimodalImports = "use std::collections::HashMap;\nuse std::sync::Arc;\n\nuse arrow_array::types::Float32Type;\nuse arrow_array::{\n BinaryArray, FixedSizeListArray, Int32Array, Int64Array, LargeBinaryArray, RecordBatch,\n RecordBatchIterator, StringArray,\n};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures_util::TryStreamExt;\nuse lancedb::connect;\nuse lancedb::database::CreateTableMode;\nuse lancedb::query::{ExecutableQuery, QueryBase};\n"; +export const RsLegacyBlobApiIngest = "let legacy_blob_rows = vec![\n (1_i64, b\"fake_video_bytes_1\".to_vec()),\n (2_i64, b\"fake_video_bytes_2\".to_vec()),\n];\n\nlet legacy_blob_schema = Arc::new(legacy_blob_schema);\nlet legacy_blob_batch = RecordBatch::try_new(\n legacy_blob_schema.clone(),\n vec![\n Arc::new(Int64Array::from_iter_values(\n legacy_blob_rows.iter().map(|row| row.0),\n )),\n Arc::new(LargeBinaryArray::from_iter_values(\n legacy_blob_rows.iter().map(|row| row.1.as_slice()),\n )),\n ],\n)\n.unwrap();\nlet legacy_blob_reader: Box = Box::new(\n RecordBatchIterator::new(vec![Ok(legacy_blob_batch)].into_iter(), legacy_blob_schema),\n);\nlet legacy_blob_table = db\n .create_table(\"legacy_videos\", legacy_blob_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; + +export const RsLegacyBlobApiSchema = "let legacy_blob_metadata = HashMap::from([(\n \"lance-encoding:blob\".to_string(),\n \"true\".to_string(),\n)]);\nlet legacy_blob_schema = Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"video\", DataType::LargeBinary, true).with_metadata(legacy_blob_metadata),\n]);\n"; + +export const RsMultimodalImports = "use std::collections::HashMap;\nuse std::sync::Arc;\n\nuse arrow_array::types::Float32Type;\nuse arrow_array::{\n Array, BinaryArray, FixedSizeListArray, Int32Array, Int64Array, LargeBinaryArray, RecordBatch,\n RecordBatchIterator, StringArray,\n};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures_util::TryStreamExt;\nuse lancedb::connect;\nuse lancedb::database::CreateTableMode;\nuse lancedb::query::{ExecutableQuery, QueryBase};\n"; export const RsProcessResults = "for batch in &results {\n let filenames = batch\n .column_by_name(\"filename\")\n .unwrap()\n .as_any()\n .downcast_ref::()\n .unwrap();\n let images = batch\n .column_by_name(\"image_blob\")\n .unwrap()\n .as_any()\n .downcast_ref::()\n .unwrap();\n\n for row in 0..batch.num_rows() {\n let image_bytes = images.value(row);\n println!(\n \"Retrieved image: {}, Byte length: {}\",\n filenames.value(row),\n image_bytes.len()\n );\n }\n}\n"; diff --git a/docs/web/sql/data-types.mdx b/docs/web/sql/data-types.mdx index 863212385..051368525 100644 --- a/docs/web/sql/data-types.mdx +++ b/docs/web/sql/data-types.mdx @@ -3,7 +3,7 @@ title: "SQL data types" sidebarTitle: "Data Types" description: "How Lance columns appear to SQL, including the types SQL has no name for." icon: shapes -keywords: ["sql types", "arrow", "cast", "pseudo-columns", "vector", "blob", "json"] +keywords: ["sql types", "arrow", "cast", "pseudo-columns", "vector", "json"] --- Enterprise [SQL runs in LanceDB Enterprise.](/sql) @@ -61,7 +61,7 @@ describes the full mapping. ## Types SQL has no name for {#no-name} -Three column kinds you will meet that no SQL type name covers. +Some column kinds have no SQL type name. ### Vectors {#vectors} @@ -94,17 +94,6 @@ WHERE json_get_int(data, 'status') = 200 `json_get`, its typed variants, and `json_extract` are **index-accelerated** by a JSON scalar index. See [Functions](/sql/scalar-functions#json). -### Blobs {#blobs} - -A blob column is read either as its binary contents or as a **description**: a -pointer plus metadata, without fetching the bytes. That choice changes the -column's Arrow type, and therefore the schema your query sees. - - -The default reads blob columns as descriptions and other binary columns as -binary. A query selecting a blob column and expecting bytes gets a struct. - - ### bfloat16 {#bfloat16} Sixteen-bit floats in the bfloat16 layout, stored as `fixed_size_binary(2)`. diff --git a/docs/web/sql/ddl/table.mdx b/docs/web/sql/ddl/table.mdx index 11e7a3900..2d1ac6cbc 100644 --- a/docs/web/sql/ddl/table.mdx +++ b/docs/web/sql/ddl/table.mdx @@ -16,6 +16,7 @@ CREATE TABLE recent AS SELECT * FROM documents WHERE created_at > now() - INTERV ``` Column types are Arrow types under SQL names. See [Data types](/sql/data-types). + A table inside a [namespace](/sql/ddl/namespace) is written as a dotted path: `prod.events.clicks`.