mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-27 22:16:14 +00:00
97 lines
17 KiB
Plaintext
97 lines
17 KiB
Plaintext
{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */}
|
|
|
|
export const PyBlobApiBytes = "rows = (\n tbl.search().where(\"id = 1\").select([\"id\", \"image\"]).to_list(blob_mode=\"bytes\")\n)\n";
|
|
|
|
export const PyBlobApiFetch = "rows = tbl.search().select([\"id\"]).with_row_id(True).to_list()\nrow_ids = [row[\"_rowid\"] for row in rows]\npayloads = tbl.fetch_blobs(\"image\", row_ids)\n";
|
|
|
|
export const PyBlobApiFiles = "from PIL import ExifTags\n\nrows = tbl.search().where(\"id = 1\").select([\"label\"]).with_row_id(True).to_list()\nwith tbl.fetch_blob_files(\"image\", [rows[0][\"_rowid\"]])[0] as image_file:\n make = Image.open(image_file).getexif().get(ExifTags.Base.Make)\nprint(rows[0][\"label\"], make)\n";
|
|
|
|
export const PyBlobApiIngest = "import io\nfrom PIL import Image\n\ndef jpeg_with_make(color, make):\n image = Image.new(\"RGB\", (8, 8), color=color)\n exif = Image.Exif()\n exif[0x010F] = make\n buf = io.BytesIO()\n image.save(buf, format=\"JPEG\", exif=exif)\n return buf.getvalue()\n\ntbl = db.create_table(\n \"photos\",\n data=[\n {\"id\": 1, \"label\": \"red\", \"image\": jpeg_with_make(\"red\", \"TestCam\")},\n {\"id\": 2, \"label\": \"blue\", \"image\": jpeg_with_make(\"blue\", \"OtherCam\")},\n ],\n schema=schema,\n mode=\"overwrite\",\n)\n";
|
|
|
|
export const PyBlobApiRanges = "slices = tbl.fetch_blob_ranges(\"image\", [(row_ids[0], 0, 2)])\n\nwith tbl.fetch_blob_files(\"image\", [row_ids[0]])[0] as image_file:\n parts = image_file.read_ranges([(0, 2)])\n";
|
|
|
|
export const PyBlobApiSchema = "import lancedb\nimport pyarrow as pa\n\nschema = pa.schema(\n [\n pa.field(\"id\", pa.int64()),\n pa.field(\"label\", pa.string()),\n lancedb.blob(\"image\"),\n ]\n)\n";
|
|
|
|
export const PyBlobApiSetup = "import lancedb\n\ndb = lancedb.connect(\"./blob_db\")\n";
|
|
|
|
export const PyCreateDummyData = "def create_dummy_image(color):\n img = Image.new(\"RGB\", (100, 100), color=color)\n buf = io.BytesIO()\n img.save(buf, format=\"PNG\")\n return buf.getvalue()\n\ndata = [\n {\n \"id\": 1,\n \"filename\": \"red_square.png\",\n \"vector\": np.random.rand(128).astype(np.float32),\n \"image_blob\": create_dummy_image(\"red\"),\n \"label\": \"red\",\n },\n {\n \"id\": 2,\n \"filename\": \"blue_square.png\",\n \"vector\": np.random.rand(128).astype(np.float32),\n \"image_blob\": create_dummy_image(\"blue\"),\n \"label\": \"blue\",\n },\n]\n";
|
|
|
|
export const PyDefineSchema = "schema = pa.schema(\n [\n pa.field(\"id\", pa.int32()),\n pa.field(\"filename\", pa.string()),\n pa.field(\"vector\", pa.list_(pa.float32(), 128)),\n pa.field(\"image_blob\", pa.binary()),\n pa.field(\"label\", pa.string()),\n ]\n)\n";
|
|
|
|
export const PyIngestData = "tbl = db.create_table(\"images\", data=data, schema=schema, mode=\"overwrite\")\n";
|
|
|
|
export const PyLegacyBlobApiIngest = "import lancedb\n\ndb = lancedb.connect(\"./blob_db\")\n\n# Create sample data\ndata = [\n {\"id\": 1, \"video\": b\"fake_video_bytes_1\"},\n {\"id\": 2, \"video\": b\"fake_video_bytes_2\"},\n]\n\n# Create the table\ntbl = db.create_table(\"videos\", data=data, schema=schema)\n";
|
|
|
|
export const PyLegacyBlobApiSchema = "import pyarrow as pa\n\n# Define schema with Blob API metadata for lazy loading\nschema = pa.schema(\n [\n pa.field(\"id\", pa.int64()),\n pa.field(\n \"video\",\n pa.large_binary(),\n metadata={\"lance-encoding:blob\": \"true\"}, # Enable Blob API\n ),\n ]\n)\n";
|
|
|
|
export const PyLegacyBlobApiToPandas = "# Default: blob columns come back lazily\ndf_lazy = tbl.to_pandas()\n\n# Materialize blob bytes eagerly\ndf_bytes = tbl.to_pandas(blob_mode=\"bytes\")\n\n# Return descriptors instead of payloads\ndf_desc = tbl.to_pandas(blob_mode=\"descriptions\")\n\n# Forward extra kwargs to PyArrow's to_pandas\ndf_typed = tbl.to_pandas(split_blocks=True, self_destruct=True)\n";
|
|
|
|
export const PyLegacyQueryToPandasKwargs = "# Plain scan query: blob_mode is supported end to end\ndf_lazy = (\n tbl.search()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"lazy\")\n)\n\n# Same call shape works on async query builders\ndf_bytes = await (\n tbl_async.query()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"bytes\")\n)\n\n# Vector / FTS / hybrid queries can't materialize blob columns,\n# so omit them from the projection\ndf_vec = (\n tbl.search(query_vector)\n .limit(10)\n .select([\"id\", \"vector\"])\n .to_pandas(split_blocks=True, self_destruct=True)\n)\n";
|
|
|
|
export const PyMultimodalImports = "import io\nimport lancedb\nimport numpy as np\nimport pyarrow as pa\nfrom PIL import Image\n";
|
|
|
|
export const PyProcessResults = "for _, row in results.iterrows():\n image_bytes = row[\"image_blob\"]\n image = Image.open(io.BytesIO(image_bytes))\n print(f\"Retrieved image: {row['filename']}, Size: {image.size}\")\n";
|
|
|
|
export const PySearchData = "query_vector = np.random.rand(128).astype(np.float32)\nresults = tbl.search(query_vector).limit(1).to_pandas()\n";
|
|
|
|
export const TsBlobApiBytesMode = "const withBytes = await blobTable\n .query()\n .select([\"id\", \"image\"])\n .toArray({ blobMode: \"bytes\" });\nconst firstImage: Uint8Array | null = withBytes[0].image;\n";
|
|
|
|
export const TsBlobApiFetch = "const payloads = await blobTable.fetchBlobs(\"image\", rowIds);\n";
|
|
|
|
export const TsBlobApiFiles = "const rows = await blobTable\n .query()\n .select([\"id\", \"label\"])\n .withRowId()\n .toArray();\nconst rowIds = rows.map((row) => row._rowid as bigint);\n\nconst [image] = await blobTable.fetchBlobFiles(\"image\", [rowIds[0]!]);\nconst header = await image!.readRange(0n, 6n);\nawait image!.close();\n";
|
|
|
|
export const TsBlobApiIngest = "const blobTable = await db.createEmptyTable(\"photos\", blobSchema, {\n mode: \"overwrite\",\n});\n\nawait blobTable.add([\n { id: 1n, label: \"door\", image: Buffer.from(\"inline_image_bytes\") },\n { id: 2n, label: \"window\", image: Buffer.from(\"second_image_bytes\") },\n]);\n";
|
|
|
|
export const TsBlobApiRanges = "const [secondImage] = await blobTable.fetchBlobFiles(\"image\", [rowIds[1]!]);\nconst parts = await secondImage!.readRanges([\n { start: 0n, end: 6n },\n { start: 7n, end: 12n },\n]);\nawait secondImage!.close();\n";
|
|
|
|
export const TsBlobApiSchema = "const blobSchema = new arrow.Schema([\n new arrow.Field(\"id\", new arrow.Int64()),\n new arrow.Field(\"label\", new arrow.Utf8()),\n lancedb.blob(\"image\"),\n]);\n";
|
|
|
|
export const TsBlobApiSetup = "const db = await lancedb.connect(\"./blob_db\");\n";
|
|
|
|
export const TsCreateDummyData = "const createDummyImage = (color: string): Uint8Array => {\n const pngHeader = Uint8Array.from([137, 80, 78, 71, 13, 10, 26, 10]);\n return Buffer.concat([Buffer.from(pngHeader), Buffer.from(color, \"utf8\")]);\n};\n\nconst data = [\n {\n id: 1,\n filename: \"red_square.png\",\n vector: Array.from({ length: 128 }, (_, i) => (i % 16) / 16),\n image_blob: createDummyImage(\"red\"),\n label: \"red\",\n },\n {\n id: 2,\n filename: \"blue_square.png\",\n vector: Array.from({ length: 128 }, (_, i) => ((i + 8) % 16) / 16),\n image_blob: createDummyImage(\"blue\"),\n label: \"blue\",\n },\n];\n";
|
|
|
|
export const TsDefineSchema = "const schema = new arrow.Schema([\n new arrow.Field(\"id\", new arrow.Int32()),\n new arrow.Field(\"filename\", new arrow.Utf8()),\n new arrow.Field(\n \"vector\",\n new arrow.FixedSizeList(\n 128,\n new arrow.Field(\"item\", new arrow.Float32(), true),\n ),\n ),\n new arrow.Field(\"image_blob\", new arrow.Binary()),\n new arrow.Field(\"label\", new arrow.Utf8()),\n]);\n";
|
|
|
|
export const TsIngestData = "const multimodalData = lancedb.makeArrowTable(data, { schema });\nconst tbl = await db.createTable(\"images\", multimodalData, {\n mode: \"overwrite\",\n});\n";
|
|
|
|
export const TsLegacyBlobApiIngest = "const legacyBlobData = lancedb.makeArrowTable(\n [\n { id: 1, video: Buffer.from(\"fake_video_bytes_1\") },\n { id: 2, video: Buffer.from(\"fake_video_bytes_2\") },\n ],\n { schema: legacyBlobSchema },\n);\nconst legacyBlobTable = await db.createTable(\n \"legacy_videos\",\n legacyBlobData,\n {\n mode: \"overwrite\",\n },\n);\n";
|
|
|
|
export const TsLegacyBlobApiSchema = "const legacyBlobSchema = new arrow.Schema([\n new arrow.Field(\"id\", new arrow.Int64()),\n new arrow.Field(\n \"video\",\n new arrow.LargeBinary(),\n true,\n new Map([[\"lance-encoding:blob\", \"true\"]]),\n ),\n]);\n";
|
|
|
|
export const TsMultimodalImports = "import * as arrow from \"apache-arrow\";\nimport { Buffer } from \"node:buffer\";\nimport * as lancedb from \"@lancedb/lancedb\";\n";
|
|
|
|
export const TsProcessResults = "for (const row of results) {\n const imageBytes = row.image_blob as Uint8Array;\n console.log(\n `Retrieved image: ${row.filename}, Byte length: ${imageBytes.length}`,\n );\n}\n";
|
|
|
|
export const TsSearchData = "const queryVector = Array.from({ length: 128 }, (_, i) => (i % 16) / 16);\nconst results = await tbl.search(queryVector).limit(1).toArray();\n";
|
|
|
|
export const RsBlobApiFetch = "let payloads = blob_table.fetch_blobs(\"image\", &row_ids).await.unwrap();\n";
|
|
|
|
export const RsBlobApiFiles = "let batches = blob_table\n .query()\n .with_row_id()\n .execute()\n .await\n .unwrap()\n .try_collect::<Vec<_>>()\n .await\n .unwrap();\nlet row_ids = batches[0]\n .column_by_name(\"_rowid\")\n .unwrap()\n .as_any()\n .downcast_ref::<UInt64Array>()\n .unwrap()\n .values()\n .to_vec();\n\nlet handles = blob_table\n .fetch_blob_files(\"image\", &row_ids[..1])\n .await\n .unwrap();\nlet header = handles[0].as_ref().unwrap().read_range(0..6).await.unwrap();\nhandles[0].as_ref().unwrap().close().await.unwrap();\n";
|
|
|
|
export const RsBlobApiImports = "use std::sync::Arc;\n\nuse arrow_array::{\n Array, Int64Array, LargeBinaryArray, RecordBatch, RecordBatchIterator, StringArray, UInt64Array,\n};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures_util::TryStreamExt;\nuse lancedb::blob::{BlobRangeRequest, blob};\nuse lancedb::connect;\nuse lancedb::database::CreateTableMode;\nuse lancedb::query::{ExecutableQuery, QueryBase};\n";
|
|
|
|
export const RsBlobApiIngest = "let blob_table = db\n .create_empty_table(\"photos\", blob_schema)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n\nlet input_schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"label\", DataType::Utf8, true),\n Field::new(\"image\", DataType::LargeBinary, true),\n]));\nlet input_batch = RecordBatch::try_new(\n input_schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1_i64, 2_i64])),\n Arc::new(StringArray::from(vec![Some(\"door\"), Some(\"window\")])),\n Arc::new(LargeBinaryArray::from_iter_values([\n b\"inline_image_bytes\".as_slice(),\n b\"second_image_bytes\".as_slice(),\n ])),\n ],\n)\n.unwrap();\nlet input_reader: Box<dyn arrow_array::RecordBatchReader + Send> = Box::new(\n RecordBatchIterator::new(vec![Ok(input_batch)].into_iter(), input_schema),\n);\nblob_table.add(input_reader).execute().await.unwrap();\n";
|
|
|
|
export const RsBlobApiRanges = "let slices = blob_table\n .fetch_blob_ranges(\n \"image\",\n [\n BlobRangeRequest::new(row_ids[0], 0, 6),\n BlobRangeRequest::new(row_ids[1], 0, 6),\n BlobRangeRequest::new(row_ids[1], 7, 5),\n ],\n )\n .await\n .unwrap();\n\nlet second_handles = blob_table\n .fetch_blob_files(\"image\", &row_ids[1..2])\n .await\n .unwrap();\nlet second_image = second_handles[0].as_ref().unwrap();\nlet parts = second_image.read_ranges(&[0..6, 7..12]).await.unwrap();\nsecond_image.close().await.unwrap();\n";
|
|
|
|
export const RsBlobApiSchema = "let blob_schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"label\", DataType::Utf8, true),\n blob(\"image\", true),\n]));\n";
|
|
|
|
export const RsBlobApiSetup = "let db = connect(\"./blob_db\").execute().await.unwrap();\n";
|
|
|
|
export const RsCreateDummyData = "let create_dummy_image = |color: u8| -> Vec<u8> {\n let mut png_like = vec![137, 80, 78, 71, 13, 10, 26, 10];\n png_like.push(color);\n png_like\n};\n\nlet data = vec![\n (\n 1_i32,\n \"red_square.png\",\n vec![0.1_f32; 128],\n create_dummy_image(1),\n \"red\",\n ),\n (\n 2_i32,\n \"blue_square.png\",\n vec![0.2_f32; 128],\n create_dummy_image(2),\n \"blue\",\n ),\n];\n";
|
|
|
|
export const RsDefineSchema = "let schema = Schema::new(vec![\n Field::new(\"id\", DataType::Int32, false),\n Field::new(\"filename\", DataType::Utf8, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 128),\n false,\n ),\n Field::new(\"image_blob\", DataType::Binary, false),\n Field::new(\"label\", DataType::Utf8, false),\n]);\n";
|
|
|
|
export const RsIngestData = "let schema = Arc::new(schema);\nlet image_batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int32Array::from_iter_values(data.iter().map(|row| row.0))),\n Arc::new(StringArray::from_iter_values(data.iter().map(|row| row.1))),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(\n data.iter()\n .map(|row| Some(row.2.iter().copied().map(Some).collect::<Vec<_>>())),\n 128,\n ),\n ),\n Arc::new(BinaryArray::from_iter_values(\n data.iter().map(|row| row.3.as_slice()),\n )),\n Arc::new(StringArray::from_iter_values(data.iter().map(|row| row.4))),\n ],\n)\n.unwrap();\nlet image_reader: Box<dyn arrow_array::RecordBatchReader + Send> = Box::new(\n RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), schema.clone()),\n);\nlet table = db\n .create_table(\"images\", image_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n";
|
|
|
|
export const RsLegacyBlobApiIngest = "let legacy_blob_rows = vec![\n (1_i64, b\"fake_video_bytes_1\".to_vec()),\n (2_i64, b\"fake_video_bytes_2\".to_vec()),\n];\n\nlet legacy_blob_schema = Arc::new(legacy_blob_schema);\nlet legacy_blob_batch = RecordBatch::try_new(\n legacy_blob_schema.clone(),\n vec![\n Arc::new(Int64Array::from_iter_values(\n legacy_blob_rows.iter().map(|row| row.0),\n )),\n Arc::new(LargeBinaryArray::from_iter_values(\n legacy_blob_rows.iter().map(|row| row.1.as_slice()),\n )),\n ],\n)\n.unwrap();\nlet legacy_blob_reader: Box<dyn arrow_array::RecordBatchReader + Send> = Box::new(\n RecordBatchIterator::new(vec![Ok(legacy_blob_batch)].into_iter(), legacy_blob_schema),\n);\nlet legacy_blob_table = db\n .create_table(\"legacy_videos\", legacy_blob_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n";
|
|
|
|
export const RsLegacyBlobApiSchema = "let legacy_blob_metadata = HashMap::from([(\n \"lance-encoding:blob\".to_string(),\n \"true\".to_string(),\n)]);\nlet legacy_blob_schema = Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"video\", DataType::LargeBinary, true).with_metadata(legacy_blob_metadata),\n]);\n";
|
|
|
|
export const RsMultimodalImports = "use std::collections::HashMap;\nuse std::sync::Arc;\n\nuse arrow_array::types::Float32Type;\nuse arrow_array::{\n Array, BinaryArray, FixedSizeListArray, Int32Array, Int64Array, LargeBinaryArray, RecordBatch,\n RecordBatchIterator, StringArray,\n};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures_util::TryStreamExt;\nuse lancedb::connect;\nuse lancedb::database::CreateTableMode;\nuse lancedb::query::{ExecutableQuery, QueryBase};\n";
|
|
|
|
export const RsProcessResults = "for batch in &results {\n let filenames = batch\n .column_by_name(\"filename\")\n .unwrap()\n .as_any()\n .downcast_ref::<StringArray>()\n .unwrap();\n let images = batch\n .column_by_name(\"image_blob\")\n .unwrap()\n .as_any()\n .downcast_ref::<BinaryArray>()\n .unwrap();\n\n for row in 0..batch.num_rows() {\n let image_bytes = images.value(row);\n println!(\n \"Retrieved image: {}, Byte length: {}\",\n filenames.value(row),\n image_bytes.len()\n );\n }\n}\n";
|
|
|
|
export const RsSearchData = "let query_vector = vec![0.1_f32; 128];\nlet results = table\n .query()\n .nearest_to(query_vector)\n .unwrap()\n .limit(1)\n .execute()\n .await\n .unwrap()\n .try_collect::<Vec<_>>()\n .await\n .unwrap();\n";
|
|
|