mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-30 23:36:52 +00:00
Jack Ye
ddb318567a
## Summary - Complete the SQL reference: `ATTACH DATABASE`, the `CHANGES` clause, and `dedupe()`/`duplicate_pairs()` (verified against current sophon source; these were recently renamed from `vector_dedup`/`vector_duplicate_pairs`). Fix the blob-column default-projection claim and add the previously undocumented `blob_descriptor()`. - Enrich the gated Functions draft (still `.mintignore`'d, pre-release): chained Functions over blob outputs, GPU-produced blobs, Secrets, multi-input/output binding, `code=[module]` packaging. - Upgrade the materialized-view and computed-column auto-refresh drafts with precise, verified catch-up semantics, and add the missing `ALTER MATERIALIZED VIEW` / `CREATE ... WITH(...)` sections (also gated — the automatic-refresh scheduler is still dev/staging-only). - Thread one running example (video frames → GPU extract/caption → dedup → auto-refreshing materialized view) through Functions, materialized views, and the LeRobotDataset integration guide instead of a separate tutorial. - Rename "LanceDB OSS"/"OSS" to "LanceDB Embedded"/"Embedded" throughout (52 files); activate the existing availability-comparison mechanism on `basics/offerings.mdx` for a full feature-by-feature comparison table.
67 lines
16 KiB
Plaintext
67 lines
16 KiB
Plaintext
{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */}
|
|
|
|
export const PyQuickstartAddData = "table.add(\n [\n {\n \"frame_id\": \"4\",\n \"video_id\": \"clip-2\",\n \"caption\": \"a gripper lowering a cup onto the table\",\n \"stats\": {\"motion\": 5, \"sharpness\": 4, \"exposure\": 4, \"framing\": 2},\n \"embedding\": [0.3, 0.9, 0.6, 0.8],\n }\n ]\n)\n";
|
|
|
|
export const PyQuickstartAddFeature = "table.add_columns(\n {\n \"quality_score\": \"cast(((stats.motion + stats.sharpness + stats.exposure + stats.framing) / 4.0) as float)\"\n }\n)\n";
|
|
|
|
export const PyQuickstartComputedFeature = "table.add_columns(computed={\"caption_length\": \"length(caption)\"})\ntable.refresh_column(\"caption_length\")\n";
|
|
|
|
export const PyQuickstartComputedFeatureEnterprise = "table.add_columns(computed={\"caption_length\": \"length(caption)\"})\njob = table.refresh_column_async(\"caption_length\")\njob.wait()\n";
|
|
|
|
export const PyQuickstartCreateTable = "table = db.create_table(\"frames\", data=data, mode=\"overwrite\")\n";
|
|
|
|
export const PyQuickstartCreateTableAsync = "async_table = await async_db.create_table(\n \"frames\",\n data=data,\n mode=\"overwrite\",\n)\n";
|
|
|
|
export const PyQuickstartCurateWithMetadata = "curated = (\n table.search(query_vector)\n .where(\"stats.motion >= 4\")\n .select([\"frame_id\", \"video_id\", \"caption\", \"_distance\"])\n .limit(2)\n .to_polars()\n)\nprint(curated)\n";
|
|
|
|
export const PyQuickstartData = "data = [\n {\n \"frame_id\": \"1\",\n \"video_id\": \"clip-1\",\n \"caption\": \"a robot arm reaching toward a cup\",\n \"stats\": {\"motion\": 3, \"sharpness\": 4, \"exposure\": 4, \"framing\": 3},\n \"embedding\": [0.7, 0.1, 0.9, 0.7],\n },\n {\n \"frame_id\": \"2\",\n \"video_id\": \"clip-1\",\n \"caption\": \"a two-finger gripper closing on a cup\",\n \"stats\": {\"motion\": 5, \"sharpness\": 4, \"exposure\": 5, \"framing\": 2},\n \"embedding\": [0.2, 0.9, 0.4, 0.9],\n },\n {\n \"frame_id\": \"3\",\n \"video_id\": \"clip-2\",\n \"caption\": \"a robot arm holding a cup steady\",\n \"stats\": {\"motion\": 2, \"sharpness\": 3, \"exposure\": 3, \"framing\": 4},\n \"embedding\": [0.9, 0.1, 0.5, 0.4],\n },\n]\n";
|
|
|
|
export const PyQuickstartMultimodalBytes = "from pathlib import Path\n\nimage_bytes = Path(\"data/sample-frame.jpg\").read_bytes()\n\nmultimodal_table = db.create_table(\n \"frame_images\",\n data=[\n {\n \"frame_id\": \"thumb-1\",\n \"caption\": \"Sample frame thumbnail\",\n \"image\": image_bytes,\n \"embedding\": [0.9, 0.1, 0.5, 0.4],\n }\n ],\n mode=\"overwrite\",\n)\n";
|
|
|
|
export const PyQuickstartOutputPandas = "# Ensure you run `pip install pandas` beforehand\nresult = table.search(query_vector).limit(2).to_pandas()\nprint(result)\n";
|
|
|
|
export const PyQuickstartQueryFeature = "features = table.search().select([\"frame_id\", \"video_id\", \"quality_score\"]).to_polars()\nprint(features)\n";
|
|
|
|
export const PyQuickstartTraining = "import torch\nfrom lancedb.permutation import Permutation\n\ntable.add_columns(\n {\n \"motion\": \"cast(stats.motion as float)\",\n \"sharpness\": \"cast(stats.sharpness as float)\",\n }\n)\ndataset = Permutation.identity(table).select_columns([\"motion\", \"sharpness\"])\ndataloader = torch.utils.data.DataLoader(dataset, batch_size=1024, shuffle=True)\n";
|
|
|
|
export const PyQuickstartVectorSearch1 = "# Search for frames similar to \"a gripper closing on an object\"\nquery_vector = [0.2, 0.8, 0.4, 0.9]\n\n# Ensure you run `pip install polars` beforehand\nresult = (\n table.search(query_vector)\n .select([\"frame_id\", \"video_id\", \"caption\", \"_distance\"])\n .limit(2)\n .to_polars()\n)\nprint(result)\n";
|
|
|
|
export const PyQuickstartVectorSearch1Async = "# Search for frames similar to \"a gripper closing on an object\"\nquery_vector = [0.2, 0.8, 0.4, 0.9]\n\n# Ensure you run `pip install polars` beforehand\nasync_result = await (\n await async_table.search(query_vector)\n).select([\"frame_id\", \"video_id\", \"caption\", \"_distance\"]).limit(2).to_polars()\nprint(async_result)\n";
|
|
|
|
export const TsQuickstartAddData = "await table.add([\n {\n frame_id: \"4\",\n video_id: \"clip-2\",\n caption: \"a gripper lowering a cup onto the table\",\n stats: { motion: 5, sharpness: 4, exposure: 4, framing: 2 },\n embedding: [0.3, 0.9, 0.6, 0.8],\n },\n]);\n";
|
|
|
|
export const TsQuickstartAddFeature = "await table.addColumns([\n {\n name: \"quality_score\",\n valueSql:\n \"cast(((stats.motion + stats.sharpness + stats.exposure + stats.framing) / 4.0) as float)\",\n },\n]);\n";
|
|
|
|
export const TsQuickstartCreateTable = "const table = await db.createTable(\"frames\", data, { mode: \"overwrite\" });\n";
|
|
|
|
export const TsQuickstartCurateWithMetadata = "const curated = await table\n .search(queryVector)\n .where(\"stats.motion >= 4\")\n .select([\"frame_id\", \"video_id\", \"caption\", \"_distance\"])\n .limit(2)\n .toArray();\nconsole.table(curated);\n";
|
|
|
|
export const TsQuickstartData = "const data = [\n {\n frame_id: \"1\",\n video_id: \"clip-1\",\n caption: \"a robot arm reaching toward a cup\",\n stats: { motion: 3, sharpness: 4, exposure: 4, framing: 3 },\n embedding: [0.7, 0.1, 0.9, 0.7],\n },\n {\n frame_id: \"2\",\n video_id: \"clip-1\",\n caption: \"a two-finger gripper closing on a cup\",\n stats: { motion: 5, sharpness: 4, exposure: 5, framing: 2 },\n embedding: [0.2, 0.9, 0.4, 0.9],\n },\n {\n frame_id: \"3\",\n video_id: \"clip-2\",\n caption: \"a robot arm holding a cup steady\",\n stats: { motion: 2, sharpness: 3, exposure: 3, framing: 4 },\n embedding: [0.9, 0.1, 0.5, 0.4],\n },\n];\n";
|
|
|
|
export const TsQuickstartMultimodalBytes = "const arrow = await import(\"apache-arrow\");\nconst { readFile } = await import(\"node:fs/promises\");\n\nconst imageBytes = await readFile(\"data/sample-frame.jpg\");\nconst imageSchema = new arrow.Schema([\n new arrow.Field(\"frame_id\", new arrow.Utf8()),\n new arrow.Field(\"caption\", new arrow.Utf8()),\n new arrow.Field(\"image\", new arrow.Binary()),\n new arrow.Field(\n \"embedding\",\n new arrow.FixedSizeList(\n 4,\n new arrow.Field(\"item\", new arrow.Float32(), true),\n ),\n ),\n]);\nconst imageData = lancedb.makeArrowTable(\n [\n {\n frame_id: \"thumb-1\",\n caption: \"Sample frame thumbnail\",\n image: imageBytes,\n embedding: [0.9, 0.1, 0.5, 0.4],\n },\n ],\n { schema: imageSchema },\n);\nconst multimodalTable = await db.createTable(\n \"frame_images\",\n imageData,\n { mode: \"overwrite\" },\n);\n";
|
|
|
|
export const TsQuickstartQueryFeature = "const features = await table\n .query()\n .select([\"frame_id\", \"video_id\", \"quality_score\"])\n .toArray();\nconsole.table(features);\n";
|
|
|
|
export const TsQuickstartVectorSearch1 = "// Search for frames similar to \"a gripper closing on an object\"\nconst queryVector = [0.2, 0.8, 0.4, 0.9];\n\nconst result = await table\n .search(queryVector)\n .select([\"frame_id\", \"video_id\", \"caption\", \"_distance\"])\n .limit(2)\n .toArray();\nconsole.table(result);\n";
|
|
|
|
export const RsQuickstartAddData = "let more_data = vec![Frame {\n frame_id: \"4\".to_string(),\n video_id: \"clip-2\".to_string(),\n caption: \"a gripper lowering a cup onto the table\".to_string(),\n stats: Stats {\n motion: 5,\n sharpness: 4,\n exposure: 4,\n framing: 2,\n },\n embedding: [0.3, 0.9, 0.6, 0.8],\n}];\ntable\n .add(frames_to_reader(schema.clone(), &more_data))\n .execute()\n .await\n .unwrap();\n";
|
|
|
|
export const RsQuickstartAddFeature = "table\n .add_columns(\n NewColumnTransform::SqlExpressions(vec![(\n \"quality_score\".to_string(),\n \"cast(((stats.motion + stats.sharpness + stats.exposure + stats.framing) / 4.0) as float)\"\n .to_string(),\n )]),\n None,\n )\n .await\n .unwrap();\n";
|
|
|
|
export const RsQuickstartCreateTable = "use lancedb::database::CreateTableMode;\n\nlet schema = frames_schema();\nlet table = db\n .create_table(\"frames\", frames_to_reader(schema.clone(), &data))\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n";
|
|
|
|
export const RsQuickstartCurateWithMetadata = "let curated = table\n .query()\n .nearest_to(&query_vector)\n .unwrap()\n .only_if(\"stats.motion >= 4\")\n .select(Select::Columns(vec![\n \"frame_id\".to_string(),\n \"video_id\".to_string(),\n \"caption\".to_string(),\n \"_distance\".to_string(),\n ]))\n .limit(2)\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{curated:?}\");\n";
|
|
|
|
export const RsQuickstartData = "let data = vec![\n Frame {\n frame_id: \"1\".to_string(),\n video_id: \"clip-1\".to_string(),\n caption: \"a robot arm reaching toward a cup\".to_string(),\n stats: Stats {\n motion: 3,\n sharpness: 4,\n exposure: 4,\n framing: 3,\n },\n embedding: [0.7, 0.1, 0.9, 0.7],\n },\n Frame {\n frame_id: \"2\".to_string(),\n video_id: \"clip-1\".to_string(),\n caption: \"a two-finger gripper closing on a cup\".to_string(),\n stats: Stats {\n motion: 5,\n sharpness: 4,\n exposure: 5,\n framing: 2,\n },\n embedding: [0.2, 0.9, 0.4, 0.9],\n },\n Frame {\n frame_id: \"3\".to_string(),\n video_id: \"clip-2\".to_string(),\n caption: \"a robot arm holding a cup steady\".to_string(),\n stats: Stats {\n motion: 2,\n sharpness: 3,\n exposure: 3,\n framing: 4,\n },\n embedding: [0.9, 0.1, 0.5, 0.4],\n },\n];\n";
|
|
|
|
export const RsQuickstartDefineStruct = "use std::sync::Arc;\n\nuse lancedb::arrow::arrow_array::types::Float32Type;\nuse lancedb::arrow::arrow_array::{\n FixedSizeListArray, Int8Array, LargeStringArray, RecordBatch, RecordBatchIterator,\n RecordBatchReader, StructArray,\n};\nuse lancedb::arrow::arrow_schema::{DataType, Field, FieldRef, Fields, Schema};\n\n// Define structs representing the data schema\n#[derive(Debug, Clone)]\nstruct Stats {\n motion: i8,\n sharpness: i8,\n exposure: i8,\n framing: i8,\n}\n\n#[derive(Debug, Clone)]\nstruct Frame {\n frame_id: String,\n video_id: String,\n caption: String,\n stats: Stats,\n embedding: [f32; 4],\n}\n\nfn frames_schema() -> Arc<Schema> {\n Arc::new(Schema::new(vec![\n Field::new(\"frame_id\", DataType::LargeUtf8, false),\n Field::new(\"video_id\", DataType::LargeUtf8, false),\n Field::new(\"caption\", DataType::LargeUtf8, false),\n Field::new(\n \"stats\",\n DataType::Struct(Fields::from(vec![\n Arc::new(Field::new(\"motion\", DataType::Int8, false)),\n Arc::new(Field::new(\"sharpness\", DataType::Int8, false)),\n Arc::new(Field::new(\"exposure\", DataType::Int8, false)),\n Arc::new(Field::new(\"framing\", DataType::Int8, false)),\n ])),\n false,\n ),\n Field::new(\n \"embedding\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 4),\n false,\n ),\n ]))\n}\n\nfn frames_to_reader(\n schema: Arc<Schema>,\n rows: &[Frame],\n) -> Box<dyn RecordBatchReader + Send> {\n let frame_ids = LargeStringArray::from_iter_values(rows.iter().map(|row| row.frame_id.as_str()));\n let video_ids = LargeStringArray::from_iter_values(rows.iter().map(|row| row.video_id.as_str()));\n let captions =\n LargeStringArray::from_iter_values(rows.iter().map(|row| row.caption.as_str()));\n\n let motion = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.motion));\n let sharpness = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.sharpness));\n let exposure = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.exposure));\n let framing = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.framing));\n let stats_fields: Vec<FieldRef> = vec![\n Arc::new(Field::new(\"motion\", DataType::Int8, false)),\n Arc::new(Field::new(\"sharpness\", DataType::Int8, false)),\n Arc::new(Field::new(\"exposure\", DataType::Int8, false)),\n Arc::new(Field::new(\"framing\", DataType::Int8, false)),\n ];\n let stats = StructArray::new(\n stats_fields.into(),\n vec![\n Arc::new(motion),\n Arc::new(sharpness),\n Arc::new(exposure),\n Arc::new(framing),\n ],\n None,\n );\n\n let embeddings = FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(\n rows.iter()\n .map(|row| Some(row.embedding.iter().copied().map(Some).collect::<Vec<_>>())),\n 4,\n );\n\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(frame_ids),\n Arc::new(video_ids),\n Arc::new(captions),\n Arc::new(stats),\n Arc::new(embeddings),\n ],\n )\n .unwrap();\n\n Box::new(RecordBatchIterator::new(\n vec![Ok(batch)].into_iter(),\n schema,\n ))\n}\n";
|
|
|
|
export const RsQuickstartMultimodalBytes = "use lancedb::arrow::arrow_array::BinaryArray;\n\nlet image_bytes = std::fs::read(\"data/sample-frame.jpg\").unwrap();\n\nlet image_schema = Arc::new(Schema::new(vec![\n Field::new(\"frame_id\", DataType::LargeUtf8, false),\n Field::new(\"caption\", DataType::LargeUtf8, false),\n Field::new(\"image\", DataType::Binary, false),\n Field::new(\n \"embedding\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 4),\n false,\n ),\n]));\nlet image_embeddings = [[0.9_f32, 0.1, 0.5, 0.4]];\nlet image_batch = RecordBatch::try_new(\n image_schema.clone(),\n vec![\n Arc::new(LargeStringArray::from_iter_values([\"thumb-1\"])),\n Arc::new(LargeStringArray::from_iter_values([\n \"Sample frame thumbnail\",\n ])),\n Arc::new(BinaryArray::from_iter_values([image_bytes.as_slice()])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(\n image_embeddings\n .iter()\n .map(|vector| Some(vector.iter().copied().map(Some).collect::<Vec<_>>())),\n 4,\n ),\n ),\n ],\n)\n.unwrap();\nlet image_reader: Box<dyn RecordBatchReader + Send> = Box::new(RecordBatchIterator::new(\n vec![Ok(image_batch)].into_iter(),\n image_schema,\n));\nlet multimodal_table = db\n .create_table(\"frame_images\", image_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n";
|
|
|
|
export const RsQuickstartOutputArray = "let result = table\n .query()\n .nearest_to(&query_vector)\n .unwrap()\n .select(Select::Columns(vec![\n \"frame_id\".to_string(),\n \"video_id\".to_string(),\n \"caption\".to_string(),\n \"_distance\".to_string(),\n ]))\n .limit(2)\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{result:?}\");\n";
|
|
|
|
export const RsQuickstartQueryFeature = "let features = table\n .query()\n .select(Select::Columns(vec![\n \"frame_id\".to_string(),\n \"video_id\".to_string(),\n \"quality_score\".to_string(),\n ]))\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{features:?}\");\n";
|
|
|
|
export const RsQuickstartVectorSearch1 = "use lancedb::arrow::IntoPolars;\nuse lancedb::query::{ExecutableQuery, QueryBase, Select};\n\n// Search for frames similar to \"a gripper closing on an object\"\nlet query_vector = [0.2, 0.8, 0.4, 0.9];\n\nlet result = table\n .query()\n .nearest_to(&query_vector)\n .unwrap()\n .select(Select::Columns(vec![\n \"frame_id\".to_string(),\n \"video_id\".to_string(),\n \"caption\".to_string(),\n \"_distance\".to_string(),\n ]))\n .limit(2)\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{result:?}\");\n";
|
|
|