mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-13 14:16:54 +00:00
Jack YeandClaude Sonnet 5
de95ed3f6d
A broad documentation pass across the areas LanceDB Enterprise and Embedded diverge on most: **Data Curation** -- GPU Function execution, Slurm scheduling (and what Slurm is, for readers new to it), blob late materialization as its own page, joins, aggregation/profiling, sampling, near-duplicate detection, chunking, distributed shuffle, and IVF-partition clustering. **Feature Engineering** -- computed columns and materialized views, restructured to lead with the Function-based path and show the SQL form side by side with Python throughout; a single Auto Refresh page instead of duplicating scheduling details on every refresh page; adaptive execution; checkpointing; GPU and Slurm placement for refresh workers; a Training Cache page for the PyTorch data loader's automatic cache benefit. **Security & Governance** -- an Audit Log reference (schema, access, grants), Secrets promoted to an ordinary catalog object with their own SQL DDL and a vending/rotation guide, and privilege-table fixes verified against the authorization source. **Deployment & Maintenance** -- Horizontal Scaling and OpenTelemetry moved into a real Deployment area; Automated Cleanup folded into Intelligent Auto-Maintenance (every other maintenance decision already lived there); RTE Compaction and Advanced Conflict Resolution documented in depth. **Landing page** -- the Enterprise feature comparison table reorganized (several redundant or stale rows removed, several new areas added), and the "why Enterprise" bullets replaced with ones tied to a concrete page (Performance, Scale, Fault tolerance, Multicloud deployment, Simplify operations) instead of vague promises. Plus a full pass addressing lancedb-gatekeeper's review: a real test-safety bug (a quickstart test could silently overwrite and delete a deployment's own `frames` table), a fabricated SQL statement with no backing implementation, an actual dedup-guarantee bug in `take_row_ids`, and dozens of documentation-accuracy fixes verified against source rather than taken on faith -- each either fixed, or left with an explicit note on why it needs a feature or deployment owner's sign-off instead of being guessed at. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_018PXMCbRH6i5w47TsjeEUMT --------- Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
25 lines
2.2 KiB
Plaintext
25 lines
2.2 KiB
Plaintext
{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */}
|
|
|
|
export const PyFunctionApply = "from lancedb import col\n\ntable.add_columns({\"doubled\": version(value=col(\"value\"))})\n";
|
|
|
|
export const PyFunctionClassApply = "from lancedb import col\n\napplication = embedder(text=col(\"text\"), model=\"all-mpnet-base-v2\")\n";
|
|
|
|
export const PyFunctionClassDefine = "@udf(pip=[\"sentence-transformers\"])\nclass embed:\n def __init__(self, model: str = \"all-MiniLM-L6-v2\"):\n # Runs on the worker, once per instance: only the worker needs the package.\n from sentence_transformers import SentenceTransformer\n\n self.model = SentenceTransformer(model)\n\n def __call__(self, text: str) -> list[float]:\n return self.model.encode(text).tolist()\n\n def close(self):\n del self.model\n";
|
|
|
|
export const PyFunctionClassRegister = "embedder = db.create_function_async(embed).wait()\n";
|
|
|
|
export const PyFunctionConnect = "import os\n\nimport lancedb\n\ndb = lancedb.connect(\n uri=os.environ[\"LANCEDB_URI\"], # db://your-database\n api_key=os.environ[\"LANCEDB_API_KEY\"],\n region=os.environ.get(\"LANCEDB_REGION\", \"us-east-1\"),\n host_override=os.environ[\"LANCEDB_HOST_OVERRIDE\"],\n sql_host_override=os.environ[\"LANCEDB_SQL_HOST_OVERRIDE\"],\n)\n";
|
|
|
|
export const PyFunctionCreateTable = "import pyarrow as pa\n\nschema = pa.schema([pa.field(\"value\", pa.float64(), nullable=False)])\ntable = db.create_table(\n \"function_demo\",\n [{\"value\": 1.0}, {\"value\": 2.0}, {\"value\": 3.0}],\n schema=schema,\n)\n";
|
|
|
|
export const PyFunctionDefine = "from lancedb import udf\n\n\n@udf\ndef double(value: float) -> float:\n return value * 2\n";
|
|
|
|
export const PyFunctionInspect = "rows = table.search().select([\"value\", \"doubled\"]).to_list()\nprint(sorted((row[\"value\"], row[\"doubled\"]) for row in rows))\n# [(1.0, 2.0), (2.0, 4.0), (3.0, 6.0)]\n";
|
|
|
|
export const PyFunctionLocalCall = "print(double(1.5))\n# 3.0\n";
|
|
|
|
export const PyFunctionRefresh = "refresh = table.refresh_column_async(\"doubled\")\nresult = refresh.wait()\nprint(result.rows_assigned)\n# 3\n";
|
|
|
|
export const PyFunctionRegister = "job = db.create_function_async(double)\nversion = job.wait()\n";
|
|
|