From c12a6dce9f8df52ee4650d42faa73d1883dec362 Mon Sep 17 00:00:00 2001 From: prrao87 <35005448+prrao87@users.noreply.github.com> Date: Tue, 14 Jul 2026 10:57:36 -0400 Subject: [PATCH] Revert "add LanceDB agent skill for portable pipelines" This reverts commit 8ea78e3fbcb26718112ab4ddec55a91804b869d3. --- .agents/skills/lancedb/SKILL.md | 79 -------- .../references/python/api_reference.md | 105 ----------- .../lancedb/references/python/patterns.md | 173 ------------------ .../lancedb/references/python/performance.md | 131 ------------- .../references/typescript/api_reference.md | 78 -------- .../lancedb/references/typescript/patterns.md | 100 ---------- .../references/typescript/performance.md | 78 -------- .../lancedb/scripts/check_materialization.py | 135 -------------- 8 files changed, 879 deletions(-) delete mode 100644 .agents/skills/lancedb/SKILL.md delete mode 100644 .agents/skills/lancedb/references/python/api_reference.md delete mode 100644 .agents/skills/lancedb/references/python/patterns.md delete mode 100644 .agents/skills/lancedb/references/python/performance.md delete mode 100644 .agents/skills/lancedb/references/typescript/api_reference.md delete mode 100644 .agents/skills/lancedb/references/typescript/patterns.md delete mode 100644 .agents/skills/lancedb/references/typescript/performance.md delete mode 100644 .agents/skills/lancedb/scripts/check_materialization.py diff --git a/.agents/skills/lancedb/SKILL.md b/.agents/skills/lancedb/SKILL.md deleted file mode 100644 index 2ed62fc39..000000000 --- a/.agents/skills/lancedb/SKILL.md +++ /dev/null @@ -1,79 +0,0 @@ ---- -name: lancedb -description: Use when writing, reviewing, debugging, or documenting LanceDB pipelines in Python or TypeScript, especially code that should work across local LanceDB OSS tables and remote LanceDB Enterprise/Cloud tables. Helps avoid non-portable full-table materialization, choose idiomatic query/search patterns, and apply LanceDB performance defaults for ingestion, indexing, filtering, and diagnostics. ---- - -# Building LanceDB Pipelines - -Use this skill to produce LanceDB pipelines that are portable between local and remote tables (for LanceDB Enterprise/Cloud) and idiomatic for the selected SDK. - -## LanceDB Table Modes - -LanceDB has two common execution modes: - -- **Local table**: embedded, open source, in-process LanceDB. The client opens data from a local path or object storage URI and executes queries in the application process. -- **Remote table**: LanceDB Enterprise/Cloud table opened through a `db://...` URI. The data may be very large, commonly backed by object storage, and queried through a remote service. - -Do NOT assume local-only table helpers exist on remote tables. If the user asks for LanceDB Enterprise, Cloud, `db://...`, production remote access, or a remote table, focus on the remote table path: use `search()` / `query()`, keep reads bounded with `select()` and `limit()`, and avoid table-level full materialization APIs. - -## Workflow - -1. Identify the SDK: Python, TypeScript, or both. -2. Identify the table mode: local/embedded OSS, remote Enterprise/Cloud, or portable across both. If the user says "LanceDB Enterprise", choose the remote table path. -3. Read the matching language branch before writing or changing code: - - Python patterns: `references/python/patterns.md` - - Python API quick reference: `references/python/api_reference.md` - - Python performance guidance: `references/python/performance.md` - - TypeScript patterns: `references/typescript/patterns.md` - - TypeScript API quick reference: `references/typescript/api_reference.md` - - TypeScript performance guidance: `references/typescript/performance.md` -4. Start with `patterns.md` for the selected SDK. Read `api_reference.md` when choosing method names or return collectors. Read `performance.md` when the task involves ingestion, indexing, filtering, query tuning, diagnostics, or large datasets. -5. For Python schemas, favor Pydantic models and validate records before writing. Use PyArrow schemas when Arrow-native, streaming, or highly dynamic data makes them materially better suited. -6. Prefer `search()` or `query()` builders with explicit `select()` and `limit()` for reads. -7. Avoid table-level full materialization in remote or portable code. This is the main local-vs-remote read pitfall. -8. After a successful embedded OSS ingestion, call `table.optimize()`. Do not call it for Enterprise/Cloud; remote maintenance is automatic. -9. For remote Enterprise/Cloud writes, never drop-then-reuse or `mode="overwrite"` the same table name — see "Enterprise: never drop-then-reuse the same table name" below. This is the main local-vs-remote write pitfall. -10. If reviewing an existing file or repo, run `scripts/check_materialization.py` on the relevant paths and inspect each finding before editing. -11. Cross-check unfamiliar or non-trivial API claims against the source tree instead of relying on memory. - -## Core Portability Rule - -Do not write code that assumes a local table API will exist on a remote table. Remote tables can be very large, so whole-table materialization helpers are intentionally unavailable or unsafe. - -This does **not** mean result conversion is forbidden. Bounded query/search result collection is normal: - -- Python: `table.search(...).select([...]).limit(10).to_pandas()` -- TypeScript: `await table.search(...).select([...]).limit(10).toArray()` - -The unsafe pattern is table-level or unbounded collection, plus local-only dataset escape hatches in remote code: - -- Python: `table.to_pandas()`, `table.to_arrow()`, `table.to_polars()`; `table.to_lance()` is local/OSS-only dataset access, not materialization -- TypeScript: `await table.toArrow()`, `await table.query().toArray()` without `limit()` - -## Enterprise: never drop-then-reuse the same table name - -LanceDB Enterprise/Cloud splits a **control plane** (DDL: create/drop/rename) from a **data plane** (query nodes that serve reads). Query nodes cache the resolved dataset for a table name for up to `table_cache_ttl` — **default 300 seconds (5 minutes)**. After you drop or overwrite a table, the control plane updates immediately but the data plane keeps serving the *old* dataset until that cache entry expires. During the window the two planes disagree. - -The failure this causes: you `drop_table("t")` then immediately `create_table("t", ...)` (or `create_table("t", ..., mode="overwrite")`). The DDL returns success, but every query against `t` returns **`500 Internal Server Error`** (the query node resolves the stale/deleted dataset), and a fresh `describe` may still show the *old* schema/version. It looks like your write silently failed; it didn't — the name is cached. - -**`mode="overwrite"` has the same problem** — it is a drop+create of the same name under the hood. - -Rules for portable Enterprise ingestion: - -1. **Never reuse a table name you just dropped/overwrote within the cache TTL.** Do not use `mode="overwrite"` to replace an existing Enterprise table in place. -2. To (re)load data, **write to a fresh table name** (e.g. `_v2`, or a run-stamped suffix). A brand-new name has no cached data-plane entry, so writes and reads work immediately. -3. Before creating, `list_tables()` and **fail loudly if the name already exists** rather than overwriting — prompt for a new name. -4. To land on a specific final name that is currently occupied by an old table: drop the old table, **wait out the TTL (~5 min), then `rename_table(fresh_name, final_name)`**. Renaming onto a name whose old dataset is still cached hits the same race, so the wait is mandatory. `rename_table` is a supported control-plane op. -5. When you hand a table name back to a human, tell them which step still needs the propagation wait (usually: "the old `t` was dropped; run the rename in ~5 minutes"). - -This is Enterprise/Cloud-specific. Local/OSS tables have no separate data plane, so `mode="overwrite"` and immediate same-name reuse are fine there. - -## Script - -Run the scanner when reviewing or modifying an existing codebase: - -```bash -python skills/lancedb/scripts/check_materialization.py path/to/file_or_dir -``` - -The script reports likely unsafe full-table materialization in Python and TypeScript. Treat results as review prompts, not automatic proof of a bug. diff --git a/.agents/skills/lancedb/references/python/api_reference.md b/.agents/skills/lancedb/references/python/api_reference.md deleted file mode 100644 index 2c0e34c72..000000000 --- a/.agents/skills/lancedb/references/python/api_reference.md +++ /dev/null @@ -1,105 +0,0 @@ -# Python API Reference - -Quick method reference for Python LanceDB code. Cross-check source for non-trivial claims. - -## Connect - -```python -import lancedb - -db = lancedb.connect("./camelot-db") # local/OSS -db = lancedb.connect("db://my-db", api_key=api_key, region=region) # remote -``` - -**Place the local database directory next to the script/entrypoint that opens it** (i.e. resolve the path relative to the script, `Path(__file__).parent / "camelot-db"`), not buried under a shared `data/` folder. The Lance dataset is the database, not a data file — keeping it beside its code makes ownership obvious and paths stable regardless of the working directory the script is launched from. - -**Do not name the directory `lancedb`** (e.g. `./lancedb`, `./data/lancedb`). It collides with the imported `lancedb` package name, which is confusing to read and easy to shadow in scripts. Give it a name derived from the repo or dataset with a clear prefix/suffix — for example `./-db`, `./_lancedb`, or `./vectordb`. - -Async: - -```python -db = await lancedb.connect_async("./camelot-db") -``` - -## Table Reads - -| Task | Preferred API | -| --- | --- | -| Vector search | `table.search(query_vector).limit(k)` | -| Full scan with filters/projection (sync) | `table.search().where(...).select(...).limit(...)` | -| Full scan with filters/projection (async) | `table.query().where(...).select(...).limit(...)` | -| Filter | `.where("col > 10")` | -| Projection | `.select(["id", "text"])` | -| Bound result count | `.limit(20)` | -| Collect bounded result as Python objects (default, no extra deps) | `.to_list()` on query/search result | -| Collect bounded result as Arrow (default, `pyarrow` always available) | `.to_arrow()` on query/search result | -| Collect bounded result as pandas (only if project uses pandas) | `.to_pandas()` on query/search result | -| Collect bounded result as Polars (only if project uses polars) | `.to_polars()` on query/search result | - -## Sync vs Async Scan API - -The plain-scan entry point differs between the sync and async clients. **Verified against `lancedb` 0.34.0** — re-check if the pinned version changes: - -- **Sync** (`lancedb.connect(...)`): the table has **no `.query()` method**. Use `.search()` with no argument for a plain scan; it returns a query builder that supports `.where()`, `.select()`, `.limit()`, and the `.to_list()` / `.to_arrow()` / `.to_pandas()` / `.to_polars()` collectors. - ```python - rows = table.search().where("status = 'ready'").select(["id", "text"]).limit(20).to_list() - ``` -- **Async** (`lancedb.connect_async(...)`): the table has **both** `.query()` and `.search()`. Use `.query()` for a plain scan. - ```python - rows = await async_table.query().where("status = 'ready'").select(["id", "text"]).limit(20).to_list() - ``` - -Do not call `table.query()` on a sync table — it raises `AttributeError`. - -## Local vs Remote Table Methods - -| API | Local table | Remote table | Agent guidance | -| --- | --- | --- | --- | -| `table.search(...)` | Yes | Yes | Preferred read path (sync + async) | -| `table.query()` | Async only | Async only | Sync scan path is `table.search()`; `.query()` is the async scan builder | -| `table.to_pandas()` | Yes | No / unsafe for portability | Avoid in portable code | -| `table.to_arrow()` | Yes | No / unsafe for portability | Avoid in portable code | -| `table.to_polars()` | Yes | No / unsafe for portability | Avoid in portable code | -| `table.to_lance()` | Yes | No | Local/OSS escape hatch only | - -## Indexes - -Use `create_index(...)` for vector indexes and modern index configs. Use scalar indexes for filtered or merge keys. - -Common calls: - -```python -table.create_index("vector") -table.create_scalar_index("status") -table.create_fts_index("text") -``` - -Check source docs before specifying advanced index config names or parameters. - -## Filtering And Recall Knobs - -```python -table.search(query_vector).where("status = 'ready'") # pre-filter by default -table.search(query_vector).where("status = 'ready'", prefilter=False) -table.search(query_vector).limit(10).refine_factor(20) -table.search(query_vector).limit(10).nprobes(50) -``` - -Use post-filtering only when fewer than `limit` results are acceptable. - -## Diagnostics - -```python -print(table.search(query_vector).where("year > 2000").limit(10).analyze_plan()) -print(table.index_stats("vector_idx")) -``` - -Use these before changing indexes or search tuning. - -## Maintenance - -```python -table.optimize() -``` - -Call this after every successful local/OSS ingestion. It handles compaction, cleanup of old versions according to retention, and index optimization. Do not add this for LanceDB Enterprise/Cloud remote tables; Enterprise handles compaction and cleanup automatically from cluster configuration. diff --git a/.agents/skills/lancedb/references/python/patterns.md b/.agents/skills/lancedb/references/python/patterns.md deleted file mode 100644 index 4d6be43ef..000000000 --- a/.agents/skills/lancedb/references/python/patterns.md +++ /dev/null @@ -1,173 +0,0 @@ -# Python Patterns - -Use these patterns when writing Python code with `lancedb`. - -## Before Writing Code - -Choose the output type from what the project actually depends on. **Do not assume `pandas` or `polars` is installed** — they are heavy dependencies that many LanceDB projects do not use. `pyarrow`, by contrast, ships as a LanceDB dependency and is always available, so it is a safe default to lean on. - -Default output (after applying `select()` and `limit()`): - -- **Python objects**: `.to_list()` — a list of dicts, no extra dependencies. Prefer this for scripts, examples, and agent-generated code unless there is a reason to do otherwise. -- **PyArrow**: `.to_arrow()` — a `pyarrow.Table`, when the surrounding code is Arrow-native or you need columnar/zero-copy handoff. - -Only reach for a DataFrame when the project *already* declares that dependency: - -- Pandas projects (pandas in `pyproject.toml`/requirements): `.to_pandas()`. -- Polars projects (polars declared): `.to_polars()`. - -If unsure, check the dependency manifest or the imports in surrounding files. When in doubt, use `.to_list()` or `.to_arrow()`. - -## Schema Design and Validation - -Favor `LanceModel` and Pydantic validation for Python schemas. They keep field -types readable, validate source records before a write, and map directly to a -LanceDB schema. Use `Vector(dimension)` for fixed-size vectors: - -```python -from lancedb.pydantic import LanceModel, Vector - -class Document(LanceModel): - id: int - text: str - vector: Vector(384, nullable=False) - -rows = [Document.model_validate(row) for row in source_rows] -table = db.create_table("documents", schema=Document) -table.add(rows) -``` - -Use PyArrow schemas instead when the pipeline is already Arrow-native, needs -record-batch streaming, or has runtime schema requirements that would make a -Pydantic model harder to understand. Declare Pydantic as a direct project -dependency when application code imports it, even if LanceDB also depends on it. - -## Recommended Patterns - -### Bounded search or query - -Use this for application reads, examples, notebooks, and agent-generated scripts: - -```python -results = ( - table.search(query_vector) - .where("status = 'ready'") - .select(["id", "text"]) - .limit(20) - .to_list() # or .to_arrow(); .to_pandas()/.to_polars() only if the project uses them -) -``` - -Why: `search()` works across local and remote tables and on both the sync and async clients. `select()` avoids fetching unused columns. `limit()` prevents accidental full-table reads. `.to_list()` and `.to_arrow()` avoid assuming pandas/polars is installed (see "Before Writing Code"). - -For a **plain scan** (no query vector), the entry point differs by client: - -```python -# Sync client: no .query() method — use .search() with no argument. -rows = table.search().where("status = 'ready'").select(["id", "text"]).limit(20).to_list() - -# Async client: use .query(). -rows = await async_table.query().where("status = 'ready'").select(["id", "text"]).limit(20).to_list() -``` - -`table.query()` on a sync table raises `AttributeError` (verified on `lancedb` 0.34.0). See the "Sync vs Async Scan API" section in `api_reference.md`. - -### Bounded query result conversion - -It is fine to collect bounded query/search results: - -```python -arrow_table = table.search().select(["id"]).limit(100).to_arrow() # sync plain scan -rows = table.search(query_vector).limit(10).to_list() -df = table.search(query_vector).limit(10).to_pandas() # only if pandas is a project dep -``` - -### Local-only Lance dataset API - -`table.to_lance()` does not itself materialize the full dataset. It returns the underlying `lance.LanceDataset`, making the table accessible through the PyLance dataset API. Use it when the task is explicitly local/OSS and needs Lance dataset methods not exposed by LanceDB: - -```python -# Local/OSS only: RemoteTable does not expose table.to_lance(). -ds = table.to_lance() -for batch in ds.to_batches(columns=["id", "text"], batch_size=10_000): - process(batch) -``` - -### Async Python - -Keep the same shape and bound the result before collecting: - -```python -results = await ( - async_table.query() - .where("status = 'ready'") - .select(["id", "text"]) - .limit(20) - .to_list() # or .to_arrow() -) -``` - -## Anti-Patterns - -**Avoid the following anti-patterns in your code.** - -### Table-level full materialization - -Avoid whole-table collectors in portable or large-table code: - -```python -df = table.to_pandas() -arrow_table = table.to_arrow() -polars_df = table.to_polars() -``` - -Why: local tables expose these whole-table collectors, but remote tables intentionally do not — a remote production table can be far larger than a local development table, so it is easy to accidentally pull the entire table into memory. - -`table.to_lance()` is different: it is not a full materialization call, but it is still local/OSS-only and should not appear in code meant to run against remote Enterprise tables. - -### Unbounded result collection - -Avoid query/search collection without a meaningful limit: - -```python -rows = table.search().to_list() # unbounded plain scan -rows = table.search(query_vector).to_list() # unbounded vector search -``` - -Prefer `select(...).limit(...)` before collecting; for large reads, stream in batches instead. - -### Per-row writes - -Avoid loops that write one row per call: - -```python -for row in rows: - table.add([row]) # one commit + fragment per row -``` - -Each `add()` creates a new version and fragment. Pass the whole batch in a single call, or chunk very large inputs: - -```python -table.add(rows) # single commit -# for very large inputs, add batches of several thousand rows -``` - -After the final successful write to an embedded OSS table, call -`table.optimize()`. Skip this for Enterprise/Cloud tables because their -maintenance is automatic. - -### Drop-then-reuse the same table name (Enterprise/Cloud) - -Avoid dropping or overwriting a remote table and then reusing that name right away: - -```python -db.drop_table("my_table") -table = db.create_table("my_table", data=rows) # reads 500 for ~5 min -table = db.create_table("my_table", data=rows, mode="overwrite") # same problem -``` - -Why: Enterprise/Cloud splits DDL (control plane) from query serving (data plane). The data plane caches the dataset behind a table name for up to `table_cache_ttl` (default 300s / 5 min), so after a drop/overwrite the DDL succeeds but queries against the reused name return `500 Internal Server Error` until the cache expires — and a fresh `describe` may still show the old schema. Instead, write to a **fresh name**, use `list_tables()` and fail if it already exists, then `rename_table(fresh, final)` onto the final name only after the old table's drop has propagated (~5 min). See the "Enterprise: never drop-then-reuse the same table name" section in `SKILL.md`. Local/OSS tables have no separate data plane — overwrite freely there. - -### Guessing performance fixes - -Avoid changing `nprobes`, `refine_factor`, or index types before checking the query plan and index stats. Diagnose first, then tune one knob at a time. diff --git a/.agents/skills/lancedb/references/python/performance.md b/.agents/skills/lancedb/references/python/performance.md deleted file mode 100644 index 5fd27440b..000000000 --- a/.agents/skills/lancedb/references/python/performance.md +++ /dev/null @@ -1,131 +0,0 @@ -# Python Performance Guidance - -Use this when writing Python code that ingests data, queries large tables, builds indexes, or investigates latency. - -## Ingestion - -### Recommended: validate schemas and records with Pydantic - -Favor `LanceModel` for readable Python schema definitions and validate source -records before writing. Use PyArrow directly for Arrow-native or streaming -pipelines where it is the clearer representation. - -```python -from lancedb.pydantic import LanceModel, Vector - -class Document(LanceModel): - id: int - text: str - vector: Vector(384, nullable=False) - -rows = [Document.model_validate(row) for row in source_rows] -table = db.create_table("documents", schema=Document) -table.add(rows) -``` - -### Recommended: bulk ingestion for materialized data - -```python -table.add(arrow_table) -table.add(df) -table.add(pa.dataset("data/", format="parquet")) -``` - -For very large initial loads, create the table empty first, then call `add(...)`. Passing data directly to `create_table(name, data)` can skip the auto-parallel write path. - -### Recommended: iterator ingestion for generated or streamed data - -```python -def batches(): - for raw in source: - vectors = model.encode(raw["text"]) - yield pa.RecordBatch.from_pydict({**raw, "vector": vectors}) - -table.add(batches()) -``` - -Use chunks of several thousand rows or more when practical. Tiny batches and per-row writes create many small fragments. - -### Anti-pattern: per-row `add()` - -```python -for row in rows: - table.add([row]) -``` - -Each call creates a version and fragment. This slows ingestion and later queries. - -## Indexing - -- Build a vector index once brute-force vector search becomes too slow. As a rule of thumb, local brute force is fine below roughly 100K vectors; beyond that, build an index. -- Use `IVF_PQ` as the general-purpose default. Enterprise builds this automatically. -- Use scalar indexes for filtered columns and merge/upsert keys. -- Use `BTREE` for mostly distinct numeric/string/temporal columns, `BITMAP` for booleans and low-cardinality columns, and `LABEL_LIST` for list membership queries. -- Keep full-text defaults unless phrase queries require position data. - -## Querying - -Always be explicit: - -```python -table.search(query_vector).select(["id", "title"]).limit(20) -``` - -- `select()` reduces bytes read and transferred. -- `limit()` prevents accidental full-table materialization. -- Pre-filtering is the default and guarantees returned rows satisfy the predicate. -- Use post-filtering only when fewer than `limit` results are acceptable. - -## Recall Tuning - -Tune one knob at a time: - -- Quantized indexes: raise `refine_factor` to rescore more candidates on full vectors. -- HNSW-backed indexes: raise `ef`; start around `1.5 * k`, increase toward `10 * k` if recall is short. -- IVF candidate breadth: `nprobes` is auto-tuned; override only when a selective pre-filter leaves too few neighbors. - -## Maintenance - -After every successful embedded OSS/local ingestion, call `table.optimize()`. -Do not add this to LanceDB Enterprise/Cloud remote table code; remote compaction -and cleanup are handled automatically based on the Enterprise cluster -configuration. - -Why local maintenance is needed: - -- Frequent writes can create many small fragments. Queries then need to scan across more files, which can increase latency. -- Updates, deletes, and appends create new table versions. Old versions are retained for time travel and rollback, which can grow disk usage. -- Indexes may have newly added rows that are not yet fully optimized into the index structure. - -For local/OSS tables, run `optimize()` after the final successful ingestion -write. Also run it after later batches of update/delete operations or on a -regular maintenance schedule: - -```python -table.optimize() -``` - -If the user wants more aggressive local disk cleanup, pass a shorter cleanup retention window: - -```python -from datetime import timedelta - -table.optimize(cleanup_older_than=timedelta(days=1)) -``` - -Do not use very short cleanup windows when the application depends on time travel, rollback, or old versions. - -## Diagnostics - -Before changing code or indexes, inspect: - -```python -print(table.search(query_vector).where("year > 2000").limit(10).analyze_plan()) -print(table.index_stats("vector_idx")) -``` - -Look for high scan bytes, missing indexes, fragmented data, and unindexed rows. - -## Python Multiprocessing - -When using multiprocessing, use `spawn` rather than `fork`. LanceDB is multi-threaded internally, and `fork` plus a multi-threaded process is unsafe. diff --git a/.agents/skills/lancedb/references/typescript/api_reference.md b/.agents/skills/lancedb/references/typescript/api_reference.md deleted file mode 100644 index aec07c9cb..000000000 --- a/.agents/skills/lancedb/references/typescript/api_reference.md +++ /dev/null @@ -1,78 +0,0 @@ -# TypeScript API Reference - -Quick method reference for TypeScript LanceDB code. Cross-check source for non-trivial claims. - -## Connect - -```typescript -import * as lancedb from "@lancedb/lancedb"; - -const db = await lancedb.connect("./camelot-db"); -``` - -**Place the local database directory next to the script/entrypoint that opens it** (resolve the path relative to the module, e.g. via `import.meta.dirname` / `__dirname`), not buried under a shared `data/` folder. The Lance dataset is the database, not a data file — keeping it beside its code makes ownership obvious and paths stable regardless of the working directory the script is launched from. - -**Do not name the directory `lancedb`** (e.g. `./lancedb`, `./data/lancedb`). It collides with the imported `lancedb` package/namespace, which is confusing to read. Give it a name derived from the repo or dataset with a clear prefix/suffix — for example `./-db`, `./_lancedb`, or `./vectordb`. - -Remote connections use `db://...` plus Enterprise/Cloud credentials and deployment settings. Check current source/docs for exact connection options. - -## Table Reads - -| Task | Preferred API | -| --- | --- | -| Vector search | `table.search(queryVector).limit(k)` | -| Full scan with filters/projection | `table.query().where(...).select(...).limit(...)` | -| Filter | `.where("col > 10")` | -| Projection | `.select(["id", "text"])` | -| Bound result count | `.limit(20)` | -| Collect bounded result as objects | `.toArray()` on query/search result | -| Collect bounded result as Arrow | `.toArrow()` on query/search result | -| Stream result batches | `for await (const batch of table.query()...)` | - -## Local vs Remote Safety - -| API | Agent guidance | -| --- | --- | -| `table.search(...)` | Preferred read path | -| `table.query()` | Preferred scan/filter path | -| `await table.toArrow()` | Avoid in portable or large-table code | -| `await table.query().toArray()` with no `limit()` | Avoid; unbounded collection | -| `await table.query().toArrow()` with no `limit()` | Avoid; unbounded collection | - -## Indexes - -```typescript -await table.createIndex("vector"); -await table.createIndex("status"); -``` - -Use vector indexes for large vector search workloads and scalar indexes for filtered columns or merge/upsert keys. Check source/docs before specifying advanced index options. - -## Filtering And Recall Knobs - -```typescript -await table.search(queryVector).where("status = 'ready'").limit(10).toArray(); -await table.search(queryVector).limit(10).refineFactor(20).toArray(); -await table.search(queryVector).limit(10).nprobes(50).toArray(); -await table.search(queryVector).limit(10).ef(100).toArray(); -await table.search(queryVector).where("status = 'ready'").postfilter().limit(10).toArray(); -``` - -Use `postfilter()` only when fewer than `limit` results are acceptable. - -## Diagnostics - -```typescript -console.log(await table.search(queryVector).where("year > 2000").limit(10).analyzePlan()); -console.log(await table.indexStats("vector_idx")); -``` - -Use these before changing indexes or search tuning. - -## Maintenance - -```typescript -await table.optimize(); -``` - -Call this after every successful local/OSS ingestion. It handles compaction, cleanup of old versions according to retention, and index optimization. Do not add this for LanceDB Enterprise/Cloud remote tables; Enterprise handles compaction and cleanup automatically from cluster configuration. diff --git a/.agents/skills/lancedb/references/typescript/patterns.md b/.agents/skills/lancedb/references/typescript/patterns.md deleted file mode 100644 index 1aa380968..000000000 --- a/.agents/skills/lancedb/references/typescript/patterns.md +++ /dev/null @@ -1,100 +0,0 @@ -# TypeScript Patterns - -Use these patterns when writing TypeScript code with `@lancedb/lancedb`. - -## Recommended Patterns - -### Bounded query - -Use this for application reads, scripts, and examples: - -```typescript -const rows = await table - .query() - .where("status = 'ready'") - .select(["id", "text"]) - .limit(20) - .toArray(); -``` - -### Bounded vector search - -```typescript -const rows = await table - .search(queryVector) - .select(["id", "text"]) - .limit(20) - .toArray(); -``` - -### Batch streaming for larger reads - -When the task needs many rows, avoid collecting everything at once: - -```typescript -for await (const batch of table - .query() - .where("status = 'ready'") - .select(["id", "text"]) - .limit(10_000)) { - process(batch); -} -``` - -## Anti-Patterns - -**Avoid the following anti-patterns in your code.** - -### Table-level full materialization - -Avoid whole-table collectors in portable or large-table code: - -```typescript -const tableArrow = await table.toArrow(); -``` - -Why: local tables expose these whole-table collectors, but remote tables intentionally do not — a remote production table can be far larger than a local development table, so it is easy to accidentally pull the entire table into memory. - -### Unbounded result collection - -Avoid query/search collection without a meaningful limit: - -```typescript -const rows = await table.query().toArray(); // unbounded plain scan -const rows = await table.search(queryVector).toArray(); // unbounded vector search -``` - -Prefer `select(...).limit(...)` before collecting; for large reads, stream in batches instead. - -### Per-row writes - -Avoid loops that write one row per call: - -```typescript -for (const row of rows) { - await table.add([row]); // one commit + fragment per row -} -``` - -Each `add()` creates a new version and fragment. Pass the whole batch in a single call, or chunk very large inputs: - -```typescript -await table.add(rows); // single commit -// for very large inputs, add in chunks of several thousand rows -``` - -### Drop-then-reuse the same table name (Enterprise/Cloud) - -Avoid dropping or overwriting a remote table and then reusing that name right away: - -```typescript -await db.dropTable("my_table"); -const table = await db.createTable("my_table", rows); // reads 500 for ~5 min -const table = await db.createTable("my_table", rows, { mode: "overwrite" }); // same problem -``` - -Why: Enterprise/Cloud splits DDL (control plane) from query serving (data plane). The data plane caches the dataset behind a table name for up to `table_cache_ttl` (default 300s / 5 min), so after a drop/overwrite the DDL succeeds but queries against the reused name return `500 Internal Server Error` until the cache expires — and a fresh `describe` may still show the old schema. Instead, write to a **fresh name**, use `tableNames()` and fail if it already exists, then `renameTable(fresh, final)` onto the final name only after the old table's drop has propagated (~5 min). See the "Enterprise: never drop-then-reuse the same table name" section in `SKILL.md`. Local/OSS tables have no separate data plane — overwrite freely there. - -### Guessing performance fixes - -Avoid changing `nprobes`, `refineFactor`, `ef`, or index settings before checking `analyzePlan()` and `indexStats(...)`. Diagnose first, then tune one knob at a time. diff --git a/.agents/skills/lancedb/references/typescript/performance.md b/.agents/skills/lancedb/references/typescript/performance.md deleted file mode 100644 index 9bf07e9ae..000000000 --- a/.agents/skills/lancedb/references/typescript/performance.md +++ /dev/null @@ -1,78 +0,0 @@ -# TypeScript Performance Guidance - -Use this when writing TypeScript code that ingests data, queries large tables, builds indexes, or investigates latency. - -## Ingestion - -- Prefer bulk or batched writes. -- Avoid per-row write loops; they create many small commits/fragments. -- For generated data, accumulate reasonable batches before adding. -- For file-backed data, prefer APIs that stream from Arrow/Parquet-style inputs when available. - -## Indexing - -- Build a vector index once brute-force vector search becomes too slow. As a rule of thumb, local brute force is fine below roughly 100K vectors; beyond that, build an index. -- Use the general-purpose vector index defaults unless the task has explicit recall/latency requirements. -- Build scalar indexes for filtered columns and merge/upsert keys. -- Use full-text index phrase options only when phrase queries require them. - -## Querying - -Always be explicit: - -```typescript -await table.search(queryVector).select(["id", "title"]).limit(20).toArray(); -``` - -- `select()` reduces bytes read and transferred. -- `limit()` prevents accidental full-table collection. -- Pre-filtering is the default behavior. Use `postfilter()` only when fewer than `limit` results are acceptable. - -## Recall Tuning - -Tune one knob at a time: - -- Quantized indexes: raise `refineFactor(...)` to rescore more candidates on full vectors. -- HNSW-backed indexes: raise `ef(...)`; start around `1.5 * k`, increase toward `10 * k` if recall is short. -- IVF candidate breadth: `nprobes(...)` is usually auto-tuned; override only when a selective pre-filter leaves too few neighbors. - -## Maintenance - -After every successful embedded OSS/local ingestion, call `table.optimize()`. -Do not add this to LanceDB Enterprise/Cloud remote table code; remote compaction -and cleanup are handled automatically based on the Enterprise cluster -configuration. - -Why local maintenance is needed: - -- Frequent writes can create many small fragments. Queries then need to scan across more files, which can increase latency. -- Updates, deletes, and appends create new table versions. Old versions are retained for time travel and rollback, which can grow disk usage. -- Indexes may have newly added rows that are not yet fully optimized into the index structure. - -For local/OSS tables, run `optimize()` after the final successful ingestion -write. Also run it after later batches of update/delete operations or on a -regular maintenance schedule: - -```typescript -await table.optimize(); -``` - -If the user wants more aggressive local disk cleanup, pass a shorter cleanup retention window: - -```typescript -const olderThan = new Date(Date.now() - 24 * 60 * 60 * 1000); -await table.optimize({ cleanupOlderThan: olderThan }); -``` - -Do not use very short cleanup windows when the application depends on time travel, rollback, or old versions. - -## Diagnostics - -Before changing code or indexes, inspect: - -```typescript -console.log(await table.search(queryVector).where("year > 2000").limit(10).analyzePlan()); -console.log(await table.indexStats("vector_idx")); -``` - -Look for high scan cost, missing indexes, fragmented data, and unindexed rows. diff --git a/.agents/skills/lancedb/scripts/check_materialization.py b/.agents/skills/lancedb/scripts/check_materialization.py deleted file mode 100644 index cbd8abc04..000000000 --- a/.agents/skills/lancedb/scripts/check_materialization.py +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env python3 -"""Scan Python and TypeScript for likely unsafe LanceDB materialization.""" - -from __future__ import annotations - -import argparse -import re -import sys -from dataclasses import dataclass -from pathlib import Path - - -PY_FULL_TABLE = re.compile(r"\b\w+\.(to_pandas|to_arrow|to_polars)\s*\(") -TS_TABLE_TO_ARROW = re.compile(r"\b\w+\.toArrow\s*\(") -TS_QUERY_COLLECTOR = re.compile(r"\.query\s*\(\s*\)[\s\S]*?\.to(Array|Arrow)\s*\(") - - -@dataclass -class Finding: - path: Path - line: int - message: str - text: str - - -def iter_files(paths: list[Path]) -> list[Path]: - files: list[Path] = [] - for path in paths: - if path.is_dir(): - files.extend( - p - for p in path.rglob("*") - if p.suffix in {".py", ".ts", ".tsx"} and "node_modules" not in p.parts - ) - elif path.suffix in {".py", ".ts", ".tsx"}: - files.append(path) - return sorted(set(files)) - - -def line_number(text: str, offset: int) -> int: - return text.count("\n", 0, offset) + 1 - - -def scan_python(path: Path, text: str) -> list[Finding]: - findings: list[Finding] = [] - for match in PY_FULL_TABLE.finditer(text): - line_start = text.rfind("\n", 0, match.start()) + 1 - line_end = text.find("\n", match.start()) - if line_end == -1: - line_end = len(text) - line = text[line_start:line_end].strip() - if ".search(" in line or ".query(" in line: - continue - findings.append( - Finding( - path, - line_number(text, match.start()), - f"Review Python `{match.group(1)}()` call; table-level materialization is not portable to remote tables.", - line, - ) - ) - return findings - - -def statement_around(text: str, start: int, end: int) -> str: - before = max(text.rfind(";", 0, start), text.rfind("\n\n", 0, start)) - after_candidates = [pos for pos in (text.find(";", end), text.find("\n\n", end)) if pos != -1] - after = min(after_candidates) if after_candidates else len(text) - return text[before + 1 : after].strip() - - -def scan_typescript(path: Path, text: str) -> list[Finding]: - findings: list[Finding] = [] - for match in TS_TABLE_TO_ARROW.finditer(text): - stmt = statement_around(text, match.start(), match.end()) - if ".query(" in stmt or ".search(" in stmt: - continue - findings.append( - Finding( - path, - line_number(text, match.start()), - "Review TypeScript `table.toArrow()`-style call; table-level materialization is not portable for large/remote tables.", - stmt.splitlines()[0].strip(), - ) - ) - - for match in TS_QUERY_COLLECTOR.finditer(text): - stmt = statement_around(text, match.start(), match.end()) - if ".limit(" in stmt: - continue - findings.append( - Finding( - path, - line_number(text, match.start()), - "Review unbounded TypeScript query collection; add `limit()` or stream batches.", - stmt.splitlines()[0].strip(), - ) - ) - return findings - - -def scan_file(path: Path) -> list[Finding]: - text = path.read_text(encoding="utf-8", errors="replace") - if path.suffix == ".py": - return scan_python(path, text) - if path.suffix in {".ts", ".tsx"}: - return scan_typescript(path, text) - return [] - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("paths", nargs="+", type=Path) - parser.add_argument( - "--no-fail", action="store_true", help="Always exit 0 after reporting findings." - ) - args = parser.parse_args() - - findings: list[Finding] = [] - for path in iter_files(args.paths): - findings.extend(scan_file(path)) - - for finding in findings: - print(f"{finding.path}:{finding.line}: {finding.message}") - print(f" {finding.text}") - - if findings: - print( - f"\n{len(findings)} finding(s). Review manually; bounded query result conversion may be OK." - ) - return 0 if args.no_fail or not findings else 1 - - -if __name__ == "__main__": - sys.exit(main())