From 0bd0944062a4929d9f3103056c4a1e4f4fef9928 Mon Sep 17 00:00:00 2001 From: Dan Tasse <105866+dantasse@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:29:06 -0400 Subject: [PATCH] feat: branch skill updates for merge (#3685) Skill updates for branch merging. Terra/Sol can do an end-to-end "create 3 branches, add a column, generate embeddings, merge the best" workflow now. --- .../skills/lancedb/references/branch_ops.md | 67 ++++++++++++++++++- .../references/python/api_reference.md | 11 ++- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/plugins/lancedb/skills/lancedb/references/branch_ops.md b/plugins/lancedb/skills/lancedb/references/branch_ops.md index bc05671e8..e94c7e6db 100644 --- a/plugins/lancedb/skills/lancedb/references/branch_ops.md +++ b/plugins/lancedb/skills/lancedb/references/branch_ops.md @@ -2,7 +2,7 @@ Manage branches on a LanceDB table: list what exists, create new ones, delete stale ones, and direct read/write operations at a specific branch without touching main. Use for branch lifecycle tasks, experimental/isolated table versions, targeting an operation at a non-main branch, or confirming a mutation did not affect main. -Works on local/OSS and remote Enterprise/Cloud tables. +Works on local/OSS and remote Enterprise/Cloud tables, except merging a branch into main, which is Enterprise-only. ## The branch model (important) @@ -101,6 +101,69 @@ assert b"lancedb:description" not in (table.schema.field("category").metadata or Two handles on the same branch see each other's writes (e.g. `table.branches.create("exp")` and `db.open_table(name, branch="exp")`); main stays isolated. +## Merging a branch into main (Enterprise only) + +Merge is available through the SDKs (`table.branches.merge(...)`) on **Enterprise tables only** — it is not supported on Cloud or local/OSS tables, which raise `NotSupported`. + +`merge` takes the branch to merge **from** and a `dry_run` flag. Both the SDK method and the underlying REST endpoint **actually merge by default** (`dry_run=False`); pass `dry_run=True` to only preview. A rejected merge is **not an exception** — it returns a result with `status="rejected"` rather than raising, so inspect the return value. Use `branches.diff(from_branch)` to inspect a branch's pending diff without attempting a merge. + +```python +exp = "experiment-reindex" + +# preview only — returns status="ready" if it would merge cleanly +preview = table.branches.merge(exp, dry_run=True) + +# actually merge (default) +result = table.branches.merge(exp) +if result["status"] == "merged": + print("landed at", result["mainVersionAfter"]) +elif result["status"] == "rejected": + print(result["diff"]["mergeBlockers"]) # why it was refused + +# inspect a branch's pending diff without merging +diff = table.branches.diff(exp) +``` + +Async: `await table.branches.merge(exp)`, `await table.branches.diff(exp)`. + +```typescript +const branches = await table.branches(); +const exp = "experiment-reindex"; + +// preview only (second arg is dryRun) +const preview = await branches.merge(exp, true); + +// actually merge (default) +const result = await branches.merge(exp); +if (result.status === "merged") { + console.log("landed at", result.mainVersionAfter); +} else if (result.status === "rejected") { + console.log(result.diff.mergeBlockers); +} + +const diff = await branches.diff(exp); +``` + +The result is the wire JSON, containing `status` (`ready` on a passing dry run, `merged` on success, `rejected` when refused — also `notImplemented`/`unknown`), the branch `diff` (including `mergeBlockers` explaining any rejection), a `preview` of the columns that would be promoted, and — after a real merge — `mainVersionAfter`. + +### Merge preconditions + +Merge only **promotes newly added columns** onto main; it does not replay arbitrary commits. Practically, a branch is mergeable only if it has **exactly one commit since it was created, and that commit added a column**. The merge is rejected (`status: "rejected"`, with `mergeBlockers` set) if: + +- the branch was forked from another branch rather than directly from main +- main has advanced since the branch was forked +- the branch's rows changed since the fork (row counts must match main exactly) +- the branch removed columns or changed a column's type/nullability +- the branch added no columns (index-only changes are not merged) + +### Adding a column in a single commit + +Because the branch must contain just one column-adding commit, add the column with its values in one operation rather than add-then-backfill: + +1. **SQL transformation** — `add_columns` with a SQL expression computed from existing columns, so the column lands populated in one commit. +2. **Precompute the values** — compute the column's values externally, then add the fully-populated column in a single operation (e.g. via `merge_insert`/`add_columns` with the data ready). +3. **Lance-format-level data evolution (pylance)** — use Lance's data evolution with backfill, documented at . + ## Quick reference | Goal | Python | TypeScript | @@ -113,5 +176,7 @@ Two handles on the same branch see each other's writes (e.g. `table.branches.cre | Delete branch | `table.branches.delete(name)` | `await branches.delete(name)` | | Which branch is this handle on? | `table.current_branch()` (`None` = main) | `table.currentBranch()` (`null` = main) | | Target main | use the original (non-branch) handle | use the original (non-branch) handle | +| Merge branch into main (Enterprise only) | `table.branches.merge(from_branch, dry_run=False)` | `await branches.merge(fromBranch, dryRun)` | +| Preview a branch's pending diff (Enterprise only) | `table.branches.diff(from_branch)` | `await branches.diff(fromBranch)` | Branch names must be non-empty; empty names raise a validation error. diff --git a/plugins/lancedb/skills/lancedb/references/python/api_reference.md b/plugins/lancedb/skills/lancedb/references/python/api_reference.md index 323800670..bbb209630 100644 --- a/plugins/lancedb/skills/lancedb/references/python/api_reference.md +++ b/plugins/lancedb/skills/lancedb/references/python/api_reference.md @@ -4,12 +4,19 @@ Quick method reference for Python LanceDB code. Cross-check source for non-trivi ## Connect +If you're connecting to a remote database, use this: ```python import lancedb -db = lancedb.connect("./camelot-db") # local/OSS -db = lancedb.connect("db://my-db", api_key=api_key, region=region) # remote +db = lancedb.connect("db://my-db", api_key=api_key, host_override=host_override) # remote ``` +(values may be found in LANCEDB_API_KEY and LANCEDB_HOST_OVERRIDE, either in env vars or a .env file) + +If you're connecting to a local table using OSS LanceDB, use this: +```python +db = lancedb.connect("./camelot-db") # local/OSS +``` +If you're not sure which, or if you can't find the api_key or host_override params, ask the user. **Place the local database directory next to the script/entrypoint that opens it** (i.e. resolve the path relative to the script, `Path(__file__).parent / "camelot-db"`), not buried under a shared `data/` folder. The Lance dataset is the database, not a data file — keeping it beside its code makes ownership obvious and paths stable regardless of the working directory the script is launched from.