rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-08-18 03:58:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into ayush/seq-packing

Browse Source 
# Conflicts:
#	python/python/lancedb/streaming.py
This commit is contained in:
Ayush Chaurasia
2026-08-11 23:37:40 +05:30
parent 028a225b05 a615306f39
commit 51bc3f824b
6 changed files with 795 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/docs-link-check.yml
+70 -49
View File
@@ -36,7 +36,9 @@ jobs:
permissions:
contents: read
outputs:
checker_outcome: ${{ steps.lychee.outcome }}
exit_code: ${{ steps.lychee.outputs.exit_code }}
status: ${{ steps.validate.outputs.status }}
steps:
- name: Checkout
uses: actions/checkout@v6
@@ -50,6 +52,7 @@ jobs:
- name: Check links
id: lychee
continue-on-error: true
uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0
with:
# Restricted to http(s) on purpose. Much of docs/src is generated
@@ -68,38 +71,50 @@ jobs:
format: json
output: ./lychee/out.json
jobSummary: false
# The report, not a red build, is the signal for broken links. The
# validation step below still fails the run if the check itself
# breaks.
# The report issue, not a red workflow run, is the signal for link
# findings and checker failures alike.
fail: false
- name: Validate report
id: validate
# lychee does not reserve exit code 2 for broken links: its CLI
# parser also exits 2 on an invalid option, before any link was
# checked or any report written. Only a parseable report whose
# counts agree with the exit code counts as a link verdict; anything
# else fails here, and the report job below is skipped entirely, so
# the tracking issue is never touched. Exit 2 covers timeouts as
# well as errors, and a timed-out host is exactly the transient
# unavailability this report exists to surface, so both count as
# findings. Requiring total > 0 also catches a glob that silently
# stopped matching any file.
if: steps.lychee.outputs.exit_code == 0 || steps.lychee.outputs.exit_code == 2
# counts agree with a completed exit code (0 or 2) counts as a link
# verdict. Everything else becomes a checker-error report instead of
# failing the workflow. Exit 2 covers timeouts as well as errors, and a
# timed-out host is exactly the transient unavailability this report
# exists to surface, so both count as findings. Requiring total > 0
# also catches a glob that silently stopped matching any file.
if: always()
env:
CHECKER_OUTCOME: ${{ steps.lychee.outcome }}
EXIT_CODE: ${{ steps.lychee.outputs.exit_code }}
run: |
jq -e --argjson code "$EXIT_CODE" '
(.total > 0) and
(if $code == 0
then .errors == 0 and .timeouts == 0
and (.error_map | length == 0) and (.timeout_map | length == 0)
else (.errors + .timeouts) > 0
and ((.error_map | length) + (.timeout_map | length)) > 0
end)
' ./lychee/out.json
status=checker-error
if [[ "$CHECKER_OUTCOME" == success ]] &&
[[ "$EXIT_CODE" == 0 || "$EXIT_CODE" == 2 ]] &&
jq -e --argjson code "$EXIT_CODE" '
(.total > 0) and
(if $code == 0
then .errors == 0 and .timeouts == 0
and (.error_map | length == 0) and (.timeout_map | length == 0)
else (.errors + .timeouts) > 0
and ((.error_map | length) + (.timeout_map | length)) > 0
end)
' ./lychee/out.json
then
if [[ "$EXIT_CODE" == 0 ]]; then
status=healthy
else
status=findings
fi
fi
echo "status=$status" >> "$GITHUB_OUTPUT"
echo "Validated link check as $status"
- name: Upload report
if: steps.lychee.outputs.exit_code == 2
if: steps.validate.outputs.status == 'findings'
uses: actions/upload-artifact@v7
with:
name: link-report
@@ -115,26 +130,11 @@ jobs:
permissions:
issues: write
env:
CHECKER_OUTCOME: ${{ needs.scan.outputs.checker_outcome }}
EXIT_CODE: ${{ needs.scan.outputs.exit_code }}
STATUS: ${{ needs.scan.outputs.status }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- name: Classify checker result
# lychee exits 0 when every link resolves and 2 when links fail,
# both already cross-checked against the report by the scan job's
# validation step. Anything else (1 runtime, 3 bad config) means the
# check never produced a link verdict, which must surface as a failed
# run rather than be published as "broken documentation links".
run: |
case "$EXIT_CODE" in
0|2)
echo "lychee exit code $EXIT_CODE"
;;
*)
echo "::error::lychee exited with '$EXIT_CODE': the link check did not complete. Leaving the report issue untouched."
exit 1
;;
esac
- name: Find existing report issue
id: report
# Matched on title alone, and through search rather than a listing:
@@ -144,7 +144,7 @@ jobs:
# Closed issues are included because a healthy run closes the report:
# an open-only lookup would forget that identity and the next failing
# run would open a duplicate. The oldest match stays the canonical
# report and is reopened below when links break again.
# report and is reopened below when a problem recurs.
run: |
match=$(gh issue list --repo "$GITHUB_REPOSITORY" --state all \
--search "in:title \"$REPORT_TITLE\" author:app/github-actions" \
@@ -154,14 +154,14 @@ jobs:
echo "state=$(jq -r '.state // empty' <<<"$match")" >> "$GITHUB_OUTPUT"
- name: Download report
if: env.EXIT_CODE == 2
if: env.STATUS == 'findings'
uses: actions/download-artifact@v8
with:
name: link-report
path: ./lychee
- name: Compose report
if: env.EXIT_CODE == 2
if: env.STATUS == 'findings'
run: |
run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
{
@@ -185,22 +185,41 @@ jobs:
' ./lychee/out.json
} > ./lychee/issue.md
- name: Compose checker error report
if: env.STATUS == 'checker-error'
run: |
mkdir -p ./lychee
run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
{
echo "The documentation link check did not complete in [the latest run]($run_url)."
echo
echo "This issue is rewritten by every scheduled run and closed automatically once a trustworthy run finds that all links resolve."
echo
echo "The checker did not produce a trustworthy link verdict. Treat the previous result, if any, as stale until a later run completes."
echo
echo "* Action outcome: \`$CHECKER_OUTCOME\`"
echo "* Exit code: \`${EXIT_CODE:-not reported}\`"
echo "* Verdict validation: \`failed\`"
} > ./lychee/issue.md
- name: Reopen report issue
# A healthy run closes the report, and the issue action below only
# rewrites the body of whatever number it is given. Without an
# explicit reopen, the 2 -> 0 -> 2 sequence would keep rewriting a
# closed issue while links are broken. A CLOSED state implies the
# lookup found a canonical issue, so no separate emptiness check.
if: env.EXIT_CODE == 2 && steps.report.outputs.state == 'CLOSED'
# explicit reopen, a later finding or checker error would rewrite a
# closed issue. A CLOSED state implies the lookup found a canonical
# issue, so no separate emptiness check.
if: >-
env.STATUS != 'healthy' &&
steps.report.outputs.state == 'CLOSED'
env:
ISSUE_NUMBER: ${{ steps.report.outputs.number }}
run: |
run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
gh issue reopen "$ISSUE_NUMBER" --repo "$GITHUB_REPOSITORY" \
--comment "Broken documentation links found again in [the latest run]($run_url)."
--comment "The documentation link checker reported a problem again in [the latest run]($run_url)."
- name: Report broken links
if: env.EXIT_CODE == 2
- name: Report link-check problem
if: env.STATUS != 'healthy'
uses: peter-evans/create-issue-from-file@fca9117c27cdc29c6c4db3b86c48e4115a786710 # v6.0.0
with:
# Empty on the first failing run, which creates the issue; afterwards
@@ -213,7 +232,9 @@ jobs:
- name: Close report issue once links are healthy
# An OPEN state implies the lookup found a canonical issue; a report
# that is already closed needs nothing.
if: env.EXIT_CODE == 0 && steps.report.outputs.state == 'OPEN'
if: >-
env.STATUS == 'healthy' &&
steps.report.outputs.state == 'OPEN'
env:
ISSUE_NUMBER: ${{ steps.report.outputs.number }}
run: |
python/python/lancedb/streaming.py
+320 -35
View File
@@ -11,6 +11,11 @@ Provides StreamingDataset, a PyTorch IterableDataset that guarantees:
- **Resumability**: state_dict / load_state_dict capture per-split consumption
counts so training can resume from an exact mid-epoch position even when the
distributed topology changes between runs.
Transform failures on bad rows (e.g. nulls or NaNs from incomplete data) can
be tolerated with ``on_transform_error="skip"``; see the parameter
documentation on StreamingDataset for how this interacts with the guarantees
above.
"""
import ctypes
@@ -24,7 +29,7 @@ from collections import deque
from concurrent.futures import ThreadPoolExecutor
from copy import deepcopy
from multiprocessing import RawArray
from typing import Any, Callable, cast, Iterator, Optional
from typing import Any, Callable, cast, Iterator, Optional, Union
import pyarrow as pa
import torch
@@ -154,6 +159,49 @@ class StreamingDataset(IterableDataset):
Padding token id used to complete blocks when a split runs out of
real tokens mid-cycle or at epoch end. Required with
``pack_sequences``, ignored otherwise.
on_transform_error:
What to do when the transform raises an exception:
- ``"raise"`` (the default): the exception propagates and iteration
aborts.
- ``"skip"``: the failing rows are dropped and iteration continues.
- ``"warn"``: like ``"skip"``, but a warning is logged for each
failing batch.
- a callable ``handler(exc) -> bool``: called with the exception;
return ``True`` to skip the failing rows or ``False`` to re-raise.
Useful to skip only expected error types (compatible with
``webdataset.handlers`` style handlers).
When a batch fails, the transform is re-invoked on each single-row
slice of the batch so that only the rows that actually fail are
dropped. Transforms should therefore be deterministic and accept
batches of any size (including one row). Skipped rows are counted in
``rows_skipped``.
Skipping weakens the elastic-determinism guarantee at the end of the
epoch: splits that lose more rows than others run dry earlier, and
each rank's iterator ends at the last cycle where every split *it
owns* still has a row. Because bad rows are not distributed evenly
across splits, this means one rank's iterator can yield noticeably
fewer or more steps than another rank's *in the same run* — there is
no cross-rank coordination that stops every rank at the same global
step. This is generally safe for asynchronous or single-rank use,
but synchronous distributed training (e.g. ranks that call
``all_reduce`` every step) can hang or deadlock if one rank's
iterator is exhausted while others are still stepping; callers doing
synchronous multi-rank training with ``on_transform_error != "raise"``
are responsible for their own cross-rank stopping mechanism (e.g.
broadcasting a stop signal on ``StopIteration``). The final few
global steps can also differ across topologies (bounded by the skew
in bad-row counts across splits). The sequence of samples yielded
from each split remains deterministic. Mid-epoch
checkpoints remain exact provided the transform fails
deterministically; in multi-rank training each rank must save its
own ``state_dict`` and the states must be combined with
``merge_state_dicts`` before resuming on a different topology.
Prefer the ``filter`` parameter when bad rows can be expressed as a
SQL predicate (e.g. ``"col IS NOT NULL"``) — filtering happens before
splits are built, so every guarantee is fully preserved.
worker_info_override:
If set, used in place of ``torch.utils.data.get_worker_info()`` to
determine the DataLoader worker assignment. Intended for unit tests
@@ -182,6 +230,7 @@ class StreamingDataset(IterableDataset):
pack_sequences: Optional[int] = None,
eos_id: Optional[int] = None,
pad_id: Optional[int] = None,
on_transform_error: Union[str, Callable[[Exception], bool]] = "raise",
connection_factory: Optional[Callable[[str], Any]] = None,
worker_info_override=None,
):
@@ -221,6 +270,13 @@ class StreamingDataset(IterableDataset):
f"pack_sequences requires a list-typed token column; "
f"{columns[0]} has type {field.type}"
)
if on_transform_error not in ("raise", "skip", "warn") and not callable(
on_transform_error
):
raise ValueError(
"on_transform_error must be 'raise', 'skip', 'warn', or a "
f"callable, got {on_transform_error!r}"
)
self._table = table
self._num_splits = num_splits
@@ -239,6 +295,7 @@ class StreamingDataset(IterableDataset):
self._pack_sequences = pack_sequences
self._eos_id = eos_id
self._pad_id = pad_id
self._on_transform_error = on_transform_error
self._connection_factory = connection_factory
self._worker_info_override = worker_info_override
@@ -260,19 +317,28 @@ class StreamingDataset(IterableDataset):
# in the main process. RawArray is picklable via the forkserver
# reduction protocol so it survives the dataset pickle round-trip.
# Layout: [unscanned_rows, raw_rows, cooked_rows, consumed_rows,
# bytes_loaded, fetch_time_us, transform_time_us]
self._worker_stats: RawArray = RawArray(ctypes.c_int64, 7)
# bytes_loaded, fetch_time_us, transform_time_us,
# rows_skipped]
self._worker_stats: RawArray = RawArray(ctypes.c_int64, 8)
# Cumulative bytes of Arrow buffer data fetched across all iterations.
self._bytes_loaded: int = 0
# Cumulative seconds spent in LanceDB I/O and in transform functions.
self._fetch_time: float = 0.0
self._transform_time: float = 0.0
# Cumulative rows dropped by on_transform_error across all iterations.
self._rows_skipped: int = 0
# Number of samples each split has already been consumed. At global
# step boundaries all splits have consumed this many samples, so a
# single scalar captures the topology-independent checkpoint state.
self._resume_offset: int = 0
# Permutation position each split has consumed through, keyed by
# global split index. Equal to _resume_offset for every split unless
# on_transform_error skipped rows, in which case skipped positions
# push the watermark of the affected splits further ahead. Splits
# this instance has never iterated have no entry.
self._resume_positions: dict[int, int] = {}
# Build the permutation table once, deterministically.
builder = permutation_builder(table)
@@ -343,6 +409,7 @@ class StreamingDataset(IterableDataset):
# Set identity transform on each Permutation so __getitems__ returns
# the raw RecordBatch. Stage 2 applies the real transform.
permutations: list[Permutation] = []
initial_positions: list[int] = []
for split_idx in my_splits:
perm = Permutation.from_tables(
self._table, self._perm_table, split=split_idx
@@ -350,21 +417,27 @@ class StreamingDataset(IterableDataset):
if self._columns is not None:
perm = perm.select_columns(self._columns)
perm = perm.with_transform(lambda batch: batch)
# Packing consumes documents at different rates per split, so it
# tracks per-split offsets; row mode uses the uniform offset.
skip = (
# Packing tracks documents consumed per split. Row mode tracks
# absolute permutation positions so transform failures can skip
# rows without making resume repeat them.
start_pos = (
self._pack_consumed[split_idx]
if self._pack_sequences is not None
else self._resume_offset
else self._resume_positions.get(split_idx, self._resume_offset)
)
if skip > 0:
perm = perm.with_skip(skip)
if start_pos > 0:
perm = perm.with_skip(start_pos)
initial_positions.append(start_pos)
permutations.append(perm)
n = len(permutations)
split_sizes = [perm.num_rows for perm in permutations]
initial_offset = self._resume_offset
local_consumed = [0] * n
# Permutation position each split has consumed through (absolute,
# i.e. counted from the start of the unskipped split). Runs ahead of
# initial + local_consumed when rows are skipped.
pos_consumed = list(initial_positions)
batch_size = self._read_batch_size
max_prefetch = self._prefetch_batches
@@ -387,12 +460,14 @@ class StreamingDataset(IterableDataset):
else Transforms.arrow2python
)
# Per-split pipeline state.
# Per-split pipeline state. Batches are paired with the absolute
# permutation position of their first row so that skipped rows can be
# accounted for in pos_consumed.
fetch_head = [0] * n
io_pending = [deque() for _ in range(n)] # Future[RecordBatch]
raw_batches = [deque() for _ in range(n)] # RecordBatch — fetched, awaiting tx
tx_pending = [deque() for _ in range(n)] # Future[list[Any]]
cooked = [deque() for _ in range(n)] # rows ready to yield
io_pending = [deque() for _ in range(n)] # (abs_start, Future[RecordBatch])
raw_batches = [deque() for _ in range(n)] # (abs_start, RecordBatch)
tx_pending = [deque() for _ in range(n)] # Future[list[(abs_pos, row)]]
cooked = [deque() for _ in range(n)] # (abs_pos, row) ready to yield
# Limit simultaneous transforms to transform_workers across all splits.
tx_semaphore = threading.Semaphore(transform_workers)
@@ -415,7 +490,8 @@ class StreamingDataset(IterableDataset):
fetch_head[i] += fetch
perm_i = permutations[i]
indices = list(range(start, start + fetch))
io_pending[i].append(io_pool.submit(_io_call, perm_i, indices))
abs_start = initial_positions[i] + start
io_pending[i].append((abs_start, io_pool.submit(_io_call, perm_i, indices)))
def _fill_io(i: int) -> None:
while len(io_pending[i]) < max_prefetch and fetch_head[i] < split_sizes[i]:
@@ -423,15 +499,72 @@ class StreamingDataset(IterableDataset):
def _drain_io(i: int) -> None:
"""Move completed I/O futures into raw_batches non-blockingly."""
while io_pending[i] and io_pending[i][0].done():
raw_batches[i].append(io_pending[i].popleft().result())
while io_pending[i] and io_pending[i][0][1].done():
abs_start, fut = io_pending[i].popleft()
raw_batches[i].append((abs_start, fut.result()))
# ── Stage 2 helpers ───────────────────────────────────────────────────
def _tx_call_guarded(batch):
on_error = self._on_transform_error
def _should_skip(exc: Exception) -> bool:
if on_error == "raise":
return False
if callable(on_error):
return bool(on_error(exc))
return True # "skip" or "warn"
def _check_row_count(rows: list, num_rows: int) -> None:
if len(rows) != num_rows:
raise ValueError(
f"transform returned {len(rows)} rows for a batch of "
f"{num_rows}; transforms must return exactly one output "
"row per input row. To drop bad rows, raise inside the "
"transform and pass on_transform_error='skip'."
)
def _transform_isolated(abs_start, batch, batch_exc):
"""Re-run the transform on single-row slices, dropping failures."""
out = []
skipped = 0
first_exc = None
for j in range(batch.num_rows):
try:
rows = list(final_transform(batch.slice(j, 1)))
except Exception as exc:
if not _should_skip(exc):
raise
skipped += 1
if first_exc is None:
first_exc = exc
continue
_check_row_count(rows, 1)
out.append((abs_start + j, rows[0]))
self._rows_skipped += skipped
if skipped and on_error == "warn":
logger.warning(
"Skipped %d of %d rows whose transform failed (first error: %r)",
skipped,
batch.num_rows,
first_exc if first_exc is not None else batch_exc,
)
return out
def _transform_batch(abs_start, batch):
"""Apply the transform, returning [(abs_pos, row), ...]."""
try:
rows = list(final_transform(batch))
except Exception as exc:
if not _should_skip(exc):
raise
return _transform_isolated(abs_start, batch, exc)
_check_row_count(rows, batch.num_rows)
return [(abs_start + j, row) for j, row in enumerate(rows)]
def _tx_call_guarded(abs_start, batch):
try:
t0 = time.perf_counter()
result = final_transform(batch)
result = _transform_batch(abs_start, batch)
self._transform_time += time.perf_counter() - t0
return result
finally:
@@ -440,8 +573,8 @@ class StreamingDataset(IterableDataset):
def _try_submit_tx(i: int) -> None:
"""Submit transforms for raw_batches[i] up to available capacity."""
while raw_batches[i] and tx_semaphore.acquire(blocking=False):
batch = raw_batches[i].popleft()
tx_pending[i].append(tx_pool.submit(_tx_call_guarded, batch))
abs_start, batch = raw_batches[i].popleft()
tx_pending[i].append(tx_pool.submit(_tx_call_guarded, abs_start, batch))
def _drain_tx(i: int) -> None:
"""Move completed transform futures into cooked non-blockingly."""
@@ -469,11 +602,14 @@ class StreamingDataset(IterableDataset):
# Acquire a transform slot (may block briefly if all
# transform_workers are busy with other splits).
tx_semaphore.acquire()
batch = raw_batches[i].popleft()
tx_pending[i].append(tx_pool.submit(_tx_call_guarded, batch))
abs_start, batch = raw_batches[i].popleft()
tx_pending[i].append(
tx_pool.submit(_tx_call_guarded, abs_start, batch)
)
elif io_pending[i]:
# Block on the oldest in-flight I/O fetch.
raw_batches[i].append(io_pending[i].popleft().result())
abs_start, fut = io_pending[i].popleft()
raw_batches[i].append((abs_start, fut.result()))
_advance(i)
else:
break # split exhausted
@@ -496,10 +632,12 @@ class StreamingDataset(IterableDataset):
if not cooked[i]:
return
buf["starts"].append(len(buf["tokens"]))
buf["tokens"].extend(cooked[i].popleft())
pos, tokens = cooked[i].popleft()
buf["tokens"].extend(tokens)
buf["tokens"].append(eos_id)
pack_consumed[my_splits[i]] += 1
local_consumed[i] += 1
pos_consumed[i] = pos + 1
_advance(i)
def _emit_block(i: int) -> dict[str, Any]:
@@ -540,12 +678,13 @@ class StreamingDataset(IterableDataset):
# worker process.
ws = self._worker_stats
ws[0] = sum(split_sizes[j] - fetch_head[j] for j in range(n))
ws[1] = sum(batch.num_rows for q in raw_batches for batch in q)
ws[1] = sum(batch.num_rows for q in raw_batches for _, batch in q)
ws[2] = sum(len(q) for q in cooked)
ws[3] = sum(local_consumed)
ws[4] = self._bytes_loaded
ws[5] = int(self._fetch_time * 1_000_000)
ws[6] = int(self._transform_time * 1_000_000)
ws[7] = self._rows_skipped
try:
for i in range(n):
@@ -577,15 +716,28 @@ class StreamingDataset(IterableDataset):
return
while True:
# Stop when any split is exhausted (all exhaust
# simultaneously: equal split sizes + round-robin).
if any(local_consumed[i] >= split_sizes[i] for i in range(n)):
# A cycle only runs if every split can still produce a
# row. Without skips all splits exhaust simultaneously
# (equal split sizes + round-robin); when
# on_transform_error drops rows a split can run dry
# early, ending the epoch at the last complete cycle.
# This check only sees splits owned by this rank/worker
# (my_splits) — there is no cross-rank coordination, so
# a different rank with fewer skipped rows keeps going;
# see the on_transform_error docstring.
exhausted = False
for i in range(n):
_ensure_cooked(i)
if not cooked[i]:
exhausted = True
break
if exhausted:
break
for i in range(n):
_ensure_cooked(i)
row = cooked[i].popleft()
pos, row = cooked[i].popleft()
local_consumed[i] += 1
pos_consumed[i] = pos + 1
_advance(i)
# After the last split in each cycle: update the
@@ -594,10 +746,25 @@ class StreamingDataset(IterableDataset):
# even when __iter__ runs in a worker process.
if i == n - 1:
self._resume_offset = initial_offset + local_consumed[i]
for j, split_idx in enumerate(my_splits):
self._resume_positions[split_idx] = pos_consumed[j]
_update_stats()
yield row
finally:
# Final stats flush: the per-cycle write above never runs
# when iteration ends mid-cycle (e.g. a split whose rows
# were all skipped before completing a single cycle), so
# counters like rows_skipped would otherwise be stale.
ws = self._worker_stats
ws[0] = sum(split_sizes[j] - fetch_head[j] for j in range(n))
ws[1] = 0 # queue-depth properties document 0 when idle
ws[2] = 0
ws[3] = sum(local_consumed)
ws[4] = self._bytes_loaded
ws[5] = int(self._fetch_time * 1_000_000)
ws[6] = int(self._transform_time * 1_000_000)
ws[7] = self._rows_skipped
self._raw_batches_ref = None
self._cooked_ref = None
self._fetch_head_ref = None
@@ -651,7 +818,7 @@ class StreamingDataset(IterableDataset):
batches. Returns 0 when not iterating.
"""
if self._raw_batches_ref is not None:
return sum(batch.num_rows for q in self._raw_batches_ref for batch in q)
return sum(batch.num_rows for q in self._raw_batches_ref for _, batch in q)
return int(self._worker_stats[1])
@property
@@ -681,6 +848,19 @@ class StreamingDataset(IterableDataset):
)
return int(self._worker_stats[0])
@property
def rows_skipped(self) -> int:
"""Number of rows dropped because their transform raised an exception.
Only ever non-zero when ``on_transform_error`` is set to ``"skip"``,
``"warn"``, or a callable that returned ``True``. Accumulates across
multiple iterations of the same dataset instance and is never reset
automatically.
"""
if self._raw_batches_ref is not None:
return self._rows_skipped
return int(self._worker_stats[7])
@property
def consumed_rows(self) -> int:
"""Number of rows already yielded to the caller across all splits.
@@ -742,9 +922,16 @@ class StreamingDataset(IterableDataset):
def state_dict(self) -> dict:
"""Snapshot the dataset's consumption state.
In row mode the returned dict is topology-independent: at global step
boundaries every split has been consumed the same number of times, so
the per-split count is a single uniform value.
In row mode, the returned dict is topology-independent at global step
boundaries. ``positions_consumed_per_split`` records how far each
split's permutation has advanced, which can differ from the sample
count when ``on_transform_error`` skips rows. Combine state dicts from
every rank with
[merge_state_dicts][lancedb.streaming.StreamingDataset.merge_state_dicts]
before resuming on a different topology.
Packed state also includes partial token buffers and is not topology-
independent because packing and padding happen within each iterator.
"""
if self._pack_sequences is not None:
return {
@@ -757,11 +944,16 @@ class StreamingDataset(IterableDataset):
"samples_consumed_per_split": list(self._pack_consumed),
"pack_buffers": deepcopy(self._pack_buffers),
}
positions = [
self._resume_positions.get(split, self._resume_offset)
for split in range(self._num_splits)
]
return {
"shuffle_seed": self._shuffle_seed,
"num_splits": self._num_splits,
"epoch": self._epoch,
"samples_consumed_per_split": [self._resume_offset] * self._num_splits,
"positions_consumed_per_split": positions,
}
def load_state_dict(self, state: dict) -> None:
@@ -804,3 +996,96 @@ class StreamingDataset(IterableDataset):
self._resume_offset = consumed[0] if consumed else 0
else:
self._resume_offset = int(consumed)
# Older checkpoints predate positions_consumed_per_split; without
# skipped rows positions equal sample counts, so falling back to
# _resume_offset (the .get default in __iter__) is exact.
positions = state.get("positions_consumed_per_split")
if positions is None:
self._resume_positions = {}
else:
self._resume_positions = {
split: int(pos) for split, pos in enumerate(positions)
}
@staticmethod
def merge_state_dicts(states: list[dict]) -> dict:
"""Merge state dicts saved by different ranks into one exact state.
Only needed when ``on_transform_error`` skips rows in multi-rank
training: each rank then knows the exact permutation position only for
its own splits, and records a lower bound for the rest. Because
exactly one rank owns each split, the elementwise maximum across all
ranks' ``positions_consumed_per_split`` recovers the exact position of
every split. Without skipped rows every rank's state is already
identical and merging is a no-op.
Raises ``ValueError`` if the states are empty or were not produced by
the same run (mismatched seed, split count, epoch, or sample counts).
The merge is always all-to-all and topology-agnostic: collect the
``state_dict()`` from every rank of the *previous* run into one list,
merge that whole list, and hand the identical merged result to every
rank of the *next* run — regardless of whether the rank count grew,
shrank, or stayed the same. There is no pairwise or subset merging
step, because each split's exact position is only known to whichever
rank owned that split, and the elementwise maximum needs every rank's
contribution to be correct.
For example, checkpointing 8 ranks and resuming on 4 (the same
pattern applies when growing, e.g. 4 ranks resuming on 8)::
states = [ds.state_dict() for ds in previous_run_datasets] # 8
merged = StreamingDataset.merge_state_dicts(states)
for ds in resumed_datasets: # now only 4 ranks
ds.load_state_dict(merged) # same dict on every rank
The rank count on either side never affects the merge itself, since
``merge_state_dicts`` only cares about the list of states it is
given. Each split's position is recovered by elementwise maximum;
here rank 0 owned split 0 (and skipped two rows there) while rank 1
owned split 1 (and skipped one row):
>>> rank0 = {
... "shuffle_seed": 0, "num_splits": 2, "epoch": 0,
... "samples_consumed_per_split": [3, 3],
... "positions_consumed_per_split": [5, 3],
... }
>>> rank1 = {
... "shuffle_seed": 0, "num_splits": 2, "epoch": 0,
... "samples_consumed_per_split": [3, 3],
... "positions_consumed_per_split": [3, 4],
... }
>>> merged = StreamingDataset.merge_state_dicts([rank0, rank1])
>>> merged["positions_consumed_per_split"]
[5, 4]
"""
if not states:
raise ValueError("merge_state_dicts requires at least one state dict")
first = states[0]
for state in states[1:]:
for key in ("shuffle_seed", "num_splits", "epoch"):
if state[key] != first[key]:
raise ValueError(
f"{key} mismatch across state dicts: "
f"{state[key]} != {first[key]}"
)
if (
state["samples_consumed_per_split"]
!= first["samples_consumed_per_split"]
):
raise ValueError(
"samples_consumed_per_split mismatch across state dicts; "
"state_dict() must be called at the same global step "
"boundary on every rank"
)
merged = dict(first)
all_positions = [
state.get(
"positions_consumed_per_split", state["samples_consumed_per_split"]
)
for state in states
]
merged["positions_consumed_per_split"] = [
max(per_split) for per_split in zip(*all_positions)
]
return merged
python/python/tests/test_elastic_dataloader.py
+402
View File
@@ -1456,6 +1456,408 @@ def test_shuffle_clump_size_yields_all_rows(lance_table):
)
# ---------------------------------------------------------------------------
# on_transform_error tests
# ---------------------------------------------------------------------------
class BadRowError(ValueError):
"""Raised by the failing transforms below when a batch contains a bad id."""
def _failing_transform(bad_ids: set):
"""A transform that raises BadRowError whenever the batch has a bad id.
Raises on the full batch and on any single-row slice containing a bad id,
so per-row isolation drops exactly the bad rows.
"""
def transform(batch: pa.RecordBatch) -> list:
ids = batch.column("id").to_pylist()
bad = sorted(set(ids) & bad_ids)
if bad:
raise BadRowError(f"bad ids in batch: {bad}")
return [{"id": i} for i in ids]
return transform
def _sequential_split_members(table) -> list[list[int]]:
"""Return each split's ids in yield order for shuffle=False.
With a single rank and no workers the round-robin yields one row per split
per cycle, so item k of a clean run belongs to split k % NUM_SPLITS.
"""
ds = StreamingDataset(table, num_splits=NUM_SPLITS, shuffle=False)
members: list[list[int]] = [[] for _ in range(NUM_SPLITS)]
for k, row in enumerate(ds):
members[k % NUM_SPLITS].append(row["id"])
return members
def test_on_transform_error_default_raises(lance_table):
"""By default a transform exception propagates and aborts iteration."""
ds = StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
shuffle_seed=SHUFFLE_SEED,
transform=_failing_transform({7}),
)
with pytest.raises(BadRowError):
list(ds)
def test_on_transform_error_invalid_value(lance_table):
with pytest.raises(ValueError, match="on_transform_error"):
StreamingDataset(lance_table, num_splits=NUM_SPLITS, on_transform_error="bogus")
def test_on_transform_error_skip_drops_bad_rows(lance_table):
"""With one bad row per split, 'skip' yields every good row exactly once
and counts the dropped rows in rows_skipped."""
members = _sequential_split_members(lance_table)
bad_ids = {members[i][4] for i in range(NUM_SPLITS)}
ds = StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
shuffle=False,
transform=_failing_transform(bad_ids),
on_transform_error="skip",
)
assert ds.rows_skipped == 0
ids = [row["id"] for row in ds]
assert sorted(ids) == sorted(set(range(NUM_ROWS)) - bad_ids)
assert ds.rows_skipped == NUM_SPLITS
def test_on_transform_error_skip_uneven_ends_at_last_complete_cycle(lance_table):
"""When one split loses more rows than the others, the epoch ends at the
last cycle where every split still has a row — no crash, no bad rows, and
every step remains one sample per split."""
members = _sequential_split_members(lance_table)
bad_ids = set(members[0][:3]) # all 3 bad rows in split 0
ds = StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
shuffle=False,
transform=_failing_transform(bad_ids),
on_transform_error="skip",
)
items = [row["id"] for row in ds]
rows_per_split = NUM_ROWS // NUM_SPLITS
expected_cycles = rows_per_split - len(bad_ids)
assert len(items) == expected_cycles * NUM_SPLITS
assert len(set(items)) == len(items), "duplicate samples yielded"
assert not set(items) & bad_ids, "a bad row was yielded"
# Split 0 contributed exactly its surviving rows, in order, one per cycle.
survivors = [i for i in members[0] if i not in bad_ids]
assert items[0::NUM_SPLITS] == survivors[:expected_cycles]
def test_on_transform_error_warn_logs(lance_table, caplog):
"""'warn' skips like 'skip' but logs a warning for the failing batch."""
members = _sequential_split_members(lance_table)
bad_ids = {members[i][3] for i in range(NUM_SPLITS)}
ds = StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
shuffle=False,
transform=_failing_transform(bad_ids),
on_transform_error="warn",
)
with caplog.at_level(logging.WARNING, logger="lancedb.streaming"):
items = list(ds)
assert len(items) == NUM_ROWS - NUM_SPLITS
assert ds.rows_skipped == NUM_SPLITS
assert "Skipped" in caplog.text
assert "BadRowError" in caplog.text
def test_on_transform_error_callable_selective(lance_table):
"""A callable handler can skip expected errors and re-raise the rest."""
members = _sequential_split_members(lance_table)
bad_ids = {members[i][0] for i in range(NUM_SPLITS)}
handled: list[Exception] = []
def handler(exc: Exception) -> bool:
handled.append(exc)
return isinstance(exc, BadRowError)
ds = StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
shuffle=False,
transform=_failing_transform(bad_ids),
on_transform_error=handler,
)
items = list(ds)
assert len(items) == NUM_ROWS - NUM_SPLITS
assert handled and all(isinstance(exc, BadRowError) for exc in handled)
def broken_transform(batch: pa.RecordBatch) -> list:
raise TypeError("boom")
ds2 = StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
shuffle=False,
transform=broken_transform,
on_transform_error=handler,
)
with pytest.raises(TypeError, match="boom"):
list(ds2)
def test_transform_wrong_row_count_raises(lance_table):
"""A transform that returns the wrong number of rows is an error even with
on_transform_error='skip' — silent shrinkage would corrupt accounting."""
def drops_rows(batch: pa.RecordBatch) -> list:
return batch.column("id").to_pylist()[:-1]
ds = StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
shuffle_seed=SHUFFLE_SEED,
transform=drops_rows,
on_transform_error="skip",
)
with pytest.raises(ValueError, match="one output row per input row"):
list(ds)
def test_skip_deterministic_across_runs(lance_table):
"""With a fixed seed, skipping produces the identical sample sequence on
every run — skips are data-dependent, not run-dependent."""
bad_ids = {5, 17, 46}
def run() -> tuple[list[int], int]:
ds = StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
shuffle_seed=SHUFFLE_SEED,
transform=_failing_transform(bad_ids),
on_transform_error="skip",
)
return [row["id"] for row in ds], ds.rows_skipped
ids_a, skipped_a = run()
ids_b, skipped_b = run()
assert ids_a == ids_b
assert skipped_a == skipped_b
assert not set(ids_a) & bad_ids
def test_skip_elastic_det_across_world_sizes(lance_table):
"""With equal bad-row counts per split, skipping preserves the full
elastic-determinism guarantee: identical global batches at every step for
every compatible world_size."""
members = _sequential_split_members(lance_table)
bad_ids = {members[i][6] for i in range(NUM_SPLITS)}
def collect(world_size: int) -> list[frozenset[int]]:
micro = GLOBAL_BATCH_SIZE // world_size
iters = [
iter(
StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
shuffle=False,
rank=rank,
world_size=world_size,
transform=_failing_transform(bad_ids),
on_transform_error="skip",
)
)
for rank in range(world_size)
]
_STOP = object()
batches: list[frozenset[int]] = []
while True:
step_samples: set[int] = set()
exhausted = 0
for it in iters:
for _ in range(micro):
val = next(it, _STOP)
if val is _STOP:
exhausted += 1
break
step_samples.add(val["id"])
if exhausted == len(iters):
break
assert exhausted == 0, (
"Rank iterators exhausted at different steps despite equal "
"bad-row counts per split"
)
batches.append(frozenset(step_samples))
return batches
reference = collect(1)
assert len(reference) == NUM_ROWS // NUM_SPLITS - 1
for ws in (2, 3, 4):
assert collect(ws) == reference, f"world_size={ws} diverged"
def test_resumability_with_skips_same_topology(lance_table):
"""Checkpointing mid-epoch with skipped rows resumes exactly: no sample
repeated, no sample lost, skipped rows stay skipped."""
members = _sequential_split_members(lance_table)
# Uneven skips: positions diverge across splits (2 bad in split 0, 1 in
# split 5), which only a position-based checkpoint can resume exactly.
bad_ids = {members[0][2], members[0][3], members[5][7]}
kwargs = dict(
num_splits=NUM_SPLITS,
shuffle=False,
transform=_failing_transform(bad_ids),
on_transform_error="skip",
)
reference = [row["id"] for row in StreamingDataset(lance_table, **kwargs)]
rows_per_split = NUM_ROWS // NUM_SPLITS
assert len(reference) == (rows_per_split - 2) * NUM_SPLITS
steps = 3
ds = StreamingDataset(lance_table, **kwargs)
it = iter(ds)
consumed = [next(it)["id"] for _ in range(steps * NUM_SPLITS)]
checkpoint = ds.state_dict()
it.close()
# Split 0 skipped positions 2 and 3 within its first 3 yields; split 5's
# bad row is beyond the checkpoint. Everything else is at 3 = the sample
# count.
positions = checkpoint["positions_consumed_per_split"]
assert positions[0] == 5
assert positions[1:] == [3] * (NUM_SPLITS - 1)
assert checkpoint["samples_consumed_per_split"] == [3] * NUM_SPLITS
ds2 = StreamingDataset(lance_table, **kwargs)
ds2.load_state_dict(checkpoint)
resumed = [row["id"] for row in ds2]
assert consumed == reference[: steps * NUM_SPLITS]
assert resumed == reference[steps * NUM_SPLITS :]
def test_resumability_with_skips_elastic_merge(lance_table):
"""Elastic resume with skips: each rank's checkpoint knows exact positions
only for its own splits; merge_state_dicts recovers the global state, and
a run on a different world_size continues exactly."""
members = _sequential_split_members(lance_table)
# Bad rows early in split 0 (rank 0) and split 6 (rank 1 of a ws=2 run) so
# both ranks' position vectors diverge before the checkpoint.
bad_ids = {members[0][0], members[0][2], members[6][1]}
kwargs = dict(
num_splits=NUM_SPLITS,
shuffle=False,
transform=_failing_transform(bad_ids),
on_transform_error="skip",
)
reference = [row["id"] for row in StreamingDataset(lance_table, **kwargs)]
steps = 3
world_size = 2
micro = GLOBAL_BATCH_SIZE // world_size
datasets = [
StreamingDataset(lance_table, rank=rank, world_size=world_size, **kwargs)
for rank in range(world_size)
]
iters = [iter(ds) for ds in datasets]
seen: list[frozenset[int]] = []
for _ in range(steps):
step_samples = set()
for it in iters:
for _ in range(micro):
step_samples.add(next(it)["id"])
seen.append(frozenset(step_samples))
states = [ds.state_dict() for ds in datasets]
for it in iters:
it.close()
merged = StreamingDataset.merge_state_dicts(states)
expected_positions = [3] * NUM_SPLITS
expected_positions[0] = 5 # skipped positions 0 and 2
expected_positions[6] = 4 # skipped position 1
assert merged["positions_consumed_per_split"] == expected_positions
# The first 3 global batches match the world_size=1 reference.
ref_batches = [
frozenset(reference[s * NUM_SPLITS : (s + 1) * NUM_SPLITS])
for s in range(len(reference) // NUM_SPLITS)
]
assert seen == ref_batches[:steps]
# Resume on world_size=1 from the merged state.
ds_resume = StreamingDataset(lance_table, **kwargs)
ds_resume.load_state_dict(merged)
resumed = [row["id"] for row in ds_resume]
assert resumed == reference[steps * NUM_SPLITS :]
def test_rows_skipped_flushed_when_split_entirely_bad(lance_table):
"""A split whose rows all fail never completes a cycle, so the epoch ends
immediately — but rows_skipped must still report the drops after the
iterator exits (the shared-memory counter is flushed on exhaustion)."""
members = _sequential_split_members(lance_table)
bad_ids = set(members[0]) # every row of split 0 is bad
ds = StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
shuffle=False,
transform=_failing_transform(bad_ids),
on_transform_error="skip",
)
assert list(ds) == []
assert ds.rows_skipped == len(bad_ids)
def test_merge_state_dicts_validates_consistency(lance_table):
ds = StreamingDataset(lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED)
state = ds.state_dict()
other = dict(state, shuffle_seed=SHUFFLE_SEED + 1)
with pytest.raises(ValueError, match="shuffle_seed mismatch"):
StreamingDataset.merge_state_dicts([state, other])
with pytest.raises(ValueError, match="at least one"):
StreamingDataset.merge_state_dicts([])
def test_load_state_dict_without_positions_key(lance_table):
"""Checkpoints from before positions_consumed_per_split existed still
resume exactly (positions equal sample counts when nothing is skipped)."""
reference = [
row["id"]
for row in StreamingDataset(
lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED
)
]
steps = 4
ds = StreamingDataset(lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED)
it = iter(ds)
for _ in range(steps * NUM_SPLITS):
next(it)
checkpoint = ds.state_dict()
it.close()
del checkpoint["positions_consumed_per_split"]
ds2 = StreamingDataset(
lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED
)
ds2.load_state_dict(checkpoint)
resumed = [row["id"] for row in ds2]
assert resumed == reference[steps * NUM_SPLITS :]
def test_num_splits_defaults_to_world_size(lance_table):
"""Omitting num_splits gives world_size splits (one per rank)."""
ds = StreamingDataset(
python/src/index.rs
+1 -1
View File
@@ -289,7 +289,7 @@ struct IvfHnswFlatParams {
target_partition_size: Option<u32>,
}
#[pyclass(get_all)]
#[pyclass(module = "lancedb._lancedb", get_all)]
/// A description of an index currently configured on a column
pub struct IndexConfig {
/// The type of the index
python/src/session.rs
+1 -1
View File
@@ -11,7 +11,7 @@ use pyo3::{PyResult, pyclass, pymethods};
/// Sessions allow you to configure cache sizes for index and metadata caches,
/// which can significantly impact memory use and performance. They can
/// also be re-used across multiple connections to share the same cache state.
#[pyclass(from_py_object)]
#[pyclass(module = "lancedb._lancedb", from_py_object)]
#[derive(Clone)]
pub struct Session {
pub(crate) inner: Arc<LanceSession>,
python/src/table.rs
+1 -1
View File
@@ -579,7 +579,7 @@ impl PyBlobFile {
}
}
#[pyclass(get_all, from_py_object)]
#[pyclass(module = "lancedb._lancedb", get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct FtsToken {
pub text: String,