remove deepcopy

2026-08-18 12:08:35 +00:00 · 2026-08-18 16:40:19 +05:30
parent d0405bf1db
commit 0048dde542
1 changed files with 7 additions and 1 deletions
@@ -789,7 +789,13 @@ class StreamingDataset(IterableDataset):

        def _commit_pack_state() -> None:
            self._pack_consumed = list(pack_consumed)
-            self._pack_buffers = deepcopy(pack_buffers)
+            self._pack_buffers = {
+                split: {
+                    "tokens": list(buffer["tokens"]),
+                    "starts": list(buffer["starts"]),
+                }
+                for split, buffer in pack_buffers.items()
+            }
            self._pack_blocks_emitted = list(pack_blocks_emitted)

        # ── Main loop ─────────────────────────────────────────────────────────