From 3c138bd8632f0d2fc72af0d0e50bd8cb4bd98a8b Mon Sep 17 00:00:00 2001 From: OrcaWin Date: Thu, 17 Sep 2026 20:12:19 -0700 Subject: [PATCH] Skip empty chunks in streamed agent text (#21142) * fix: skip empty chunks in streamed agent text * test: lint empty-delta retention reproducer --------- Co-authored-by: m4air --- .../empty-streamed-delta-retention/README.md | 36 + .../before.config.mjs | 30 + .../electron-results.json | 865 ++++++++++++++++++ .../empty-streamed-delta-retention/fix.patch | 7 + .../node-results.json | 864 +++++++++++++++++ .../reported.patch | 60 ++ .../reproduce.cjs | 66 ++ .../scenario.cjs | 156 ++++ .../source-versions.json | 77 ++ .../sources.cjs | 111 +++ .../validation.json | 50 + .../agent-session-delta-coalescer.ts | 4 +- ...gent-session-empty-delta-retention.test.ts | 126 +++ 13 files changed, 2451 insertions(+), 1 deletion(-) create mode 100644 docs/audits/empty-streamed-delta-retention/README.md create mode 100644 docs/audits/empty-streamed-delta-retention/before.config.mjs create mode 100644 docs/audits/empty-streamed-delta-retention/electron-results.json create mode 100644 docs/audits/empty-streamed-delta-retention/fix.patch create mode 100644 docs/audits/empty-streamed-delta-retention/node-results.json create mode 100644 docs/audits/empty-streamed-delta-retention/reported.patch create mode 100644 docs/audits/empty-streamed-delta-retention/reproduce.cjs create mode 100644 docs/audits/empty-streamed-delta-retention/scenario.cjs create mode 100644 docs/audits/empty-streamed-delta-retention/source-versions.json create mode 100644 docs/audits/empty-streamed-delta-retention/sources.cjs create mode 100644 docs/audits/empty-streamed-delta-retention/validation.json create mode 100644 src/main/native-chat/agent-session-wire/agent-session-empty-delta-retention.test.ts diff --git a/docs/audits/empty-streamed-delta-retention/README.md b/docs/audits/empty-streamed-delta-retention/README.md new file mode 100644 index 00000000000..d872c2ced4e --- /dev/null +++ b/docs/audits/empty-streamed-delta-retention/README.md @@ -0,0 +1,36 @@ +# Empty streamed deltas retain array entries + +The text coalescer charged streamed text by UTF-8 bytes but appended an array entry for every empty delta. A live stream receiving repeated empty updates could retain an increasing number of entries while both byte counters stayed zero. Flushing published a joined string and kept the entries. The actual Codex notification path accepts `delta: ''`; this diagnostic exercises its stream handler and coalescer. + +The fix skips only the empty `chunks.push` operation. Empty-stream creation, snapshots, dirty state, scheduled publication, callback receiver, backpressure and eviction remain unchanged. + +## Reproduce + +```sh +ORCA_BACKGROUND_LAUNCH=1 node docs/audits/empty-streamed-delta-retention/reproduce.cjs +``` + +The runner reverses hash-checked patches in memory and checks every bundled source dependency. It changes no product files and starts no native process or UI. A bounded CRLF control checks source and patch loading. Reports were recorded on Node 26.6.0 and Electron 43.7.0's Node 24.21.0. + +| Control | Before | Fixed | +| -------------------------------------------------------------------- | ------------------------------ | ------------------------- | +| Four batches of 16,384 empty Codex deltas, flushing each batch | 16,384 → 65,536 retained slots | 0 slots after every batch | +| Logical stream count | 1 | 1 | +| Accounted / observed text bytes | 0 / 0 | 0 / 0 | +| Scheduled callbacks / published rows in the complete caller scenario | 6 / 5 | 6 / 5 | +| Append `hé` after empty updates | Same 3-byte text | Same 3-byte text | +| Forget and disposal | Clear retained state | Clear retained state | + +The runner compares the entire recorded publication and scheduling behavior before/after. Controls also cover first-empty snapshots, failed publication and retry, rejection of a new empty key while the previous stream is backpressured, accepted eviction, callback receiver, UTF-8 truncation and an empty update after truncation. The two runtimes each execute four source phases: current/main before and fixed, plus the v1.4.198 coalescer before and with the same narrow guard. + +The permanent regression invokes the actual Codex stream caller. A temporary `Array.prototype.join` observer measures the matching chunk array only during synchronous snapshot creation, then restores the method. The baseline fails with 65,537 slots versus the expected single nonempty prefix; the other 14 coalescer controls pass. All 64 focused compatibility tests pass with the fix. See [validation.json](./validation.json). + +## Source and incident scope + +The current baseline is byte-identical to the coalescer at named main commit `291b4ddd6f1c1af480169885e0fda7f9c78ff053`. The exact v1.4.198 coalescer contains the same unconditional empty append; its surrounding implementation differs. Historical phases replace only that module and use the recorded current Codex caller/dependencies. This is a source overlay, not a packaged historical application replay. [source-versions.json](./source-versions.json) records these distinctions and named caller hashes. + +Claude's generic checkpoint API also uses the coalescer, but its ordinary provider path rejects empty text in `claude-streamed-block-identity.ts` before calling it. This artifact demonstrates the Codex path and preserves Claude compatibility; it does not claim an ordinary Claude trigger. + +Measurement instrumentation reads private map and array cardinalities without changing their contents. These are retained-entry counts, not heap, RSS or byte measurements. The fixture keeps the live stream owned until forget/disposal; it does not establish retention after all owners collect. Nonempty one-byte deltas can still have substantial array overhead within the text-byte allowance, and overflow concatenation has its own transient cost. + +No affected-host data establishes how often Codex emitted empty updates in #19831 or another incident. The finding is a reproducible code-level growth mechanism present in the reported release. It does not attribute an app-scope OOM total to this mechanism or establish its incident magnitude. No remote protocol, process liveness, process termination or terminal ownership behavior changes. diff --git a/docs/audits/empty-streamed-delta-retention/before.config.mjs b/docs/audits/empty-streamed-delta-retention/before.config.mjs new file mode 100644 index 00000000000..5ba89ae6d35 --- /dev/null +++ b/docs/audits/empty-streamed-delta-retention/before.config.mjs @@ -0,0 +1,30 @@ +import base from '../../../config/vitest.config.ts' +import { createRequire } from 'node:module' +import { join } from 'node:path' + +const require = createRequire(import.meta.url) +const { loadSources, root, versions } = require('./sources.cjs') +const baseline = loadSources().baseline +const target = join(root, versions.sourcePath).replaceAll('\\', '/') + +export default { + ...base, + test: { + ...base.test, + include: [ + 'src/main/native-chat/agent-session-wire/agent-session-delta-coalescer.test.ts', + 'src/main/native-chat/agent-session-wire/agent-session-empty-delta-retention.test.ts' + ] + }, + plugins: [ + { + name: 'exact-baseline-coalescer', + enforce: 'pre', + transform(_code, id) { + return id.replaceAll('\\', '/').split('?')[0] === target + ? { code: baseline, map: null } + : null + } + } + ] +} diff --git a/docs/audits/empty-streamed-delta-retention/electron-results.json b/docs/audits/empty-streamed-delta-retention/electron-results.json new file mode 100644 index 00000000000..d4d8ef5a026 --- /dev/null +++ b/docs/audits/empty-streamed-delta-retention/electron-results.json @@ -0,0 +1,865 @@ +{ + "runtime": { + "node": "24.21.0", + "acorn": "8.18.0", + "ada": "4.0.0", + "amaro": "1.1.11", + "ares": "1.34.8", + "brotli": "1.2.0", + "cldr": "48.0", + "icu": "78.2", + "llhttp": "9.4.3", + "merve": "1.2.2", + "modules": "148", + "napi": "10", + "nbytes": "0.1.4", + "ncrypto": "0.0.1", + "nghttp2": "1.70.0", + "nghttp3": "", + "ngtcp2": "", + "openssl": "0.0.0", + "simdjson": "4.6.7", + "simdutf": "7.7.0", + "sqlite": "3.53.4", + "tz": "2025c", + "undici": "7.29.1", + "unicode": "17.0", + "uv": "1.52.1", + "uvwasi": "0.0.23", + "v8": "15.0.245.31-electron.0", + "zlib": "1.3.2.1-motley", + "zstd": "1.6.0", + "electron": "43.7.0", + "chrome": "150.0.7871.250" + }, + "sourceVersions": { + "main": "291b4ddd6f1c1af480169885e0fda7f9c78ff053", + "reported": "e0826956fcfc532f5a1e55b5e081f2e57e553c43", + "reportedTag": "v1.4.198" + }, + "scope": "Exact current/main coalescer before/after plus exact v1.4.198 coalescer and same local guard, using current Codex stream callers/dependencies. Not a packaged historical release replay.", + "crlfLoaderControl": { + "reads": 3, + "equal": true + }, + "artifactHashes": { + "sources.cjs": "7902098e4cd0e07825684c778d7d2af74ba1438a55ccf0b9ffe6eb96f76698c8", + "scenario.cjs": "66cfb53b821f631d2f57988c2fab5d28aa0da95fb4a6a3a2b8b643a82d233550", + "reproduce.cjs": "e55b45abda1a7c6078cc3fc867f2553cf9ed6ffea10fbffdb61c99494d797fe1", + "before.config.mjs": "a55e7576be2348edddc137af9c34fca4323bd060d325a1140ab0ede66618c6e9", + "source-versions.json": "1038900fcbc3310ae4b38eb8603d0eea3d6a417c3835a70f3537d27eee6debc9", + "fix.patch": "06f11750dcd64042b7f208b00ae0feb3e0bdea6d5db66b3823ce063fce8ab97a", + "reported.patch": "9d340925bd2a875334a1a4c3ce58c4ba0f977c66bf0c6f5d66848f862fc95d59" + }, + "phases": { + "baseline": { + "sourceSha256": "48828e4ee21858075cbb87ec1caa4a82991a55f80928482615d1df7ec5dc0fb2", + "bundleSha256": "cbdd64b9e21f410645660ac33afe3bede8a58b7680641d5d98883facc0e6a120", + "samples": [ + { + "streams": 1, + "slots": 16384, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 32768, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 49152, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 65536, + "retainedBytes": 0, + "observedBytes": 0 + } + ], + "behavior": { + "scheduled": 6, + "published": 5, + "publications": [ + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "hé" + } + ] + } + } + ], + "directScheduled": 7, + "emitted": [ + { + "key": "one", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "one", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "one", + "text": "unchanged", + "snapshot": { + "text": "unchanged", + "observedBytes": 9, + "truncated": false + } + }, + { + "key": "one", + "text": "unchanged", + "snapshot": { + "text": "unchanged", + "observedBytes": 9, + "truncated": false + } + }, + { + "key": "two", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "two", + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "snapshot": { + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "observedBytes": 400, + "truncated": true + } + } + ], + "truncated": { + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "observedBytes": 400, + "truncated": true + } + }, + "controls": [ + "actual Codex empty notification stream", + "empty snapshot remains present", + "four explicit flushes", + "Unicode text retained", + "forget clears", + "dispose clears", + "first empty publication and retries", + "emit receiver preserved", + "new empty key rejected under backpressure", + "accepted eviction", + "UTF-8 truncation", + "already-truncated empty append keeps no-new-publication behavior" + ] + }, + "fixed": { + "sourceSha256": "7caf0f24250b42e0ac4fe99e93bcbe0a4acb7cb3465fdde6b248d8bf6ba0d2d0", + "bundleSha256": "3e8b7e8430737f4cb0ca8454add7730770d8cd19ab175266d8b7626a059f3912", + "samples": [ + { + "streams": 1, + "slots": 0, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 0, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 0, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 0, + "retainedBytes": 0, + "observedBytes": 0 + } + ], + "behavior": { + "scheduled": 6, + "published": 5, + "publications": [ + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "hé" + } + ] + } + } + ], + "directScheduled": 7, + "emitted": [ + { + "key": "one", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "one", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "one", + "text": "unchanged", + "snapshot": { + "text": "unchanged", + "observedBytes": 9, + "truncated": false + } + }, + { + "key": "one", + "text": "unchanged", + "snapshot": { + "text": "unchanged", + "observedBytes": 9, + "truncated": false + } + }, + { + "key": "two", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "two", + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "snapshot": { + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "observedBytes": 400, + "truncated": true + } + } + ], + "truncated": { + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "observedBytes": 400, + "truncated": true + } + }, + "controls": [ + "actual Codex empty notification stream", + "empty snapshot remains present", + "four explicit flushes", + "Unicode text retained", + "forget clears", + "dispose clears", + "first empty publication and retries", + "emit receiver preserved", + "new empty key rejected under backpressure", + "accepted eviction", + "UTF-8 truncation", + "already-truncated empty append keeps no-new-publication behavior" + ] + }, + "reported": { + "sourceSha256": "bfff14bd820a2d7be90e82db6403d2415c0fb3de998cf61d7c3830ff4c415605", + "bundleSha256": "c8e429f7f35a4a61c3189fcedcac5abc9658dab841ea927cf285f90eeccd381c", + "samples": [ + { + "streams": 1, + "slots": 16384, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 32768, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 49152, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 65536, + "retainedBytes": 0, + "observedBytes": 0 + } + ], + "behavior": { + "scheduled": 6, + "published": 5, + "publications": [ + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "hé" + } + ] + } + } + ], + "directScheduled": 7, + "emitted": [ + { + "key": "one", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "one", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "one", + "text": "unchanged", + "snapshot": { + "text": "unchanged", + "observedBytes": 9, + "truncated": false + } + }, + { + "key": "one", + "text": "unchanged", + "snapshot": { + "text": "unchanged", + "observedBytes": 9, + "truncated": false + } + }, + { + "key": "two", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "two", + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "snapshot": { + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "observedBytes": 400, + "truncated": true + } + } + ], + "truncated": { + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "observedBytes": 400, + "truncated": true + } + }, + "controls": [ + "actual Codex empty notification stream", + "empty snapshot remains present", + "four explicit flushes", + "Unicode text retained", + "forget clears", + "dispose clears", + "first empty publication and retries", + "emit receiver preserved", + "new empty key rejected under backpressure", + "accepted eviction", + "UTF-8 truncation", + "already-truncated empty append keeps no-new-publication behavior" + ] + }, + "reportedFixed": { + "sourceSha256": "2f00aec9b7a4fb24cad4e01d1bf7e1d0b881076f9760b322cdf72c39594ae89c", + "bundleSha256": "3749aae7c5eb00f3eea98c66a7b0c4e0e7114cde1d983d89951bd4c21b33e680", + "samples": [ + { + "streams": 1, + "slots": 0, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 0, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 0, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 0, + "retainedBytes": 0, + "observedBytes": 0 + } + ], + "behavior": { + "scheduled": 6, + "published": 5, + "publications": [ + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "hé" + } + ] + } + } + ], + "directScheduled": 7, + "emitted": [ + { + "key": "one", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "one", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "one", + "text": "unchanged", + "snapshot": { + "text": "unchanged", + "observedBytes": 9, + "truncated": false + } + }, + { + "key": "one", + "text": "unchanged", + "snapshot": { + "text": "unchanged", + "observedBytes": 9, + "truncated": false + } + }, + { + "key": "two", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "two", + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "snapshot": { + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "observedBytes": 400, + "truncated": true + } + } + ], + "truncated": { + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "observedBytes": 400, + "truncated": true + } + }, + "controls": [ + "actual Codex empty notification stream", + "empty snapshot remains present", + "four explicit flushes", + "Unicode text retained", + "forget clears", + "dispose clears", + "first empty publication and retries", + "emit receiver preserved", + "new empty key rejected under backpressure", + "accepted eviction", + "UTF-8 truncation", + "already-truncated empty append keeps no-new-publication behavior" + ] + } + }, + "measurement": "Read-only closure observes private Map and chunk-array cardinalities in source overlay. No heap/RSS measurement, native process or affected-host inference." +} diff --git a/docs/audits/empty-streamed-delta-retention/fix.patch b/docs/audits/empty-streamed-delta-retention/fix.patch new file mode 100644 index 00000000000..b450bfc909f --- /dev/null +++ b/docs/audits/empty-streamed-delta-retention/fix.patch @@ -0,0 +1,7 @@ +--- a/src/main/native-chat/agent-session-wire/agent-session-delta-coalescer.ts ++++ b/src/main/native-chat/agent-session-wire/agent-session-delta-coalescer.ts +@@ -233 +233,3 @@ +- current.push(delta) ++ if (delta.length > 0) { ++ current.push(delta) ++ } diff --git a/docs/audits/empty-streamed-delta-retention/node-results.json b/docs/audits/empty-streamed-delta-retention/node-results.json new file mode 100644 index 00000000000..c614c1675e7 --- /dev/null +++ b/docs/audits/empty-streamed-delta-retention/node-results.json @@ -0,0 +1,864 @@ +{ + "runtime": { + "node": "26.6.0", + "acorn": "8.17.0", + "ada": "4.0.0", + "amaro": "1.1.11", + "ares": "1.34.8", + "brotli": "1.2.0", + "cldr": "48.0", + "icu": "78.3", + "libffi": "3.7.1", + "llhttp": "9.4.3", + "merve": "1.2.2", + "modules": "147", + "napi": "10", + "nbytes": "0.1.4", + "ncrypto": "0.0.1", + "nghttp2": "1.70.0", + "nghttp3": "", + "ngtcp2": "", + "openssl": "3.6.3", + "simdjson": "4.6.6", + "simdutf": "7.7.0", + "sqlite": "3.53.4", + "tz": "2026a", + "undici": "8.9.0", + "unicode": "17.0", + "uv": "1.52.1", + "uvwasi": "0.0.23", + "v8": "14.6.202.34-node.26", + "zlib": "1.2.12", + "zstd": "1.5.7" + }, + "sourceVersions": { + "main": "291b4ddd6f1c1af480169885e0fda7f9c78ff053", + "reported": "e0826956fcfc532f5a1e55b5e081f2e57e553c43", + "reportedTag": "v1.4.198" + }, + "scope": "Exact current/main coalescer before/after plus exact v1.4.198 coalescer and same local guard, using current Codex stream callers/dependencies. Not a packaged historical release replay.", + "crlfLoaderControl": { + "reads": 3, + "equal": true + }, + "artifactHashes": { + "sources.cjs": "7902098e4cd0e07825684c778d7d2af74ba1438a55ccf0b9ffe6eb96f76698c8", + "scenario.cjs": "66cfb53b821f631d2f57988c2fab5d28aa0da95fb4a6a3a2b8b643a82d233550", + "reproduce.cjs": "e55b45abda1a7c6078cc3fc867f2553cf9ed6ffea10fbffdb61c99494d797fe1", + "before.config.mjs": "a55e7576be2348edddc137af9c34fca4323bd060d325a1140ab0ede66618c6e9", + "source-versions.json": "1038900fcbc3310ae4b38eb8603d0eea3d6a417c3835a70f3537d27eee6debc9", + "fix.patch": "06f11750dcd64042b7f208b00ae0feb3e0bdea6d5db66b3823ce063fce8ab97a", + "reported.patch": "9d340925bd2a875334a1a4c3ce58c4ba0f977c66bf0c6f5d66848f862fc95d59" + }, + "phases": { + "baseline": { + "sourceSha256": "48828e4ee21858075cbb87ec1caa4a82991a55f80928482615d1df7ec5dc0fb2", + "bundleSha256": "cbdd64b9e21f410645660ac33afe3bede8a58b7680641d5d98883facc0e6a120", + "samples": [ + { + "streams": 1, + "slots": 16384, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 32768, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 49152, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 65536, + "retainedBytes": 0, + "observedBytes": 0 + } + ], + "behavior": { + "scheduled": 6, + "published": 5, + "publications": [ + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "hé" + } + ] + } + } + ], + "directScheduled": 7, + "emitted": [ + { + "key": "one", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "one", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "one", + "text": "unchanged", + "snapshot": { + "text": "unchanged", + "observedBytes": 9, + "truncated": false + } + }, + { + "key": "one", + "text": "unchanged", + "snapshot": { + "text": "unchanged", + "observedBytes": 9, + "truncated": false + } + }, + { + "key": "two", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "two", + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "snapshot": { + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "observedBytes": 400, + "truncated": true + } + } + ], + "truncated": { + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "observedBytes": 400, + "truncated": true + } + }, + "controls": [ + "actual Codex empty notification stream", + "empty snapshot remains present", + "four explicit flushes", + "Unicode text retained", + "forget clears", + "dispose clears", + "first empty publication and retries", + "emit receiver preserved", + "new empty key rejected under backpressure", + "accepted eviction", + "UTF-8 truncation", + "already-truncated empty append keeps no-new-publication behavior" + ] + }, + "fixed": { + "sourceSha256": "7caf0f24250b42e0ac4fe99e93bcbe0a4acb7cb3465fdde6b248d8bf6ba0d2d0", + "bundleSha256": "3e8b7e8430737f4cb0ca8454add7730770d8cd19ab175266d8b7626a059f3912", + "samples": [ + { + "streams": 1, + "slots": 0, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 0, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 0, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 0, + "retainedBytes": 0, + "observedBytes": 0 + } + ], + "behavior": { + "scheduled": 6, + "published": 5, + "publications": [ + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "hé" + } + ] + } + } + ], + "directScheduled": 7, + "emitted": [ + { + "key": "one", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "one", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "one", + "text": "unchanged", + "snapshot": { + "text": "unchanged", + "observedBytes": 9, + "truncated": false + } + }, + { + "key": "one", + "text": "unchanged", + "snapshot": { + "text": "unchanged", + "observedBytes": 9, + "truncated": false + } + }, + { + "key": "two", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "two", + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "snapshot": { + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "observedBytes": 400, + "truncated": true + } + } + ], + "truncated": { + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "observedBytes": 400, + "truncated": true + } + }, + "controls": [ + "actual Codex empty notification stream", + "empty snapshot remains present", + "four explicit flushes", + "Unicode text retained", + "forget clears", + "dispose clears", + "first empty publication and retries", + "emit receiver preserved", + "new empty key rejected under backpressure", + "accepted eviction", + "UTF-8 truncation", + "already-truncated empty append keeps no-new-publication behavior" + ] + }, + "reported": { + "sourceSha256": "bfff14bd820a2d7be90e82db6403d2415c0fb3de998cf61d7c3830ff4c415605", + "bundleSha256": "c8e429f7f35a4a61c3189fcedcac5abc9658dab841ea927cf285f90eeccd381c", + "samples": [ + { + "streams": 1, + "slots": 16384, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 32768, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 49152, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 65536, + "retainedBytes": 0, + "observedBytes": 0 + } + ], + "behavior": { + "scheduled": 6, + "published": 5, + "publications": [ + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "hé" + } + ] + } + } + ], + "directScheduled": 7, + "emitted": [ + { + "key": "one", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "one", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "one", + "text": "unchanged", + "snapshot": { + "text": "unchanged", + "observedBytes": 9, + "truncated": false + } + }, + { + "key": "one", + "text": "unchanged", + "snapshot": { + "text": "unchanged", + "observedBytes": 9, + "truncated": false + } + }, + { + "key": "two", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "two", + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "snapshot": { + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "observedBytes": 400, + "truncated": true + } + } + ], + "truncated": { + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "observedBytes": 400, + "truncated": true + } + }, + "controls": [ + "actual Codex empty notification stream", + "empty snapshot remains present", + "four explicit flushes", + "Unicode text retained", + "forget clears", + "dispose clears", + "first empty publication and retries", + "emit receiver preserved", + "new empty key rejected under backpressure", + "accepted eviction", + "UTF-8 truncation", + "already-truncated empty append keeps no-new-publication behavior" + ] + }, + "reportedFixed": { + "sourceSha256": "2f00aec9b7a4fb24cad4e01d1bf7e1d0b881076f9760b322cdf72c39594ae89c", + "bundleSha256": "3749aae7c5eb00f3eea98c66a7b0c4e0e7114cde1d983d89951bd4c21b33e680", + "samples": [ + { + "streams": 1, + "slots": 0, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 0, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 0, + "retainedBytes": 0, + "observedBytes": 0 + }, + { + "streams": 1, + "slots": 0, + "retainedBytes": 0, + "observedBytes": 0 + } + ], + "behavior": { + "scheduled": 6, + "published": 5, + "publications": [ + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "" + } + ] + } + }, + { + "identity": { + "provider": "codex", + "threadId": "thread-a", + "turnId": "turn-a", + "ordinal": 0 + }, + "body": { + "kind": "message", + "role": "assistant", + "blocks": [ + { + "type": "text", + "text": "hé" + } + ] + } + } + ], + "directScheduled": 7, + "emitted": [ + { + "key": "one", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "one", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "one", + "text": "unchanged", + "snapshot": { + "text": "unchanged", + "observedBytes": 9, + "truncated": false + } + }, + { + "key": "one", + "text": "unchanged", + "snapshot": { + "text": "unchanged", + "observedBytes": 9, + "truncated": false + } + }, + { + "key": "two", + "text": "", + "snapshot": { + "text": "", + "observedBytes": 0, + "truncated": false + } + }, + { + "key": "two", + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "snapshot": { + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "observedBytes": 400, + "truncated": true + } + } + ], + "truncated": { + "text": "😀😀😀😀😀😀😀\n[Orca: streamed output truncated]", + "observedBytes": 400, + "truncated": true + } + }, + "controls": [ + "actual Codex empty notification stream", + "empty snapshot remains present", + "four explicit flushes", + "Unicode text retained", + "forget clears", + "dispose clears", + "first empty publication and retries", + "emit receiver preserved", + "new empty key rejected under backpressure", + "accepted eviction", + "UTF-8 truncation", + "already-truncated empty append keeps no-new-publication behavior" + ] + } + }, + "measurement": "Read-only closure observes private Map and chunk-array cardinalities in source overlay. No heap/RSS measurement, native process or affected-host inference." +} diff --git a/docs/audits/empty-streamed-delta-retention/reported.patch b/docs/audits/empty-streamed-delta-retention/reported.patch new file mode 100644 index 00000000000..51ded40b8a6 --- /dev/null +++ b/docs/audits/empty-streamed-delta-retention/reported.patch @@ -0,0 +1,60 @@ +--- a/src/main/native-chat/agent-session-wire/agent-session-delta-coalescer.ts ++++ b/src/main/native-chat/agent-session-wire/agent-session-delta-coalescer.ts +@@ -39,0 +40,2 @@ ++ /** The caller byte-bounds protected metadata; only ordinary streams use the count cap. */ ++ isProtected?: (key: string) => boolean +@@ -86,2 +88 @@ +- const streamOrder = new Map() +- let nextOrder = 0 ++ const evictable = new Set() +@@ -133,2 +134,2 @@ +- if (streams.size >= maxStreams) { +- const oldest = [...streamOrder.entries()].sort((a, b) => a[1] - b[1])[0]?.[0] ++ while (!deps.isProtected?.(key) && evictable.size >= maxStreams) { ++ const oldest = evictable.values().next().value +@@ -135,0 +137,4 @@ ++ if (deps.isProtected?.(oldest)) { ++ evictable.delete(oldest) ++ continue ++ } +@@ -146 +151 @@ +- streamOrder.delete(oldest) ++ evictable.delete(oldest) +@@ -147,0 +153 @@ ++ break +@@ -156 +162,5 @@ +- streamOrder.set(key, nextOrder++) ++ if (!deps.isProtected?.(key)) { ++ evictable.add(key) ++ } ++ } else if (deps.isProtected?.(key)) { ++ evictable.delete(key) +@@ -158 +168,2 @@ +- stream.observedBytes += Buffer.byteLength(delta, 'utf8') ++ const deltaBytes = Buffer.byteLength(delta, 'utf8') ++ stream.observedBytes += deltaBytes +@@ -165,0 +177 @@ ++ deltaBytes, +@@ -187 +199 @@ +- streamOrder.delete(key) ++ evictable.delete(key) +@@ -194 +206 @@ +- streamOrder.clear() ++ evictable.clear() +@@ -213,0 +226 @@ ++ deltaBytes: number, +@@ -217,2 +230 @@ +- const deltaBuffer = Buffer.from(delta, 'utf8') +- if (deltaBuffer.byteLength <= available) { ++ if (deltaBytes <= available) { +@@ -221 +233,3 @@ +- current.push(delta) ++ if (delta.length > 0) { ++ current.push(delta) ++ } +@@ -224 +238 @@ +- retainedBytes: currentBytes + deltaBuffer.byteLength, ++ retainedBytes: currentBytes + deltaBytes, +@@ -232 +246 @@ +- deltaBuffer ++ Buffer.from(delta, 'utf8') diff --git a/docs/audits/empty-streamed-delta-retention/reproduce.cjs b/docs/audits/empty-streamed-delta-retention/reproduce.cjs new file mode 100644 index 00000000000..6a947ca003e --- /dev/null +++ b/docs/audits/empty-streamed-delta-retention/reproduce.cjs @@ -0,0 +1,66 @@ +const assert = require('node:assert/strict') +const { readFileSync, writeFileSync } = require('node:fs') +const path = require('node:path') +const { scenario } = require('./scenario.cjs') +const { loadSources, sha, versions } = require('./sources.cjs') + +assert.equal(process.env.ORCA_BACKGROUND_LAUNCH, '1', 'Run with ORCA_BACKGROUND_LAUNCH=1') + +;(async () => { + const canonical = loadSources() + let crlfReads = 0 + const crlf = loadSources((file) => { + crlfReads += 1 + return readFileSync(file, 'utf8').replaceAll('\r\n', '\n').replaceAll('\n', '\r\n') + }) + assert.deepEqual(crlf, canonical) + assert.equal(crlfReads, 3) + const phases = {} + for (const phase of ['baseline', 'fixed', 'reported', 'reportedFixed']) { + phases[phase] = await scenario(phase) + } + assert.deepEqual(phases.baseline.behavior, phases.fixed.behavior) + assert.deepEqual(phases.reported.behavior, phases.reportedFixed.behavior) + const artifactHashes = Object.fromEntries( + [ + 'sources.cjs', + 'scenario.cjs', + 'reproduce.cjs', + 'before.config.mjs', + 'source-versions.json', + 'fix.patch', + 'reported.patch' + ].map((file) => [file, sha(readFileSync(path.join(__dirname, file)))]) + ) + const result = { + runtime: process.versions, + sourceVersions: versions.namedReferences, + scope: versions.scope, + crlfLoaderControl: { reads: crlfReads, equal: true }, + artifactHashes, + phases, + measurement: + 'Read-only closure observes private Map and chunk-array cardinalities in source overlay. No heap/RSS measurement, native process or affected-host inference.' + } + const output = process.argv[2] ?? path.join(__dirname, 'node-results.json') + writeFileSync(output, `${JSON.stringify(result, null, 2)}\n`) + console.log( + JSON.stringify({ + output, + phases: Object.fromEntries( + Object.entries(phases).map(([phase, result]) => [ + phase, + { + samples: result.samples, + scheduled: result.behavior.scheduled, + published: result.behavior.published + } + ]) + ), + behaviorEqual: true + }) + ) +})().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/docs/audits/empty-streamed-delta-retention/scenario.cjs b/docs/audits/empty-streamed-delta-retention/scenario.cjs new file mode 100644 index 00000000000..943d8130062 --- /dev/null +++ b/docs/audits/empty-streamed-delta-retention/scenario.cjs @@ -0,0 +1,156 @@ +const assert = require('node:assert/strict') +const { load } = require('./sources.cjs') + +async function scenario(phase) { + const readers = [] + globalThis.__orcaEmptyDeltaReaders = readers + const { + createCodexStructuredItemStreams, + createAgentSessionDeltaCoalescer, + sourceSha256, + bundleSha256 + } = await load(phase) + const fixed = phase === 'fixed' || phase === 'reportedFixed' + let scheduled = 0 + let published = 0 + const publications = [] + const streams = createCodexStructuredItemStreams({ + sink: { + appendItem(identity, body) { + published += 1 + publications.push({ identity, body }) + }, + publish() {} + }, + identityFor: () => ({ provider: 'codex', threadId: 'thread-a', turnId: 'turn-a', ordinal: 0 }), + schedule: () => { + scheduled += 1 + return () => {} + } + }) + assert.equal(readers.length, 1) + const read = readers[0] + const samples = [] + for (let batch = 0; batch < 4; batch += 1) { + for (let index = 0; index < 16384; index += 1) { + assert.deepEqual( + streams.handle('thread-a', 'item/agentMessage/delta', { itemId: 'item-a', delta: '' }), + { handled: true, admission: { accepted: true } } + ) + } + assert.equal(streams.flush(), true) + samples.push(read()) + assert.deepEqual(streams.snapshot('thread-a', 'item-a'), { + text: '', + observedBytes: 0, + truncated: false + }) + } + assert.equal(read().slots, fixed ? 0 : 65536) + assert.equal(read().retainedBytes, 0) + assert.equal(read().observedBytes, 0) + streams.handle('thread-a', 'item/agentMessage/delta', { itemId: 'item-a', delta: 'hé' }) + assert.equal(streams.flush(), true) + assert.deepEqual(streams.snapshot('thread-a', 'item-a'), { + text: 'hé', + observedBytes: 3, + truncated: false + }) + assert.equal(read().slots, fixed ? 1 : 65537) + streams.forget('thread-a', 'item-a') + assert.deepEqual(read(), { streams: 0, slots: 0, retainedBytes: 0, observedBytes: 0 }) + streams.handle('thread-a', 'item/agentMessage/delta', { + itemId: 'item-a', + delta: 'retained until dispose' + }) + streams.dispose() + assert.deepEqual(read(), { streams: 0, slots: 0, retainedBytes: 0, observedBytes: 0 }) + + let accepting = false + const pending = new Set() + const emitted = [] + let directScheduled = 0 + const deps = { + emit(key, text, snapshot) { + assert.equal(this, deps) + if (!accepting) { + return false + } + emitted.push({ key, text, snapshot }) + return true + }, + schedule(run) { + directScheduled += 1 + pending.add(run) + return () => pending.delete(run) + }, + maxStreams: 1, + maxRetainedBytes: 64, + maxTotalRetainedBytes: 64 + } + const direct = createAgentSessionDeltaCoalescer(deps) + assert.equal(direct.append('one', ''), true) + assert.deepEqual(direct.snapshot('one'), { text: '', observedBytes: 0, truncated: false }) + assert.equal(pending.size, 1) + assert.equal(direct.flushAll(), false) + assert.equal(pending.size, 1) + assert.equal(direct.append('two', ''), false) + assert.equal(direct.snapshot('two'), null) + accepting = true + assert.equal(direct.flushAll(), true) + assert.equal(pending.size, 0) + assert.equal(direct.append('one', ''), true) + assert.equal(direct.flushAll(), true) + assert.equal(emitted.length, 2) + assert.equal(direct.append('one', 'unchanged'), true) + assert.equal(direct.flushAll(), true) + accepting = false + direct.append('one', '') + assert.equal(direct.append('two', ''), false) + assert.deepEqual(direct.snapshot('one'), { + text: 'unchanged', + observedBytes: 9, + truncated: false + }) + accepting = true + assert.equal(direct.append('two', ''), true) + assert.equal(direct.snapshot('one'), null) + assert.equal(direct.flushAll(), true) + direct.append('two', '😀'.repeat(100)) + assert.equal(direct.flushAll(), true) + const truncated = direct.snapshot('two') + assert.ok(Buffer.byteLength(truncated.text, 'utf8') <= 64) + assert.equal(truncated.truncated, true) + assert.equal(truncated.observedBytes, 400) + const publicationsBeforeEmpty = emitted.length + direct.append('two', '') + assert.equal(direct.flushAll(), true) + assert.equal(emitted.length, publicationsBeforeEmpty) + assert.deepEqual(direct.snapshot('two'), truncated) + direct.dispose() + assert.equal(pending.size, 0) + assert.deepEqual(readers[1](), { streams: 0, slots: 0, retainedBytes: 0, observedBytes: 0 }) + delete globalThis.__orcaEmptyDeltaReaders + return { + sourceSha256, + bundleSha256, + samples, + behavior: { scheduled, published, publications, directScheduled, emitted, truncated }, + controls: [ + 'actual Codex empty notification stream', + 'empty snapshot remains present', + 'four explicit flushes', + 'Unicode text retained', + 'forget clears', + 'dispose clears', + 'first empty publication and retries', + 'emit receiver preserved', + 'new empty key rejected under backpressure', + 'accepted eviction', + 'UTF-8 truncation', + 'already-truncated empty append keeps no-new-publication behavior' + ] + } +} + +module.exports = { scenario } diff --git a/docs/audits/empty-streamed-delta-retention/source-versions.json b/docs/audits/empty-streamed-delta-retention/source-versions.json new file mode 100644 index 00000000000..29f6bb0c1f2 --- /dev/null +++ b/docs/audits/empty-streamed-delta-retention/source-versions.json @@ -0,0 +1,77 @@ +{ + "sourcePath": "src/main/native-chat/agent-session-wire/agent-session-delta-coalescer.ts", + "canonicalization": "CRLF to LF", + "baselineSha256": "48828e4ee21858075cbb87ec1caa4a82991a55f80928482615d1df7ec5dc0fb2", + "fixedSha256": "7caf0f24250b42e0ac4fe99e93bcbe0a4acb7cb3465fdde6b248d8bf6ba0d2d0", + "reportedSha256": "bfff14bd820a2d7be90e82db6403d2415c0fb3de998cf61d7c3830ff4c415605", + "reportedFixedSha256": "2f00aec9b7a4fb24cad4e01d1bf7e1d0b881076f9760b322cdf72c39594ae89c", + "namedReferences": { + "main": "291b4ddd6f1c1af480169885e0fda7f9c78ff053", + "reported": "e0826956fcfc532f5a1e55b5e081f2e57e553c43", + "reportedTag": "v1.4.198" + }, + "commonDependencies": { + "src/shared/agent-session-journal-item-key.ts": "09ebe4758e3f38b5b7591f07f7b4dbfd04c7cc01713c9b0afe03b07b656d1fe8", + "src/main/codex/codex-command-lifecycle.ts": "7b480d3b111304e3caff71768fb3b11ab6c36e1fa8f5b0440623a77cac2382c2", + "src/shared/native-chat-turn-status.ts": "4ce2b64fa7818106814c135e5daf77bb183f601c35f9f08e5b7ff847fba01378", + "src/shared/native-chat-tool-identity.ts": "8c98eec1a53b26a67ed48a0859c5863f5accbe67e013c4eb5a0f8b32441ee925", + "src/main/codex/codex-structured-item-stream-bounds.ts": "88e9efa8a9c163905f1a5eb5eaf316fba2f194d76c5a94ed7f55636cc0d74ae0", + "src/main/codex/codex-item-stream-retention.ts": "66d1f1d51ed6711e20f81698b14be2d003e899dbc587ac1306b6a656cf951b96", + "src/main/native-chat/agent-session-journal/journal-payload-bounds.ts": "a421da6c6eee9346746a69f8ba7ff966ce81969dfa0a93f58784cac27cf8cdad", + "src/main/codex/codex-goal-journal-rows.ts": "415846ca105300349778da7637216ca6ce978c7036219f6b3038f7228ba8724e", + "src/main/codex/codex-subagent-activity.ts": "5cdf18a4ea2c68178a67a845d4c95672a0d94bb282e8e15c3a930c6dd59151e4", + "src/main/native-chat/agent-session-wire/provider-frame-disposition.ts": "f24008cebb1b8a77b0cbd091e49140a4bf99c676bc61eedeb0a82f1263244a7c", + "src/main/native-chat/agent-session-wire/unhandled-provider-frame.ts": "f8169a024a573adfc97e4eb8c99ecc4e80f82e9f94a04a29011a6b8051cc7626", + "src/shared/raster-image-dimensions.ts": "0d7462b8b2e53bfcdcbd32803edcfb85ed0063f56d45cf230fb59aac4783d063", + "src/shared/raster-image-preview-limits.ts": "f2d836b354b3951912f1ec689d84b82aa1aade89ed8e9c6f76be2321d600e9a7", + "src/shared/raster-image-base64-preview.ts": "89baf40f4637615456888e0f35115517a8445a3b1cb640b7bc495dd9a98fcedb", + "src/shared/image-data-uri.ts": "3eb9d2bda499b8f10c74783f9de766041469b140a9b96b2b5be7d97528688ce0", + "src/main/codex/codex-item-field-readers.ts": "aa4b178c562a63995b7de63c6f5daaf7b6b610fb163fb0a357cc230314076323", + "src/main/codex/codex-image-item-translation.ts": "c4f88236c707472cbcfcdfb1a04e51c738f8741aa5847a33f730759499522f5e", + "src/main/codex/codex-command-action-class.ts": "93b7bceb66a9e5bfae680e018b61bddb5c1117fe26243088ceccc8738841f966", + "src/main/codex/codex-thread-item-identity.ts": "04b98b367a4d658fb759ebc4fc73ea4ae731ae3f238f191c8ce002b2a2c9e2a8", + "src/main/codex/codex-turn-ordinals.ts": "9b7cf66986235bcacd6c198950452c7f40bd559a3a0ce20995aac496a1e32d7f", + "src/main/codex/codex-structured-item-translation.ts": "8ab9debb279baf3a5043ffe754c8911755c6e4ca9edfc0a094671c4d5d166639", + "src/main/codex/codex-structured-item-stream-events.ts": "da2a9da9025af354ce705a190badadf822f9ac3b403eef04aa0ef8aaf6f023de", + "src/main/codex/codex-structured-item-streams.ts": "b70c3306e8bec3587342afc3bc156f40903513fa7ff3865b217ba4cc912cd89f" + }, + "callerSourceHashes": [ + { + "path": "src/main/codex/codex-structured-item-streams.ts", + "working": "b70c3306e8bec3587342afc3bc156f40903513fa7ff3865b217ba4cc912cd89f", + "main": "b70c3306e8bec3587342afc3bc156f40903513fa7ff3865b217ba4cc912cd89f", + "reported": "0f05fd8232d5f9b7a928abdd97ea04846ffade9512d61371f120dbcff00c09b6" + }, + { + "path": "src/main/codex/codex-structured-journal-translation.ts", + "working": "56e17b9649471e804d29d1d54a3e14b4298ca8af2e25c4759dc37fce000b06a1", + "main": "56e17b9649471e804d29d1d54a3e14b4298ca8af2e25c4759dc37fce000b06a1", + "reported": "7a6409082b977e481b137b19f446ee3e17d530f4f72c4d141263a49d3ca7722c" + }, + { + "path": "src/main/codex/codex-structured-provider-events.ts", + "working": "a411dc378e521ea8fc915e2a2a21476341a30f440e8233ae5649205d2a1e2e44", + "main": "a411dc378e521ea8fc915e2a2a21476341a30f440e8233ae5649205d2a1e2e44", + "reported": "69af127e2d2ee6b6d648028f16f3e6d25ea45ed61919d14642a7a554e3ad05a5" + }, + { + "path": "src/main/codex/codex-app-server-notification-schema.ts", + "working": "d551b5fb47aad42fb09d9cee2c68e1e4438d209e13869875fed68c3679ef66c8", + "main": "d551b5fb47aad42fb09d9cee2c68e1e4438d209e13869875fed68c3679ef66c8", + "reported": "d551b5fb47aad42fb09d9cee2c68e1e4438d209e13869875fed68c3679ef66c8" + }, + { + "path": "src/main/claude/claude-streamed-text-checkpoints.ts", + "working": "a0e435f5fcf73f37e48e5363f8f9245d992a0d6c0a80abdf5cc5c72b2595fe6d", + "main": "a0e435f5fcf73f37e48e5363f8f9245d992a0d6c0a80abdf5cc5c72b2595fe6d", + "reported": "a0e435f5fcf73f37e48e5363f8f9245d992a0d6c0a80abdf5cc5c72b2595fe6d" + }, + { + "path": "src/main/claude/claude-streamed-block-identity.ts", + "working": "8bc2d4a18fdecd4e27fdf2e8ea3274fcf2e3353edb75607bd0d1133546bfbdc0", + "main": "8bc2d4a18fdecd4e27fdf2e8ea3274fcf2e3353edb75607bd0d1133546bfbdc0", + "reported": "8bc2d4a18fdecd4e27fdf2e8ea3274fcf2e3353edb75607bd0d1133546bfbdc0" + } + ], + "scope": "Exact current/main coalescer before/after plus exact v1.4.198 coalescer and same local guard, using current Codex stream callers/dependencies. Not a packaged historical release replay." +} diff --git a/docs/audits/empty-streamed-delta-retention/sources.cjs b/docs/audits/empty-streamed-delta-retention/sources.cjs new file mode 100644 index 00000000000..36ec31bd52f --- /dev/null +++ b/docs/audits/empty-streamed-delta-retention/sources.cjs @@ -0,0 +1,111 @@ +const assert = require('node:assert/strict') +const { createHash } = require('node:crypto') +const { readFileSync } = require('node:fs') +const path = require('node:path') +const Module = require('node:module') +const esbuild = require('esbuild') +const { applyPatch, parsePatch, reversePatch } = require('diff') + +const root = path.resolve(__dirname, '../../..') +const canonical = (value) => value.replaceAll('\r\n', '\n') +const sha = (value) => createHash('sha256').update(value).digest('hex') +const readText = (file) => canonical(readFileSync(file, 'utf8')) +const versions = JSON.parse(readText(path.join(__dirname, 'source-versions.json'))) + +function loadSources(read = readText) { + const fixed = canonical(read(path.join(root, versions.sourcePath))) + assert.equal(sha(fixed), versions.fixedSha256, 'Fixed source drift') + const reverse = (name) => { + const patches = parsePatch(canonical(read(path.join(__dirname, name)))) + assert.equal(patches.length, 1) + assert.equal(patches[0].newFileName, `b/${versions.sourcePath}`) + const source = applyPatch(fixed, reversePatch(patches[0])) + assert.notEqual(source, false) + return source + } + const baseline = reverse('fix.patch') + const reported = reverse('reported.patch') + const marker = ' current.push(delta)' + assert.equal(reported.split(marker).length, 2) + const reportedFixed = reported.replace( + marker, + ' if (delta.length > 0) {\n current.push(delta)\n }' + ) + assert.equal(sha(baseline), versions.baselineSha256, 'Baseline source drift') + assert.equal(sha(reported), versions.reportedSha256, 'Reported source drift') + assert.equal(sha(reportedFixed), versions.reportedFixedSha256) + return { baseline, fixed, reported, reportedFixed } +} + +async function load(phase) { + const sources = loadSources() + assert.ok(Object.hasOwn(sources, phase)) + for (const [file, expected] of Object.entries(versions.commonDependencies)) { + assert.equal(sha(readText(path.join(root, file))), expected, `Dependency drift: ${file}`) + } + for (const caller of versions.callerSourceHashes) { + assert.equal( + sha(readText(path.join(root, caller.path))), + caller.working, + `Caller drift: ${caller.path}` + ) + } + const marker = ' const flushKey = (key: string): boolean => {' + const source = sources[phase] + assert.equal(source.split(marker).length, 2) + // Measurement only reads cardinalities; it never changes stream ownership or contents. + const measured = source.replace( + marker, + ` globalThis.__orcaEmptyDeltaReaders.push(() => ({ + streams: streams.size, + slots: [...streams.values()].reduce((count, stream) => count + stream.chunks.length, 0), + retainedBytes: totalRetainedBytes, + observedBytes: [...streams.values()].reduce((count, stream) => count + stream.observedBytes, 0) + }))\n${marker}` + ) + const build = await esbuild.build({ + stdin: { + contents: + "export { createCodexStructuredItemStreams } from './src/main/codex/codex-structured-item-streams'; export { createAgentSessionDeltaCoalescer } from './src/main/native-chat/agent-session-wire/agent-session-delta-coalescer'", + resolveDir: root, + loader: 'ts' + }, + absWorkingDir: root, + bundle: true, + platform: 'node', + format: 'cjs', + packages: 'external', + write: false, + metafile: true, + plugins: [ + { + name: 'read-private-array-cardinality', + setup(builder) { + builder.onLoad({ filter: /agent-session-delta-coalescer\.ts$/ }, (args) => { + assert.equal(args.path, path.join(root, versions.sourcePath)) + return { contents: measured, loader: 'ts' } + }) + } + } + ] + }) + const actualInputs = Object.keys(build.metafile.inputs) + .filter((file) => file.startsWith('src/')) + .sort() + assert.deepEqual( + actualInputs, + [...Object.keys(versions.commonDependencies), versions.sourcePath].sort() + ) + const filename = path.join(__dirname, `in-memory-${phase}.cjs`) + const loaded = new Module(filename, module) + loaded.filename = filename + loaded.paths = Module._nodeModulePaths(root) + loaded._compile(build.outputFiles[0].text, filename) + return { + ...loaded.exports, + sourceSha256: sha(source), + bundleSha256: sha(build.outputFiles[0].contents) + } +} + +module.exports = { load, loadSources, root, sha, versions } diff --git a/docs/audits/empty-streamed-delta-retention/validation.json b/docs/audits/empty-streamed-delta-retention/validation.json new file mode 100644 index 00000000000..b53bf94f6ef --- /dev/null +++ b/docs/audits/empty-streamed-delta-retention/validation.json @@ -0,0 +1,50 @@ +{ + "reviewedHead": "a8c4bed3fa4191d731bb28826d00318f18a5db0a", + "backgroundLaunch": "ORCA_BACKGROUND_LAUNCH=1 on every check", + "productHashes": { + "src/main/native-chat/agent-session-wire/agent-session-delta-coalescer.ts": "7caf0f24250b42e0ac4fe99e93bcbe0a4acb7cb3465fdde6b248d8bf6ba0d2d0", + "src/main/native-chat/agent-session-wire/agent-session-empty-delta-retention.test.ts": "2e4ed51b8c205e650cd0d3d4fcb968fdf8f509be4c4013ca1dc6e70b59703a55" + }, + "focusedTests": { + "files": 6, + "passed": 64, + "failed": 0, + "paths": [ + "src/main/native-chat/agent-session-wire/agent-session-empty-delta-retention.test.ts", + "src/main/native-chat/agent-session-wire/agent-session-delta-coalescer.test.ts", + "src/main/codex/codex-persistent-command-retention.test.ts", + "src/main/codex/codex-structured-journal-translation.test.ts", + "src/main/codex/codex-structured-journal-translation-streams.test.ts", + "src/main/claude/claude-streamed-text-checkpoints.test.ts" + ] + }, + "baselineTests": { + "config": "docs/audits/empty-streamed-delta-retention/before.config.mjs", + "passed": 14, + "expectedFailures": 1, + "failureName": "empty streamed deltas does not retain empty array slots through repeated Codex publications", + "assertion": "expected 65537 to be 1", + "scope": "Only the new retained-slot regression fails; all publication/backpressure controls pass." + }, + "checks": { + "nodeTypecheck": "passed: pnpm tc:node", + "ordinaryLint": "passed: oxlint on two product files", + "typeAwareLint": "passed: oxlint --type-aware --config config/oxlint-code-quality-type-aware.json --deny-warnings on two product files", + "changedQuality": "passed: pnpm run check:code-quality:changed HEAD, two files, zero new findings", + "format": "passed: oxfmt product and artifact files", + "diffWhitespace": "git diff --check on exact product paths; zero-context historical/fix patches" + }, + "proofReports": { + "node-results.json": "a858b703a1c94c2bc4bc817f244a76fb84bd4ea4cb2dbbb6688acef91d25905d", + "electron-results.json": "1ba7c0dd598244cc8c5af9643823470198bf4409deae60939f13dc5dc4b716d5" + }, + "independentReview": "rpc_queue_retention reviewed actual source, tests, named hashes and behavior parity; separately reran all 15 coalescer/new tests.", + "ciArtifactCorrection": { + "pullRequest": 21142, + "failedHead": "9ed7c4f5c880ab64dda8a1a8a04ca8455af42b5d", + "failedJob": "https://github.com/stablyai/orca/actions/runs/35178713175/job/105066129816", + "cause": "One-line if in scenario.cjs lacked braces. Product-only local quality omitted durable proof sources; CI correctly rejected it.", + "change": "Add braces; product code unchanged. Rerun both actual-source runtime reports and all five quality scan configurations over all six published code files with --no-ignore.", + "result": "Both runtime proofs and all five quality scans pass. CI status remains separately recorded at the observed head." + } +} diff --git a/src/main/native-chat/agent-session-wire/agent-session-delta-coalescer.ts b/src/main/native-chat/agent-session-wire/agent-session-delta-coalescer.ts index a3b32ca24f0..1d5608150ef 100644 --- a/src/main/native-chat/agent-session-wire/agent-session-delta-coalescer.ts +++ b/src/main/native-chat/agent-session-wire/agent-session-delta-coalescer.ts @@ -230,7 +230,9 @@ function appendWithinUtf8ByteLimit( if (deltaBytes <= available) { // The caller owns the per-stream array; append in place so each token is // amortized O(1) instead of copying the complete prefix on every delta. - current.push(delta) + if (delta.length > 0) { + current.push(delta) + } return { chunks: current, retainedBytes: currentBytes + deltaBytes, diff --git a/src/main/native-chat/agent-session-wire/agent-session-empty-delta-retention.test.ts b/src/main/native-chat/agent-session-wire/agent-session-empty-delta-retention.test.ts new file mode 100644 index 00000000000..4e2161b19a8 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/agent-session-empty-delta-retention.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, it, vi } from 'vitest' +import { createCodexStructuredItemStreams } from '../../codex/codex-structured-item-streams' +import { createAgentSessionDeltaCoalescer } from './agent-session-delta-coalescer' + +describe('empty streamed deltas', () => { + it('does not retain empty array slots through repeated Codex publications', () => { + const prefix = 'empty-delta-retention-prefix' + const streams = createCodexStructuredItemStreams({ + sink: { appendItem() {}, appendTombstone() {}, publish() {} }, + identityFor: () => ({ provider: 'codex', threadId: 'thread', turnId: 'turn', ordinal: 0 }), + schedule: () => () => {} + }) + const append = (delta: string) => + streams.handle('thread', 'item/agentMessage/delta', { itemId: 'item', delta }) + append(prefix) + try { + for (let batch = 0; batch < 4; batch += 1) { + for (let index = 0; index < 16_384; index += 1) { + append('') + } + expect(streams.flush()).toBe(true) + } + const originalJoin = Array.prototype.join + let retainedSlots = -1 + const spy = vi + .spyOn(Array.prototype, 'join') + .mockImplementation(function (this: unknown[], separator) { + // Byte counters cannot detect empty entries retained by the stream's chunk array. + if (separator === '' && this[0] === prefix) { + retainedSlots = this.length + } + return originalJoin.call(this, separator) + }) + let snapshot: ReturnType + try { + snapshot = streams.snapshot('thread', 'item') + } finally { + spy.mockRestore() + } + expect(retainedSlots).toBe(1) + expect(snapshot).toEqual({ + text: prefix, + observedBytes: Buffer.byteLength(prefix), + truncated: false + }) + append('é') + expect(streams.snapshot('thread', 'item')?.text).toBe(`${prefix}é`) + streams.forget('thread', 'item') + expect(streams.snapshot('thread', 'item')).toBeNull() + } finally { + streams.dispose() + } + }) + + it('preserves empty stream snapshots, scheduled publication and explicit flushes', () => { + const pending = new Set<() => void>() + const emitted: { key: string; text: string }[] = [] + const instance = createAgentSessionDeltaCoalescer({ + schedule: (run) => { + pending.add(run) + return () => { + pending.delete(run) + } + }, + emit: (key, text) => emitted.push({ key, text }) + }) + try { + expect(instance.append('empty', '')).toBe(true) + expect(instance.snapshot('empty')).toEqual({ + text: '', + observedBytes: 0, + truncated: false + }) + expect(pending.size).toBe(1) + expect(emitted).toEqual([]) + expect(instance.flushAll()).toBe(true) + expect(pending.size).toBe(0) + expect(emitted).toEqual([{ key: 'empty', text: '' }]) + instance.append('empty', 'visible') + instance.append('empty', '') + expect(pending.size).toBe(1) + expect(instance.flush('empty')).toBe(true) + expect(emitted.at(-1)).toEqual({ key: 'empty', text: 'visible' }) + instance.append('empty', '') + expect(instance.flushAll()).toBe(true) + expect(emitted).toHaveLength(3) + expect(emitted.at(-1)).toEqual({ key: 'empty', text: 'visible' }) + } finally { + instance.dispose() + } + expect(pending.size).toBe(0) + }) + + it('still refuses a new empty stream while the oldest output is backpressured', () => { + let accepting = false + const emitted: [string, string][] = [] + const instance = createAgentSessionDeltaCoalescer({ + maxStreams: 1, + schedule: () => () => {}, + emit: (key, text) => { + if (!accepting) { + return false + } + emitted.push([key, text]) + return true + } + }) + try { + instance.append('first', 'preserved') + expect(instance.append('second', '')).toBe(false) + expect(instance.snapshot('second')).toBeNull() + expect(instance.snapshot('first')?.text).toBe('preserved') + accepting = true + expect(instance.append('second', '')).toBe(true) + expect(instance.snapshot('first')).toBeNull() + expect(instance.snapshot('second')?.text).toBe('') + expect(instance.flushAll()).toBe(true) + expect(emitted).toEqual([ + ['first', 'preserved'], + ['second', ''] + ]) + } finally { + instance.dispose() + } + }) +})