mirror of
https://github.com/stablyai/orca.git
synced 2026-09-21 16:02:20 +00:00
fix(browser): bound CDP output for stalled clients (#20949)
* fix(browser): bound CDP output for stalled clients * fix(browser): log CDP outbound overflow before terminating the client The outbound queue terminated the automation client silently on overflow, so the client saw a socket close indistinguishable from a crash. Surface the cap that tripped and the backlog held when it did. The queue dropped its backlog before invoking onOverflow, so the counters were already zero at the callback. Snapshot them first and pass them through. --------- Co-authored-by: m4air <m4air@Mac.localdomain> Co-authored-by: Neil <neil@stably.ai>
This commit is contained in:
@@ -0,0 +1,15 @@
|
||||
# CDP outbound retention reproduction
|
||||
|
||||
From the worktree root:
|
||||
|
||||
```sh
|
||||
ORCA_BACKGROUND_LAUNCH=1 node docs/audits/cdp-stream-retention/reproduce.mjs
|
||||
```
|
||||
|
||||
This bundles the current response writer and connects real loopback WebSockets. The reader pauses while the producer sends at most 128 MiB. The script starts no Electron application or PTY and closes all sockets. `before.json` records the same experiment against the unbounded writer; `after.json` records the fixed writer.
|
||||
|
||||
Before the fix, 128 MiB of generated payload left 133,177,280 bytes in the WebSocket buffer, with roughly linear growth at every sample. The fixed writer uses the existing outbound queue: an 8 MiB socket soft threshold, 64 MiB held-queue cap and 4,096 queued-frame cap. The stalled connection terminates on overflow and releases its queue. The socket threshold can overshoot by one frame; a single large reply on a clear connection remains permitted, preserving large PDF/screenshot responses. This bounds accumulated backlog, not the allocation needed to construct an individual response or the aggregate across arbitrarily many browser pages.
|
||||
|
||||
Unit tests also cover a reader that drains a complete burst in order, request/session correlation, queue disposal on close/replacement, and a 65 MiB healthy reply. Debugger events and command responses share this writer.
|
||||
|
||||
This establishes an Electron-main retention mechanism when a CDP automation client stops reading. Neither #19831 nor #19768 establishes that precondition, so it is not a confirmed cause of either incident. RSS reflects GC/allocator timing; queued bytes are the direct measurement.
|
||||
@@ -0,0 +1,55 @@
|
||||
{
|
||||
"node": "v26.6.0",
|
||||
"platform": "darwin",
|
||||
"bundleSha256": "60daf233a12cd9427de1783054e397c1968bb1d23fa9aad0408972337b6d087b",
|
||||
"samples": [
|
||||
{
|
||||
"producedMiB": 16,
|
||||
"bufferedBytes": 8389120,
|
||||
"readyState": 1,
|
||||
"rss": 92291072
|
||||
},
|
||||
{
|
||||
"producedMiB": 32,
|
||||
"bufferedBytes": 8389120,
|
||||
"readyState": 1,
|
||||
"rss": 111542272
|
||||
},
|
||||
{
|
||||
"producedMiB": 48,
|
||||
"bufferedBytes": 7340480,
|
||||
"readyState": 1,
|
||||
"rss": 143523840
|
||||
},
|
||||
{
|
||||
"producedMiB": 64,
|
||||
"bufferedBytes": 7340480,
|
||||
"readyState": 1,
|
||||
"rss": 160694272
|
||||
},
|
||||
{
|
||||
"producedMiB": 80,
|
||||
"bufferedBytes": 0,
|
||||
"readyState": 3,
|
||||
"rss": 172490752
|
||||
},
|
||||
{
|
||||
"producedMiB": 96,
|
||||
"bufferedBytes": 0,
|
||||
"readyState": 3,
|
||||
"rss": 172490752
|
||||
},
|
||||
{
|
||||
"producedMiB": 112,
|
||||
"bufferedBytes": 0,
|
||||
"readyState": 3,
|
||||
"rss": 172490752
|
||||
},
|
||||
{
|
||||
"producedMiB": 128,
|
||||
"bufferedBytes": 0,
|
||||
"readyState": 3,
|
||||
"rss": 172490752
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"node": "v26.6.0",
|
||||
"samples": [
|
||||
{
|
||||
"producedMiB": 16,
|
||||
"bufferedBytes": 16778240,
|
||||
"readyState": 1,
|
||||
"rss": 86884352
|
||||
},
|
||||
{
|
||||
"producedMiB": 32,
|
||||
"bufferedBytes": 32507840,
|
||||
"readyState": 1,
|
||||
"rss": 134807552
|
||||
},
|
||||
{
|
||||
"producedMiB": 48,
|
||||
"bufferedBytes": 49286080,
|
||||
"readyState": 1,
|
||||
"rss": 160497664
|
||||
},
|
||||
{
|
||||
"producedMiB": 64,
|
||||
"bufferedBytes": 66064320,
|
||||
"readyState": 1,
|
||||
"rss": 177618944
|
||||
},
|
||||
{
|
||||
"producedMiB": 80,
|
||||
"bufferedBytes": 82842560,
|
||||
"readyState": 1,
|
||||
"rss": 194740224
|
||||
},
|
||||
{
|
||||
"producedMiB": 96,
|
||||
"bufferedBytes": 99620800,
|
||||
"readyState": 1,
|
||||
"rss": 211877888
|
||||
},
|
||||
{
|
||||
"producedMiB": 112,
|
||||
"bufferedBytes": 116399040,
|
||||
"readyState": 1,
|
||||
"rss": 235634688
|
||||
},
|
||||
{
|
||||
"producedMiB": 128,
|
||||
"bufferedBytes": 133177280,
|
||||
"readyState": 1,
|
||||
"rss": 264519680
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import { once } from 'node:events'
|
||||
import { setImmediate as nextTurn } from 'node:timers/promises'
|
||||
import { WebSocket, WebSocketServer } from 'ws'
|
||||
import { build } from 'esbuild'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { resolve } from 'node:path'
|
||||
const root = fileURLToPath(new URL('../../../', import.meta.url))
|
||||
const built = await build({
|
||||
entryPoints: [resolve(root, 'src/main/browser/cdp-client-response-writer.ts')],
|
||||
bundle: true,
|
||||
platform: 'node',
|
||||
format: 'esm',
|
||||
write: false,
|
||||
banner: {
|
||||
js: `import { createRequire } from 'node:module'; const require = createRequire(${JSON.stringify(import.meta.url)});`
|
||||
}
|
||||
})
|
||||
const bundle = built.outputFiles[0].text
|
||||
const { CdpClientResponseWriter } = await import(
|
||||
`data:text/javascript;base64,${Buffer.from(bundle).toString('base64')}`
|
||||
)
|
||||
if (process.env.ORCA_BACKGROUND_LAUNCH !== '1') {
|
||||
throw new Error('Background policy required')
|
||||
}
|
||||
const wss = new WebSocketServer({ host: '127.0.0.1', port: 0 })
|
||||
await once(wss, 'listening')
|
||||
const accepted = once(wss, 'connection')
|
||||
const address = wss.address()
|
||||
if (!address || typeof address === 'string') {
|
||||
throw new Error('Expected TCP address')
|
||||
}
|
||||
const peer = new WebSocket(`ws://127.0.0.1:${address.port}`)
|
||||
await once(peer, 'open')
|
||||
const [socket] = await accepted
|
||||
const writer = new CdpClientResponseWriter(() => socket)
|
||||
peer.pause()
|
||||
const samples = []
|
||||
try {
|
||||
for (let index = 1; index <= 128; index++) {
|
||||
writer.send({ method: 'Network.dataReceived', params: { data: 'x'.repeat(1024 * 1024) } })
|
||||
if (index % 16 === 0) {
|
||||
await nextTurn()
|
||||
samples.push({
|
||||
producedMiB: index,
|
||||
bufferedBytes: socket.bufferedAmount,
|
||||
readyState: socket.readyState,
|
||||
rss: process.memoryUsage().rss
|
||||
})
|
||||
}
|
||||
}
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
node: process.version,
|
||||
platform: process.platform,
|
||||
bundleSha256: createHash('sha256').update(bundle).digest('hex'),
|
||||
samples
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
)
|
||||
} finally {
|
||||
writer.forgetClient(socket)
|
||||
socket.terminate()
|
||||
peer.terminate()
|
||||
await new Promise((done) => wss.close(done))
|
||||
}
|
||||
Reference in New Issue
Block a user