Merge pull request #752 from nwparker/cloud/share-ws-keepalive

fix(sandbox-control): share-link WebSockets get a keepalive ping under Vercel's ~340 s idle cut
This commit is contained in:
Neil
2026-11-14 21:08:14 -08:00
committed by GitHub
3 changed files with 158 additions and 6 deletions
File diff suppressed because one or more lines are too long
@@ -6,6 +6,11 @@
// `until` (the link's expiry, at most 30 minutes out), so a revoked or expired link also loses open
// streams. Signed requests are reverse-proxied to 127.0.0.1:<port>: streaming both ways, no
// buffering, WebSocket upgrades piped raw. Anything unsigned gets a bare 404.
// Vercel closes a WebSocket on a *.vercel.run route after ~340 s with no traffic (1006). A preview
// app's socket can sit silent that long, so once an upgrade is a WebSocket the proxy watches the app's
// frames and, after KEEPALIVE_MS with nothing either way, sends the visitor a ping control frame
// between two frames (never inside one). The visitor's pong reaches the app, which RFC 6455 says to
// ignore (an unsolicited pong needs no answer).
// The key arrives on stdin (orca-sandbox-start pipes it), so it is never in this process's
// environment or argv; ORCA_SHARE_PROXY_KEY in the env is accepted for tests. Optional
// ORCA_SHARE_PROXY_PORT (default 6769). Plain Node, no dependencies.
@@ -24,6 +29,10 @@ delete process.env.ORCA_SHARE_PROXY_KEY
const BLOCKED_PORTS = new Set([2222, 6080, 6768, LISTEN_PORT])
const MAX_SKEW_SECONDS = 300
const MAX_CONNECTION_SECONDS = 3600
// Well under Vercel's ~340 s idle cut. ORCA_SHARE_PROXY_KEEPALIVE_MS is for tests.
const KEEPALIVE_MS = Number(process.env.ORCA_SHARE_PROXY_KEEPALIVE_MS || 30_000)
// FIN + ping opcode, empty payload, unmasked (server to client).
const PING_FRAME = Buffer.from([0x89, 0x00])
const HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-connection', 'transfer-encoding', 'te', 'trailer', 'upgrade']
if (KEY.length < 32) {
@@ -74,6 +83,70 @@ function refuse(res) {
res.end('not found')
}
/**
* Follows the app's WebSocket frames (server to client: header, then payload) just closely enough to
* know when the stream sits between two frames, where a control frame may be inserted.
*/
function frameBoundaries() {
let header = []
let remaining = 0
return {
push(chunk) {
let i = 0
while (i < chunk.length) {
if (remaining > 0) {
const take = Math.min(remaining, chunk.length - i)
remaining -= take
i += take
continue
}
header.push(chunk[i++])
if (header.length < 2) {
continue
}
const len7 = header[1] & 0x7f
const need = 2 + (len7 === 126 ? 2 : len7 === 127 ? 8 : 0) + (header[1] & 0x80 ? 4 : 0)
if (header.length < need) {
continue
}
const bytes = Buffer.from(header)
remaining = len7 === 126 ? bytes.readUInt16BE(2) : len7 === 127 ? Number(bytes.readBigUInt64BE(2)) : len7
header = []
}
},
atBoundary: () => header.length === 0 && remaining === 0
}
}
/** The app's answer to the upgrade: its head is passed through; only a 101 to `websocket` gets keepalives. */
function upgradeAnswer() {
let head = Buffer.alloc(0)
let state = 'head'
const frames = frameBoundaries()
return {
push(chunk) {
if (state === 'head') {
head = Buffer.concat([head, chunk])
const end = head.indexOf('\r\n\r\n')
if (end === -1) {
if (head.length > 64 * 1024) {
state = 'raw'
}
return
}
const text = head.subarray(0, end).toString('latin1')
state = /^HTTP\/1\.1 101/.test(text) && /^upgrade:\s*websocket\s*$/im.test(text) ? 'ws' : 'raw'
chunk = head.subarray(end + 4)
head = Buffer.alloc(0)
}
if (state === 'ws') {
frames.push(chunk)
}
},
canPing: () => state === 'ws' && frames.atBoundary()
}
}
/** Ends `close` at the grant's deadline; the timer goes away with the connection. */
function deadline(untilMs, close, ...emitters) {
const timer = setTimeout(close, Math.max(0, untilMs - Date.now()))
@@ -139,7 +212,28 @@ server.on('upgrade', (req, socket, head) => {
target.write(head)
}
socket.pipe(target)
target.pipe(socket)
// App to visitor by hand (with backpressure) so the keepalive knows where frames start and end.
const answer = upgradeAnswer()
let lastTraffic = Date.now()
socket.on('data', () => {
lastTraffic = Date.now()
})
target.on('data', (chunk) => {
lastTraffic = Date.now()
answer.push(chunk)
if (!socket.write(chunk)) {
target.pause()
socket.once('drain', () => target.resume())
}
})
target.on('end', () => socket.end())
const keepalive = setInterval(() => {
if (Date.now() - lastTraffic >= KEEPALIVE_MS && answer.canPing() && socket.writable) {
socket.write(PING_FRAME)
lastTraffic = Date.now()
}
}, Math.max(50, Math.floor(KEEPALIVE_MS / 3)))
socket.once('close', () => clearInterval(keepalive))
})
const close = () => {
socket.destroy()
@@ -58,10 +58,10 @@ async function freePort(): Promise<number> {
}
/** Starts the real share proxy for a session key; returns its local port. */
async function startProxy(key: string): Promise<number> {
async function startProxy(key: string, extraEnv: Record<string, string> = {}): Promise<number> {
const port = await freePort()
// The key goes in on stdin, as orca-sandbox-start pipes it; never through the environment.
const child = spawn(process.execPath, [PROXY], { env: { ORCA_SHARE_PROXY_PORT: String(port) }, stdio: ['pipe', 'pipe', 'inherit'] })
const child = spawn(process.execPath, [PROXY], { env: { ORCA_SHARE_PROXY_PORT: String(port), ...extraEnv }, stdio: ['pipe', 'pipe', 'inherit'] })
child.stdin!.end(key)
proxies.push(child)
await once(child.stdout!, 'data')
@@ -212,6 +212,64 @@ describe('share links', () => {
await once(socket, 'close')
})
it('pings an idle WebSocket well before Vercel’s ~340 s idle cut, only between frames and only for websocket upgrades', async () => {
const key = await shareProxyKey('x'.repeat(40), 'sbx', 'sess')
const port = await startProxy(key, { ORCA_SHARE_PROXY_KEEPALIVE_MS: '150' })
// An app that upgrades, then stays silent unless told to send (half) a frame.
const silent = net.createServer((conn) => {
conn.once('data', (request) => {
const protocol = /upgrade: (\S+)/i.exec(String(request))?.[1] ?? 'websocket'
conn.write(`HTTP/1.1 101 Switching Protocols\r\nupgrade: ${protocol}\r\nconnection: upgrade\r\n\r\n`)
conn.on('data', (cmd) => {
// A 10-byte text frame: header + 5 bytes now, the rest on "finish".
if (String(cmd).includes('half')) conn.write(Buffer.concat([Buffer.from([0x81, 10]), Buffer.from('hello')]))
if (String(cmd).includes('finish')) conn.write(Buffer.from('world'))
})
})
})
silent.listen(0, '127.0.0.1')
await once(silent, 'listening')
const silentPort = (silent.address() as net.AddressInfo).port
const open = async (protocol: string) => {
const header = await signProxyHeader(key, { port: silentPort, linkId: 'b'.repeat(32), linkExpiresAt: Date.now() + 60_000 }, Date.now())
const socket = net.connect(port, '127.0.0.1')
await once(socket, 'connect')
const received: Buffer[] = []
socket.on('data', (chunk) => received.push(chunk))
socket.write(`GET /ws HTTP/1.1\r\nhost: x\r\nupgrade: ${protocol}\r\nconnection: upgrade\r\nx-orca-share: ${header}\r\n\r\n`)
const after = () => Buffer.concat(received).subarray(Buffer.concat(received).indexOf('\r\n\r\n') + 4)
return { socket, after }
}
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
try {
const ws = await open('websocket')
await sleep(600)
// Idle: at least one ping frame (0x89 0x00), nothing else.
const pings = ws.after()
expect(pings.length).toBeGreaterThanOrEqual(2)
expect([...pings].every((byte, i) => byte === (i % 2 === 0 ? 0x89 : 0x00))).toBe(true)
// Mid-frame: no ping may land inside the app's frame.
ws.socket.write('half')
await sleep(80)
const before = ws.after().length
await sleep(600)
expect(ws.after().subarray(before).length).toBe(0)
ws.socket.write('finish')
await sleep(600)
const tail = ws.after().subarray(before)
expect(tail.subarray(0, 5).toString()).toBe('world')
expect([...tail.subarray(5, 7)]).toEqual([0x89, 0x00])
ws.socket.destroy()
// Another upgrade protocol is piped untouched: no frames are invented for it.
const other = await open('h2c')
await sleep(600)
expect(other.after().length).toBe(0)
other.socket.destroy()
} finally {
silent.close()
}
})
it('shows a branded page when nothing listens on the port', async () => {
const closed = await freePort()
const { visit, share } = await setup({ port: closed })