Files
windmill/multiplayer/test/helpers.mjs
T
Alexander PetricandClaude Opus 5.5 e8c3f9514e fix(multiplayer): don't drop client messages during cold-start token verification (#11343)
* fix(multiplayer): don't drop client messages during cold-start token verification

`wss.on('connection')` awaits `verifyToken()` before `setupWSConnection()`
attaches the 'message' listener. On a cold process that await includes the
first `/api/debug/jwks` fetch (~30ms on ECS). A y-websocket client sends sync
step 1 the instant the socket opens, and `ws` drops messages emitted with no
listener attached, so that step 1 was lost and never answered with step 2 —
the client's provider never became `synced`.

Buffer messages from the moment the connection is accepted and replay them, in
order, once `setupWSConnection()` has installed its handlers. Rejected
connections drop the buffer and close with the same 4401/4403 codes as before.

Also prefetch the public key at startup when WINDMILL_BASE_URL is set. That is
insurance, not the fix: a connection arriving before the prefetch resolves
still relies on the buffer.

Adds `npm test` in multiplayer/ (node:test, no docker or backend needed) with a
fake JWKS endpoint that answers with a delay, which holds the cold window open
and makes the race deterministic; wired into the existing test_extra CI job.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

* fix(multiplayer): cap what an unauthenticated peer can buffer pre-auth

Review follow-up.

The pre-auth buffer was unbounded: `ws` sets no `maxPayload` here and the JWKS
fetch has no timeout, so a peer that never authenticates could stream frames
into memory for as long as `verifyToken` was stalled. Cap it at 32 frames /
1 MiB — a real client only has sync step 1 and its first awareness update in
flight there — and close 1009 past that, dropping what was buffered.

A socket closed during verification (by the peer, or by that cap) is no longer
handed to setupWSConnection: it would be added to `doc.conns` with a 'close'
listener that can never fire.

The startup prefetch's .catch was dead code — getPublicKey() logs its own
failures and resolves to null rather than rejecting.

Test helper: pin REQUIRE_SIGNED_MULTIPLAYER_REQUESTS and BASE_INTERNAL_URL so an
ambient value cannot turn the rejection tests into false passes; bind the JWKS
server on port 0 instead of a released probe port, and retry the spawned server
on EADDRINUSE; destroy still-delayed JWKS responses on teardown, since
server.close() waits for in-flight requests.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

* test(multiplayer): gate the JWKS response instead of delaying it

Review follow-up.

The cold window was held open by a 1500 ms delay on the fake JWKS response, but
that timer started when the startup prefetch reached the fake server, not when
the client sent its first frame. A slow enough machine could load the key before
the client connected, and the race test would then pass without ever exercising
the buffer — a false pass.

The fake JWKS server now parks every response until the test calls release(), so
the server provably holds no key while the client is sending. The race test
releases only after both frames are written to the socket, and asserts the
server has not logged the key as loaded at that point; the flood test never
releases until after the cap has closed the connection.

What is left to wall-clock time is 250 ms for bytes already written to the socket
to cross loopback into an otherwise idle server, rather than a window that had to
cover process startup, connect and handshake.

Also drops the prefetch precondition from the forged-token and flood tests so
each test still maps to one behaviour. Suite runs in ~1.1s instead of ~5.3s.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-25 15:54:11 +02:00

193 lines
5.9 KiB
JavaScript

/**
* Shared helpers for the multiplayer server tests: a fake Windmill JWKS
* endpoint, multiplayer token minting and a server.mjs child process.
*/
import { spawn } from 'node:child_process'
import crypto from 'node:crypto'
import http from 'node:http'
import net from 'node:net'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
const SERVER_PATH = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'server.mjs')
export function base64url(buffer) {
return Buffer.from(buffer).toString('base64url')
}
/**
* Ask the kernel for an unused port. There is an unavoidable gap between giving
* the port up and the server.mjs child binding it, so `startMultiplayerServer`
* retries on EADDRINUSE; servers started in-process bind port 0 directly instead.
*/
async function freePort() {
return new Promise((resolve, reject) => {
const probe = net.createServer()
probe.on('error', reject)
probe.listen(0, '127.0.0.1', () => {
const { port } = probe.address()
probe.close(() => resolve(port))
})
})
}
/**
* Mint a multiplayer JWT in exactly the format the backend produces in
* `sign_multiplayer` (backend/windmill-api-debug/src/lib.rs): an EdDSA JWT with
* claims workspace_id, email, iat, exp, purpose.
*/
export function mintToken(privateKey, { workspaceId, email = 'test@windmill.dev', ttlSecs = 300 } = {}) {
const now = Math.floor(Date.now() / 1000)
const header = base64url(JSON.stringify({ alg: 'EdDSA', typ: 'JWT' }))
const claims = base64url(
JSON.stringify({
workspace_id: workspaceId,
email,
iat: now,
exp: now + ttlSecs,
purpose: 'multiplayer'
})
)
const message = `${header}.${claims}`
const signature = base64url(crypto.sign(null, Buffer.from(message), privateKey))
return `${message}.${signature}`
}
/**
* A stand-in for the Windmill backend's /api/debug/jwks, serving the public half
* of an Ed25519 key pair.
*
* With `hold: true` the response is parked indefinitely until `release()` is
* called. That is what makes the cold-start tests deterministic: the server
* cannot obtain the key, so the pre-auth window stays open for exactly as long
* as the test wants, rather than for a wall-clock delay that a slow machine
* could overrun.
*/
export async function startJwksServer({ hold = false } = {}) {
const { publicKey, privateKey } = crypto.generateKeyPairSync('ed25519')
const jwk = publicKey.export({ format: 'jwk' })
const body = JSON.stringify({
keys: [{ kty: jwk.kty, crv: jwk.crv, x: jwk.x, kid: 'test', use: 'sig', alg: 'EdDSA' }]
})
let requests = 0
let released = !hold
const held = new Set()
const respond = (res) => {
res.writeHead(200, { 'Content-Type': 'application/json' })
res.end(body)
}
const server = http.createServer((req, res) => {
if (!req.url?.startsWith('/api/debug/jwks')) {
res.writeHead(404)
res.end('not found')
return
}
requests++
if (released) {
respond(res)
} else {
held.add(res)
}
})
await new Promise((resolve, reject) => {
server.on('error', reject)
server.listen(0, '127.0.0.1', resolve)
})
const { port } = server.address()
return {
privateKey,
baseUrl: `http://127.0.0.1:${port}`,
get requests() {
return requests
},
/** Answer every parked request, and any that arrive later. */
release() {
released = true
for (const res of held) respond(res)
held.clear()
},
async close() {
// Destroy the still-parked responses rather than just dropping them:
// server.close() waits for in-flight requests, so one left hanging would
// deadlock teardown.
for (const res of held) res.destroy()
held.clear()
const closed = new Promise((resolve) => server.close(resolve))
server.closeAllConnections()
await closed
}
}
}
/**
* Start server.mjs as a child process and resolve once it is listening, retrying
* if another process grabbed the port between `freePort()` and the child binding.
*/
export async function startMultiplayerServer(env = {}, attemptsLeft = 5) {
const port = await freePort()
const child = spawn(process.execPath, [SERVER_PATH], {
// Pin the settings the tests assert on, so an ambient
// REQUIRE_SIGNED_MULTIPLAYER_REQUESTS=false cannot turn the rejection tests
// into false passes, and BASE_INTERNAL_URL cannot stand in for the fake JWKS.
env: {
...process.env,
REQUIRE_SIGNED_MULTIPLAYER_REQUESTS: 'true',
BASE_INTERNAL_URL: '',
PORT: String(port),
HOST: '127.0.0.1',
...env
},
stdio: ['ignore', 'pipe', 'pipe']
})
let output = ''
const listening = new Promise((resolve, reject) => {
const onData = (chunk) => {
output += chunk.toString()
if (output.includes('Multiplayer server running')) resolve()
}
child.stdout.on('data', onData)
child.stderr.on('data', onData)
child.once('error', reject)
child.once('exit', (code) => reject(new Error(`server exited early (code ${code}):\n${output}`)))
})
try {
await listening
} catch (error) {
if (output.includes('EADDRINUSE') && attemptsLeft > 1) {
return startMultiplayerServer(env, attemptsLeft - 1)
}
throw error
}
return {
port,
url: `ws://127.0.0.1:${port}`,
get output() {
return output
},
async close() {
if (child.exitCode !== null) return
const exited = new Promise((resolve) => child.once('exit', resolve))
child.kill('SIGKILL')
await exited
}
}
}
/** Poll `predicate` until it is true, or throw after `timeoutMs`. */
export async function waitFor(predicate, { timeoutMs = 15000, intervalMs = 10, message = 'condition' } = {}) {
const deadline = Date.now() + timeoutMs
while (Date.now() < deadline) {
if (await predicate()) return
await new Promise((resolve) => setTimeout(resolve, intervalMs))
}
throw new Error(`Timed out after ${timeoutMs}ms waiting for ${message}`)
}