mirror of
https://github.com/stablyai/orca.git
synced 2026-09-21 16:02:20 +00:00
Jinwoo Hong
164c7140fd
A host whose home cell is not live — readiness false, drained, or inside a boot window — is refused by the committed-fence branch in assignOnce() without any capacity being consulted. It answered relay_capacity_exhausted, so every cell boot and every readiness dip printed capacity rejections at 17% fleet utilisation and sent an investigation after headroom that was never short. The branch now raises RelayHomeCellUnavailableError, which carries the cell id and which of cellIsLive()'s conditions failed (draining / booting / unheard / not_ready). The director logs reason, cause and cell, and returns the new reason in the same retryable 503. Nothing on the wire reads the body: the desktop client discards it unread and branches on status only, and no log-based metric or alert parses the reason. The load harness, the only body-reading consumer, gets its own bucket so a home-cell rejection no longer inflates the capacity count. Hinted grants are now logged on whichever lane served them, so a host that failed sticky verification and was rehomed by placement leaves a record of where it landed. Unhinted placement grants stay silent.
46 lines
2.3 KiB
JavaScript
46 lines
2.3 KiB
JavaScript
// A refused home cell is not fleet capacity, so it gets its own bucket rather
|
|
// than inflating the capacity count a run is read for.
|
|
const ASSIGNMENT_REJECTION_BUCKETS = {
|
|
relay_capacity_exhausted: 'assignment_capacity_exhausted',
|
|
relay_connection_headroom_exhausted: 'assignment_capacity_exhausted',
|
|
relay_home_cell_unavailable: 'assignment_home_cell_unavailable'
|
|
}
|
|
|
|
export function relayLoadFailureReason(error) {
|
|
const message = error instanceof Error ? error.message : String(error)
|
|
const tokenExchange = /^relay token exchange failed: ([1-5][0-9]{2})$/.exec(message)
|
|
if (tokenExchange) return `token_http_${tokenExchange[1]}`
|
|
const assignment =
|
|
/^relay assignment failed: ([1-5][0-9]{2})(?: (relay_[a-z_]+))?$/.exec(message)
|
|
if (assignment?.[1] === '503' && assignment[2]) {
|
|
return ASSIGNMENT_REJECTION_BUCKETS[assignment[2]] ?? `assignment_http_${assignment[1]}`
|
|
}
|
|
if (assignment) return `assignment_http_${assignment[1]}`
|
|
const closed = /^control closed: ([0-9]{4})\b/.exec(message)
|
|
if (closed) return `control_close_${closed[1]}`
|
|
if (message === 'control open timeout') return 'control_open_timeout'
|
|
if (message === 'control response timeout') return 'control_response_timeout'
|
|
if (message === 'relay token exchange timeout') return 'token_timeout'
|
|
if (message === 'relay assignment timeout') return 'assignment_timeout'
|
|
if (message === 'WebSocket was closed before the connection was established') {
|
|
return 'socket_closed_before_open'
|
|
}
|
|
if (message === 'relay token exchange omitted token') return 'token_response_invalid'
|
|
if (message === 'relay assignment response invalid') return 'assignment_response_invalid'
|
|
if (message === 'expected host challenge') return 'host_challenge_invalid'
|
|
if (message === 'host proof challenge did not decrypt') return 'host_challenge_decrypt_failed'
|
|
if (message === 'expected host hello acknowledgement') return 'host_ack_invalid'
|
|
const socketResponse = /^Unexpected server response: ([1-5][0-9]{2})\b/.exec(message)
|
|
if (socketResponse) return `socket_http_${socketResponse[1]}`
|
|
if (/\b(?:ECONNREFUSED|ECONNRESET|EHOSTUNREACH|ETIMEDOUT)\b/.test(message)) {
|
|
return 'socket_transport'
|
|
}
|
|
return 'unknown'
|
|
}
|
|
|
|
export function discardFailedLoadSocket(socket) {
|
|
if (!socket) return
|
|
socket.on('error', () => undefined)
|
|
socket.terminate()
|
|
}
|