feat(ssh): report each connect's server decision, and show orcad.log's last lines when setup fails (#25118)

* telemetry: one enum-only event per connect decision (outcome, reason, tunnel transport, host platform, duration), plus conversion start/commit/fail, deploy failures, and the per-host move offer and its result
* errors: deploy, launch and rollback failures carry the last 40 redacted lines of the host's orcad.log, read over SSH (Windows through the node.exe host script)
* SSH Hosts: a deferred or failed setup offers its reason, log tail included, under Details

Co-authored-by: m4air <m4air@Mac.localdomain>
This commit is contained in:
OrcaWin
2026-10-03 21:16:10 -07:00
committed by GitHub
co-authored by m4air
parent 04b84870ac
commit 080b9ffdf8
32 changed files with 1324 additions and 26 deletions
+3
View File
@@ -4,6 +4,8 @@ import type { SshConnectionState, SshTarget } from '../../shared/ssh-types'
import type { HostServerOnConnectResult } from '../ssh/ssh-host-server-on-connect'
import { relayServerStatus, shouldToastManagedServerMove } from '../ssh/ssh-host-server-move-offer'
import { setSshHostServerStatus } from '../ssh/ssh-host-server-status'
import { trackSshHostServerMove } from '../ssh/ssh-host-server-telemetry'
import { knownSshHostPlatform } from '../ssh/ssh-host-platform-memo'
import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry'
import { connectionManager, getCurrentMainWindow } from './ssh-ipc-context'
import { broadcastSshState, getPublicSshState } from './ssh-renderer-broadcast'
@@ -57,6 +59,7 @@ export function recordRelayDecision(
getSshTargetRegistryStore()!.updateTarget(target.id, {
managedServerMoveOffered: { appVersion }
})
trackSshHostServerMove('offered', knownSshHostPlatform(target.id))
}
setSshHostServerStatus(target.id, relayServerStatus(decision, offerMove))
}
@@ -15,6 +15,9 @@ import { orcadMigrationDestinationFor } from '../ssh/orcad-runtime-conversion-wi
import { createManagedOrcadEnvironment } from '../ssh/orcad-runtime-deployment'
import type { HostServerOnConnectDeps } from '../ssh/ssh-host-server-on-connect'
import { setSshHostServerStatus } from '../ssh/ssh-host-server-status'
import { trackSshHostServerEvent } from '../ssh/ssh-host-server-telemetry'
import { knownSshHostPlatform } from '../ssh/ssh-host-platform-memo'
import { knownOrcadTunnelTransport } from '../ssh/orcad-tunnel-transport-memo'
import {
getSshTargetRegistryStore,
hasRegisteredDirectSshAuthority
@@ -117,6 +120,13 @@ export function hostServerOnConnectDeps(userDataPath: string): HostServerOnConne
findOrcadMigrationSourceCutoverForTarget(userDataPath, target.id)?.phase === 'source-fenced',
releaseUnreachableSetup: (target) =>
releaseUnreachableOrcadSetup({ userDataPath, claims, targetId: target.id }),
// Read at report time: a deploy or conversion learns the platform while the decision runs.
report: (target, event) =>
trackSshHostServerEvent(
event,
knownSshHostPlatform(target.id),
knownOrcadTunnelTransport(target.id)
),
progress: (target, phase) => {
setSshHostServerStatus(target.id, { kind: 'setting-up', phase })
broadcastSshState(getCurrentMainWindow, target.id, {
+7 -1
View File
@@ -38,7 +38,8 @@ function deps(
calls.push('connect')
status = options.afterConnect ?? { kind: 'managed', environmentId: 'env-1' }
}),
serverStatus: vi.fn(() => status)
serverStatus: vi.fn(() => status),
report: vi.fn()
}
}
@@ -51,6 +52,7 @@ describe('moving an SSH host to its managed server on request', () => {
})
expect(move.calls).toEqual(['terminate', 'census', 'connect'])
expect(move.relayTerminals).toHaveBeenCalledWith(target)
expect(move.report).toHaveBeenCalledWith('ssh-1', 'moved')
})
it('refuses without converting when the stop could not reach every terminal', async () => {
@@ -61,6 +63,7 @@ describe('moving an SSH host to its managed server on request', () => {
terminals: 1
})
expect(move.calls).toEqual(['terminate'])
expect(move.report).toHaveBeenCalledWith('ssh-1', 'refused_unverifiable')
})
it('refuses without converting when the census is unverifiable or still live', async () => {
@@ -78,11 +81,13 @@ describe('moving an SSH host to its managed server on request', () => {
verdict: 'live'
})
expect(live.connect).not.toHaveBeenCalled()
expect(live.report).toHaveBeenCalledWith('ssh-1', 'refused_live')
})
it('reports a connect that kept the relay for another reason', async () => {
const move = deps({ afterConnect: { kind: 'relay', reason: 'refused', detail: 'blocked' } })
await expect(moveSshHostToManagedServer('ssh-1', move)).resolves.toEqual({ outcome: 'stayed' })
expect(move.report).toHaveBeenCalledWith('ssh-1', 'stayed')
})
it('reattaches the relay first when preserved terminals need one to be stopped', async () => {
@@ -101,5 +106,6 @@ describe('moving an SSH host to its managed server on request', () => {
move.terminate.mockRejectedValueOnce(new Error('Failed to terminate SSH host sessions'))
await expect(moveSshHostToManagedServer('ssh-1', move)).rejects.toThrow('Failed to terminate')
expect(move.connect).not.toHaveBeenCalled()
expect(move.report).toHaveBeenCalledWith('ssh-1', 'failed')
})
})
+28 -1
View File
@@ -12,6 +12,11 @@ import type {
} from '../../shared/ssh-types'
import type { HostServerTerminalVerdict } from '../ssh/ssh-host-server-on-connect'
import { getSshHostServerStatus } from '../ssh/ssh-host-server-status'
import {
trackSshHostServerMove,
type SshHostServerMoveOutcome
} from '../ssh/ssh-host-server-telemetry'
import { knownSshHostPlatform } from '../ssh/ssh-host-platform-memo'
import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry'
import { connectTarget } from './ssh-connect-flow'
import { terminateSshTargetSessions } from './ssh-terminate-sessions'
@@ -22,11 +27,32 @@ export type SshManagedServerMoveDeps = {
relayTerminals: (target: SshTarget) => Promise<HostServerTerminalVerdict>
connect: (targetId: string) => Promise<unknown>
serverStatus: (targetId: string) => SshManagedServerStatus | undefined
report: (targetId: string, outcome: SshHostServerMoveOutcome) => void
}
export async function moveSshHostToManagedServer(
targetId: string,
deps: SshManagedServerMoveDeps = defaultMoveDeps()
): Promise<SshManagedServerMoveResult> {
let result: SshManagedServerMoveResult | null = null
try {
result = await moveHost(targetId, deps)
return result
} finally {
deps.report(targetId, moveOutcome(result))
}
}
function moveOutcome(result: SshManagedServerMoveResult | null): SshHostServerMoveOutcome {
if (!result) {
return 'failed'
}
return result.outcome === 'refused' ? `refused_${result.verdict}` : result.outcome
}
async function moveHost(
targetId: string,
deps: SshManagedServerMoveDeps
): Promise<SshManagedServerMoveResult> {
const target = deps.getTarget(targetId)
if (!target) {
@@ -74,6 +100,7 @@ function defaultMoveDeps(): SshManagedServerMoveDeps {
return hostServerOnConnectDeps(getAppEnvironment().getPath('userData')).relayTerminals(target)
},
connect: connectTarget,
serverStatus: getSshHostServerStatus
serverStatus: getSshHostServerStatus,
report: (targetId, outcome) => trackSshHostServerMove(outcome, knownSshHostPlatform(targetId))
}
}
+5 -1
View File
@@ -31,7 +31,11 @@ const MAIN_OWNED_TELEMETRY_EVENTS = new Set<EventName>([
'daemon_pty_cwd_readable',
'star_nag_outcome',
'feature_interaction_usage_bucket_reached',
'ssh_remote_runtime_resolved'
'ssh_remote_runtime_resolved',
'ssh_host_server_decided',
'ssh_host_server_conversion',
'ssh_host_server_deploy_failed',
'ssh_host_server_move'
])
/**
+10 -1
View File
@@ -16,6 +16,7 @@ import {
readOrcadReadinessNowCommand
} from './orcad-remote-readiness-wait'
import type { ServeReadiness } from '../server/serve-readiness'
import { withOrcadLogTail } from './orcad-remote-log-tail'
/** `exited` is proven absence; `unverifiable` means the host could not say, which is not death. */
export type OrcadReadinessFailureVerdict = 'exited' | 'unverifiable' | 'rejected'
@@ -116,5 +117,13 @@ export async function launchOrcadSlotAndAwaitReadiness(
expectation: OrcadActivationExpectation
): Promise<ServeReadiness> {
const parsed = await launchOrcadAndAwaitReadiness(target, spec)
return gatedReadiness(parsed, expectation, `orcad ${spec.fullVersion}`)
try {
return gatedReadiness(parsed, expectation, `orcad ${spec.fullVersion}`)
} catch (error) {
if (!(error instanceof OrcadActiveReadinessError)) {
throw error
}
const message = await withOrcadLogTail(target, spec.remoteInstallDir, error.message)
throw new OrcadActiveReadinessError(error.verdict, message)
}
}
+4 -4
View File
@@ -57,6 +57,7 @@ import {
type OrcadSlotIdentity
} from './orcad-recovery-slot'
import { orcadSnapshotPath, recoverOrcadIncumbent } from './orcad-incumbent-recovery'
import { withOrcadLogTail } from './orcad-remote-log-tail'
type Outcome = Extract<OrcadDeployResult, { outcome: 'installed-not-activated' }>
@@ -242,11 +243,10 @@ export async function activateInstalledOrcad(
`The candidate failed while starting: ${errorMessage(launchError)}`
]
const restored = await restoreAfterRejectedCandidate(options, transaction, incumbent, lock)
return notActivated(
code,
const located =
`${reason} Candidate stderr is at ` +
`${joinRemotePath(options.host, remoteDir, ORCAD_LOG_FILENAME)}. ${restored}`
)
`${joinRemotePath(options.host, remoteDir, ORCAD_LOG_FILENAME)}. ${restored}`
return notActivated(code, await withOrcadLogTail(options, remoteDir, located))
}
const recordAfter = withActivatedVersion(record, fullVersion, snapshot, now())
@@ -1,5 +1,6 @@
import { describe, expect, it, vi } from 'vitest'
import { OrcadManagedTunnelTransportProvider } from './orcad-managed-tunnel-transport'
import { knownOrcadTunnelTransport } from './orcad-tunnel-transport-memo'
import type { SshConnection } from './ssh-connection'
import type {
PortForwardStartOptions,
@@ -49,6 +50,7 @@ describe('the managed tunnel’s transport', () => {
})
expect(refused.stdio.start).toHaveBeenCalledWith(conn, options)
expect(refused.forward.start).not.toHaveBeenCalled()
expect(knownOrcadTunnelTransport('ssh-1')).toBe('stdio_bridge')
})
it('keeps the forward when forwarding is allowed or the answer is unverifiable', async () => {
@@ -58,6 +60,7 @@ describe('the managed tunnel’s transport', () => {
entry: { id: 'forward' }
})
expect(chosen.stdio.start).not.toHaveBeenCalled()
expect(knownOrcadTunnelTransport('ssh-1')).toBe('tcp_forward')
}
})
})
@@ -3,6 +3,7 @@
* wherever the host allows it, and the stdio bridge where its sshd refuses forwarding.
*/
import { OrcadStdioBridgePortForwardProvider } from './orcad-stdio-bridge-provider'
import { rememberOrcadTunnelTransport } from './orcad-tunnel-transport-memo'
import type { SshConnection } from './ssh-connection'
import type {
PortForwardStartOptions,
@@ -36,12 +37,16 @@ export class OrcadManagedTunnelTransportProvider implements SshPortForwardProvid
const { forwards, stdio, probe } = this.dependencies
// Why only on refusal: an unverifiable answer keeps the forward, as before this transport.
if ((await probe(conn, options.remotePort)) === 'refused' && stdio.canHandle(conn)) {
return stdio.start(conn, options)
const started = await stdio.start(conn, options)
rememberOrcadTunnelTransport(options.connectionId, 'stdio_bridge')
return started
}
const forward = forwards.find((provider) => provider.canHandle(conn))
if (!forward) {
throw new Error('SSH connection is not established')
}
return forward.start(conn, options)
const started = await forward.start(conn, options)
rememberOrcadTunnelTransport(options.connectionId, 'tcp_forward')
return started
}
}
+2
View File
@@ -17,6 +17,7 @@ import {
} from './ssh-remote-platform'
import { detectRemoteHostPlatform } from './ssh-remote-platform-detection'
import { OrcadHostUnsupportedError } from './orcad-host-unavailable'
import { rememberSshHostPlatform } from './ssh-host-platform-memo'
export type OrcadRemoteContext = {
activationRecord: OrcadActivationRecord
@@ -47,6 +48,7 @@ export async function resolveOrcadRemoteContext(
throw new Error(`Remote home is not a valid path: ${remoteHome.slice(0, 100)}`)
}
const serverTarget = await resolveOrcadDeploymentTarget({ conn: connection, host, signal })
rememberSshHostPlatform(target.id, host, serverTarget)
if (isWindowsRemoteHost(host)) {
// Every Windows host op, the activation record read included, runs on the pinned node.exe.
await prepareWindowsOrcadHost({ conn: connection, host, remoteHome, serverTarget, signal })
+10 -1
View File
@@ -109,11 +109,15 @@ type HostScript = {
comparisonResult?: string
candidateStopResult?: string
readinessAtMs?: number
orcadLog?: string
}
function scriptHost(script: HostScript): void {
mockExec.mockImplementation(async (_conn, command: string) => {
const text = String(command)
if (text.startsWith('tail -c')) {
return script.orcadLog ?? ''
}
if (text.includes('__ORCAD_RECORD_PRESENT__') && text.includes('orcad-active.json')) {
return script.activationRecord
? `__ORCAD_RECORD_PRESENT__\n${script.activationRecord}`
@@ -504,11 +508,16 @@ describe('deployOrcad', () => {
const script: HostScript = {
activationRecord: ACTIVE_OLD,
readiness: { [NEW_VERSION]: readyLine({ daemonState: 'degraded' }) },
log: []
log: [],
orcadLog: 'daemon: starting\ndaemon: socket bind failed: EACCES\n'
}
scriptHost(script)
const result = await deployOrcad(options())
expect(result).toMatchObject({ code: 'orcad_activation_daemon_degraded' })
// The error carries why the candidate failed, not only where its log is.
expect(result).toMatchObject({
reason: expect.stringMatching(/Last lines of orcad\.log:\ndaemon: starting\n.*EACCES$/u)
})
expect(
vi
.mocked(writeAtomicOrcadRemoteRecord)
+111
View File
@@ -0,0 +1,111 @@
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { runProcess } from '../../shared/child-process/run-process'
import {
ORCAD_LOG_TAIL_LINES,
ORCAD_LOG_TAIL_MAX_BYTES,
orcadLogTailCommand,
parseOrcadLogTail,
withOrcadLogTail
} from './orcad-remote-log-tail'
import { ORCAD_WINDOWS_HOST_SCRIPT } from './orcad-windows-host-script'
import { getRemoteHostPlatform } from './ssh-remote-platform'
const execCommand = vi.hoisted(() => vi.fn())
vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand }))
const linux = getRemoteHostPlatform('linux-x64')
const windows = getRemoteHostPlatform('win32-x64')
const SLOT = '/home/me/.orca-remote/orcad-1.0.0'
const WINDOWS_SLOT = 'C:\\Users\\me\\.orca-remote\\orcad-1.0.0'
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand is mocked; the connection is never used.
const conn = {} as never
let dir = ''
beforeEach(() => {
dir = mkdtempSync(join(tmpdir(), 'orcad-log-tail-'))
execCommand.mockReset()
})
afterEach(() => rmSync(dir, { recursive: true, force: true }))
function numberedLines(count: number): string {
return Array.from({ length: count }, (_, index) => `line ${index + 1}`).join('\n')
}
describe('orcad.log tail', () => {
it('keeps only the last lines and bytes, redacting secrets', () => {
const tail = parseOrcadLogTail(linux, `${numberedLines(100)}\ntoken=abc123secret\n`)
const lines = (tail ?? '').split('\n')
expect(lines).toHaveLength(ORCAD_LOG_TAIL_LINES)
expect(lines[0]).toBe('line 62')
expect(tail).not.toContain('abc123secret')
expect(tail).toContain('[redacted')
const huge = parseOrcadLogTail(linux, 'x'.repeat(ORCAD_LOG_TAIL_MAX_BYTES * 4))
expect(Buffer.byteLength(huge ?? '', 'utf8')).toBeLessThanOrEqual(ORCAD_LOG_TAIL_MAX_BYTES)
expect(parseOrcadLogTail(linux, ' \n')).toBeNull()
})
it('reads the log on Windows through the node.exe host script, never an encoded command', () => {
const command = orcadLogTailCommand(windows, WINDOWS_SLOT)
expect(command).toContain('node.exe')
expect(command).toContain(' log-tail ')
expect(command).not.toMatch(/EncodedCommand/iu)
})
it.skipIf(process.platform === 'win32')(
'runs the POSIX command against a real file',
async () => {
const slot = join(dir, 'slot')
mkdirSync(slot)
writeFileSync(join(slot, 'orcad.log'), `${numberedLines(60)}\n`)
const run = await runProcess({
program: '/bin/sh',
args: ['-c', orcadLogTailCommand(linux, slot)],
timeoutMs: 5_000
})
expect(parseOrcadLogTail(linux, run.stdout)?.split('\n').at(-1)).toBe('line 60')
const missing = await runProcess({
program: '/bin/sh',
args: ['-c', orcadLogTailCommand(linux, join(dir, 'absent'))],
timeoutMs: 5_000
})
expect(parseOrcadLogTail(linux, missing.stdout)).toBeNull()
}
)
it('reads the end of the log through the real Windows host script op', async () => {
const script = join(dir, 'host-script.js')
writeFileSync(script, ORCAD_WINDOWS_HOST_SCRIPT)
const log = join(dir, 'orcad.log')
writeFileSync(log, `${'y'.repeat(ORCAD_LOG_TAIL_MAX_BYTES * 2)}\nfatal: bind failed\n`)
const run = await runProcess({
program: process.execPath,
args: [script, 'log-tail', log, String(ORCAD_LOG_TAIL_MAX_BYTES)],
timeoutMs: 15_000
})
const tail = parseOrcadLogTail(windows, run.stdout)
expect(tail?.endsWith('fatal: bind failed')).toBe(true)
const absent = await runProcess({
program: process.execPath,
args: [script, 'log-tail', join(dir, 'absent.log'), '100'],
timeoutMs: 15_000
})
expect(parseOrcadLogTail(windows, absent.stdout)).toBeNull()
})
it('appends the tail to a failure message, and leaves it alone when the host cannot answer', async () => {
execCommand.mockResolvedValueOnce('starting\nError: EADDRINUSE\n')
await expect(withOrcadLogTail({ conn, host: linux }, SLOT, 'Launch failed.')).resolves.toBe(
'Launch failed.\nLast lines of orcad.log:\nstarting\nError: EADDRINUSE'
)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
execCommand.mockRejectedValueOnce(new Error('Connection lost'))
await expect(withOrcadLogTail({ conn, host: linux }, SLOT, 'Launch failed.')).resolves.toBe(
'Launch failed.'
)
warn.mockRestore()
})
})
+71
View File
@@ -0,0 +1,71 @@
/**
* The end of a slot's orcad.log, appended to a failed deploy, launch or rollback so the error
* says why orcad stopped rather than only where its log is. Errors and the main log only; never
* telemetry. Best-effort: a host that can't answer leaves the error as it was.
*/
import { redactString } from '../observability/redactor'
import { shellEscape } from './ssh-connection-utils'
import { ORCAD_LOG_FILENAME } from './orcad-remote-host-support'
import { execCommand } from './ssh-relay-deploy-helpers'
import { isWindowsRemoteHost, joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform'
import {
orcadWindowsBaseDir,
orcadWindowsHostOpCommand,
readOrcadWindowsEncodedAnswer
} from './orcad-remote-windows-node'
import { ORCAD_WINDOWS_LOG_TAIL_MARKER } from './orcad-windows-host-script'
import type { OrcadRemoteExecTarget } from './orcad-remote-runtime-control'
export const ORCAD_LOG_TAIL_LINES = 40
export const ORCAD_LOG_TAIL_MAX_BYTES = 8 * 1024
// Short, because it only decorates an error the caller already has.
const ORCAD_LOG_TAIL_TIMEOUT_MS = 10_000
export function orcadLogTailCommand(host: RemoteHostPlatform, slotDir: string): string {
const log = joinRemotePath(host, slotDir, ORCAD_LOG_FILENAME)
if (isWindowsRemoteHost(host)) {
return orcadWindowsHostOpCommand(host, orcadWindowsBaseDir(host, slotDir), 'log-tail', [
log,
String(ORCAD_LOG_TAIL_MAX_BYTES)
])
}
return `tail -c ${ORCAD_LOG_TAIL_MAX_BYTES} ${shellEscape(log)} 2>/dev/null || true`
}
/** The last lines of what the host printed, redacted; null when the log is empty or absent. */
export function parseOrcadLogTail(host: RemoteHostPlatform, output: string): string | null {
const raw = isWindowsRemoteHost(host)
? (readOrcadWindowsEncodedAnswer(output, ORCAD_WINDOWS_LOG_TAIL_MARKER) ?? '')
: output
const bounded = Buffer.from(raw, 'utf8').subarray(-ORCAD_LOG_TAIL_MAX_BYTES).toString('utf8')
const lines = bounded.replace(/\s+$/u, '').split(/\r?\n/u).slice(-ORCAD_LOG_TAIL_LINES)
const tail = redactString(lines.join('\n'))
return tail.trim() ? tail : null
}
export async function readOrcadLogTail(
target: OrcadRemoteExecTarget,
slotDir: string
): Promise<string | null> {
try {
// No abort signal: the failure being reported may be the cancellation itself.
const output = await execCommand(target.conn, orcadLogTailCommand(target.host, slotDir), {
wrapCommand: target.host.commandDialect !== 'powershell',
timeoutMs: ORCAD_LOG_TAIL_TIMEOUT_MS
})
return parseOrcadLogTail(target.host, output)
} catch (error) {
console.warn('[ssh] Could not read the orcad.log tail:', error)
return null
}
}
/** `message` followed by the log tail, or `message` alone when the host had none to give. */
export async function withOrcadLogTail(
target: OrcadRemoteExecTarget,
slotDir: string,
message: string
): Promise<string> {
const tail = await readOrcadLogTail(target, slotDir)
return tail ? `${message}\nLast lines of orcad.log:\n${tail}` : message
}
@@ -91,6 +91,9 @@ function scriptHost(log: string[], overrides: HostOverrides = {}): void {
const restores = [...(overrides.restores ?? [])]
mockExec.mockImplementation(async (_conn, command: string) => {
const text = String(command)
if (text.startsWith('tail -c')) {
return text.includes(TARGET) ? 'orcad: listen EADDRINUSE 127.0.0.1\n' : ''
}
if (text.includes('__ORCAD_RECORD_PRESENT__')) {
return text.includes('transaction.json')
? '__ORCAD_RECORD_ABSENT__\n'
@@ -249,6 +252,9 @@ describe('rollbackOrcad', () => {
const result = await rollbackOrcad(options())
expect(result).toMatchObject({ outcome: 'failed', code: 'orcad_activation_no_readiness' })
expect(result.outcome === 'failed' && result.reason).toContain('fresh host terminal census')
expect(result.outcome === 'failed' && result.reason).toContain(
'Last lines of orcad.log:\norcad: listen EADDRINUSE'
)
expect(log).toEqual([
`stop:${ACTIVE}`,
'rescue',
+6 -2
View File
@@ -42,6 +42,7 @@ import {
type OrcadSlotIdentity
} from './orcad-recovery-slot'
import { orcadSnapshotPath, recoverOrcadIncumbent } from './orcad-incumbent-recovery'
import { withOrcadLogTail } from './orcad-remote-log-tail'
/** A confirmed failure reads as no answer; an unconfirmed one propagates and keeps the fence. */
async function execOrEmpty(options: OrcadRollbackOptions, command: string): Promise<string> {
@@ -235,9 +236,12 @@ export async function rollbackOrcadLocked(
return {
outcome: 'failed',
code: launchError === undefined ? verdict.code : 'orcad_rollback_target_launch_failed',
reason:
reason: await withOrcadLogTail(
options,
targetDir,
`The rollback target ${safety.target} did not come up healthy: ${reason} ${recovered} ` +
`Its stderr is at ${joinRemotePath(options.host, targetDir, ORCAD_LOG_FILENAME)}.`
`Its stderr is at ${joinRemotePath(options.host, targetDir, ORCAD_LOG_FILENAME)}.`
)
}
}
@@ -0,0 +1,20 @@
/** Which transport each host's managed tunnel last started on, for connect telemetry. */
export type OrcadTunnelTransport = 'tcp_forward' | 'stdio_bridge'
// Keyed by the local target id, which never leaves this process.
const transports = new Map<string, OrcadTunnelTransport>()
export function rememberOrcadTunnelTransport(
targetId: string,
transport: OrcadTunnelTransport
): void {
transports.set(targetId, transport)
}
export function knownOrcadTunnelTransport(targetId: string): OrcadTunnelTransport | null {
return transports.get(targetId) ?? null
}
export function resetOrcadTunnelTransportMemoForTests(): void {
transports.clear()
}
+17
View File
@@ -40,6 +40,7 @@ export const ORCAD_BUILD_HASH_MARKER = '__ORCAD_BUILD_HASH__'
export const ORCAD_WINDOWS_READINESS_MARKER = '__ORCAD_READINESS__'
export const ORCAD_WINDOWS_RUNTIME_MARKER = '__ORCAD_RUNTIME__'
export const ORCAD_WINDOWS_LIVENESS_MANY_MARKER = '__ORCAD_LIVENESS__'
export const ORCAD_WINDOWS_LOG_TAIL_MARKER = '__ORCAD_LOG_TAIL__'
/** A slot whose marker names no usable runtime; the POSIX selector exits the same way. */
export const ORCAD_WINDOWS_RUNTIME_MISSING_EXIT = 78
/** The record exists but cannot be read within its bound. */
@@ -50,6 +51,7 @@ export type OrcadWindowsHostOp =
| 'liveness-many'
| 'stop'
| 'readiness-wait'
| 'log-tail'
| 'build-hash'
| 'record-read'
| 'record-publish'
@@ -163,6 +165,21 @@ const ops = {
tick()
},
// The last bytes of a log, as POSIX \`tail -c\` reads them; a missing log is empty.
'log-tail'(file, capArg) {
const cap = Number(capArg)
let fd
try { fd = fs.openSync(file, 'r') } catch { return encoded(${text(ORCAD_WINDOWS_LOG_TAIL_MARKER)}, Buffer.alloc(0)) }
let buffer = Buffer.alloc(0)
try {
const size = fs.fstatSync(fd).size
const length = Math.min(cap, size)
buffer = Buffer.alloc(length)
buffer = buffer.subarray(0, fs.readSync(fd, buffer, 0, length, size - length))
} catch { buffer = Buffer.alloc(0) } finally { fs.closeSync(fd) }
encoded(${text(ORCAD_WINDOWS_LOG_TAIL_MARKER)}, buffer)
},
'build-hash'(file) {
const digest = crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex')
answer(${text(`${ORCAD_BUILD_HASH_MARKER} `)} + digest.slice(0, 16) + '\\n')
@@ -0,0 +1,24 @@
import { beforeEach, describe, expect, it } from 'vitest'
import {
knownSshHostPlatform,
rememberSshHostPlatform,
resetSshHostPlatformMemoForTests
} from './ssh-host-platform-memo'
import { getRemoteHostPlatform } from './ssh-remote-platform'
describe('ssh host platform memo', () => {
beforeEach(() => {
resetSshHostPlatformMemoForTests()
})
it('records the libc flavor, and keeps a probed one when a later report has none', () => {
expect(knownSshHostPlatform('ssh-1')).toBeNull()
rememberSshHostPlatform('ssh-1', getRemoteHostPlatform('linux-arm64'), 'linux-arm64-musl')
expect(knownSshHostPlatform('ssh-1')).toEqual({ os: 'linux', arch: 'arm64', libc: 'musl' })
rememberSshHostPlatform('ssh-1', getRemoteHostPlatform('linux-arm64'), null)
expect(knownSshHostPlatform('ssh-1')?.libc).toBe('musl')
rememberSshHostPlatform('ssh-2', getRemoteHostPlatform('win32-x64'), 'win32-x64')
expect(knownSshHostPlatform('ssh-2')).toEqual({ os: 'win32', arch: 'x64', libc: 'none' })
})
})
+50
View File
@@ -0,0 +1,50 @@
/**
* The platform each SSH host last reported this app session, so connect telemetry can name it
* without a probe of its own. Keyed by the local target id, which never leaves this process.
*/
import type { ServerTarget } from '../../shared/node-runtime-pin'
import type { RemoteHostPlatform } from './ssh-remote-platform'
export type SshHostPlatformFacts = {
os: RemoteHostPlatform['os']
arch: RemoteHostPlatform['arch']
libc: 'glibc' | 'musl' | 'none' | 'unknown'
}
const known = new Map<string, SshHostPlatformFacts>()
function libcOf(
host: RemoteHostPlatform,
target: ServerTarget | null
): SshHostPlatformFacts['libc'] {
if (host.os !== 'linux') {
return 'none'
}
if (!target) {
return 'unknown'
}
return target.endsWith('-musl') ? 'musl' : 'glibc'
}
export function rememberSshHostPlatform(
targetId: string,
host: RemoteHostPlatform,
target: ServerTarget | null
): void {
const libc = libcOf(host, target)
const previous = known.get(targetId)
// A libc probe that did not run must not erase one that did.
known.set(targetId, {
os: host.os,
arch: host.arch,
libc: libc === 'unknown' && previous ? previous.libc : libc
})
}
export function knownSshHostPlatform(targetId: string): SshHostPlatformFacts | null {
return known.get(targetId) ?? null
}
export function resetSshHostPlatformMemoForTests(): void {
known.clear()
}
@@ -0,0 +1,186 @@
/**
* What one connect's server decision did, in raw codes; ssh-host-server-telemetry maps these onto
* enum-only events. Nothing here carries a message, path or host name.
*/
import type {
OrcadManagedConversionResult,
OrcadManagedDeployResult
} from '../../shared/orcad-managed-runtime'
import type { SshTarget } from '../../shared/ssh-types'
import type { HostServerOnConnectResult } from './ssh-host-server-on-connect'
import { classifyOrcadHostUnavailable } from './orcad-host-unavailable'
export type HostServerConnectEvent =
| {
kind: 'decided'
outcome: 'managed' | 'deployed' | 'converted' | 'relay'
reason: string
refusal: string | null
recorded: boolean
durationMs: number
}
| { kind: 'conversion'; phase: 'started' }
| { kind: 'conversion'; phase: 'committed'; durationMs: number }
| {
kind: 'conversion'
phase: 'failed'
failure: 'deferred' | 'refused' | 'error'
code: string | null
durationMs: number
}
| {
kind: 'deploy_failed'
context: 'deploy' | 'conversion'
failure: 'deferred' | 'error'
code: string | null
durationMs: number
}
/** How far one decision got, filled in as it runs. */
export type HostServerDecisionTrace = {
startedAt: number
path: 'existing' | 'deploy' | 'convert' | null
conversionStartedAt: number | null
recorded: boolean
refusal: string | null
}
export function startHostServerDecisionTrace(now = Date.now()): HostServerDecisionTrace {
return { startedAt: now, path: null, conversionStartedAt: null, recorded: false, refusal: null }
}
export function decidedEvent(
result: HostServerOnConnectResult | null,
trace: HostServerDecisionTrace,
now = Date.now()
): HostServerConnectEvent {
const base = { refusal: null, recorded: trace.recorded, durationMs: now - trace.startedAt }
if (!result) {
return { kind: 'decided', outcome: 'relay', reason: 'failed', ...base }
}
if (result.route === 'managed') {
if (trace.path === 'deploy') {
return { kind: 'decided', outcome: 'deployed', reason: 'deployed', ...base }
}
if (trace.path === 'convert') {
return { kind: 'decided', outcome: 'converted', reason: 'converted', ...base }
}
return { kind: 'decided', outcome: 'managed', reason: 'connected', ...base }
}
if (result.reason === 'orcad_unavailable') {
// Why the detail: for this reason it is always a classified code, never a message.
return { kind: 'decided', outcome: 'relay', reason: result.detail ?? 'other', ...base }
}
return {
kind: 'decided',
outcome: 'relay',
reason: result.reason,
...base,
refusal: result.reason === 'refused' ? trace.refusal : null
}
}
/** A classified code for a thrown setup failure; null when it carries none. */
export function hostServerFailureCode(error: unknown): string | null {
const unavailable = classifyOrcadHostUnavailable(error)
if (unavailable) {
return unavailable
}
if (error && typeof error === 'object' && 'code' in error && typeof error.code === 'string') {
return error.code
}
return null
}
export type HostServerReport = (target: SshTarget, event: HostServerConnectEvent) => void
export function reportHostServerEvent(
report: HostServerReport,
target: SshTarget,
event: HostServerConnectEvent
): void {
try {
report(target, event)
} catch {
// Telemetry never costs a connect.
}
}
function sinceMs(startedAt: number | null): number {
return startedAt === null ? 0 : Date.now() - startedAt
}
export function reportHostServerSetupError(
report: HostServerReport,
target: SshTarget,
trace: HostServerDecisionTrace,
error: unknown
): void {
const code = hostServerFailureCode(error)
if (trace.path === 'deploy') {
reportHostServerEvent(report, target, {
kind: 'deploy_failed',
context: 'deploy',
failure: 'error',
code,
durationMs: sinceMs(trace.startedAt)
})
} else if (trace.path === 'convert') {
reportHostServerEvent(report, target, {
kind: 'conversion',
phase: 'failed',
failure: 'error',
code,
durationMs: sinceMs(trace.conversionStartedAt)
})
}
}
export function reportHostServerDeploy(
report: HostServerReport,
target: SshTarget,
trace: HostServerDecisionTrace,
result: OrcadManagedDeployResult
): void {
if (result.outcome === 'deferred') {
reportHostServerEvent(report, target, {
kind: 'deploy_failed',
context: 'deploy',
failure: 'deferred',
code: result.code,
durationMs: sinceMs(trace.startedAt)
})
}
}
export function reportHostServerConversion(
report: HostServerReport,
target: SshTarget,
trace: HostServerDecisionTrace,
result: OrcadManagedConversionResult
): void {
const durationMs = sinceMs(trace.conversionStartedAt)
if (result.outcome === 'converted') {
reportHostServerEvent(report, target, { kind: 'conversion', phase: 'committed', durationMs })
return
}
const failure = result.outcome === 'deferred' ? 'deferred' : 'refused'
const code = result.code
reportHostServerEvent(report, target, {
kind: 'conversion',
phase: 'failed',
failure,
code,
durationMs
})
if (failure === 'deferred') {
// The conversion's own deploy is what deferred.
reportHostServerEvent(report, target, {
kind: 'deploy_failed',
context: 'conversion',
failure,
code,
durationMs
})
}
}
@@ -0,0 +1,204 @@
import { describe, expect, it, vi } from 'vitest'
import type { SshTarget } from '../../shared/ssh-types'
import {
OrcadHostUnsupportedError,
OrcadStdioBridgeUnavailableError
} from './orcad-host-unavailable'
import type { HostServerConnectEvent } from './ssh-host-server-connect-events'
import {
resolveHostServerOnConnect,
type HostServerOnConnectDeps
} from './ssh-host-server-on-connect'
const target: SshTarget = { id: 'ssh-1', label: 'Box', host: 'box', port: 22, username: 'me' }
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the orchestrator reads only the environment id.
const environment = { id: 'env-1' } as never
function deps(overrides: Partial<HostServerOnConnectDeps> = {}): HostServerOnConnectDeps {
return {
managedEnvironmentId: () => null,
ensureTunnel: vi.fn(async () => undefined),
retireRetainedSource: vi.fn(async () => undefined),
abandonConversion: vi.fn(async () => undefined),
abandonDeploy: vi.fn(async () => undefined),
hasTemplate: () => true,
recordedUnavailable: () => null,
recordUnavailable: vi.fn(),
isEmptyHost: () => false,
relayTerminals: vi.fn(async () => ({ verdict: 'exited' as const, count: 0 })),
deploy: vi.fn(async () => ({ outcome: 'created' as const, environment, activeVersion: '1' })),
convert: vi.fn(async () => ({ outcome: 'converted' as const, environment, migrationId: 'm' })),
progress: vi.fn(),
isFencedBeforeStaging: () => false,
releaseUnreachableSetup: vi.fn(async () => undefined),
report: vi.fn(),
...overrides
}
}
async function eventsOf(d: HostServerOnConnectDeps): Promise<HostServerConnectEvent[]> {
await resolveHostServerOnConnect(target, d).catch(() => undefined)
return vi.mocked(d.report).mock.calls.map(([, event]) => event)
}
function decided(events: HostServerConnectEvent[]) {
const found = events.filter((event) => event.kind === 'decided')
expect(found).toHaveLength(1)
return found[0]
}
describe('connect-decision telemetry', () => {
it('reports exactly one decision per connect, naming the outcome', async () => {
const managed = await eventsOf(deps({ managedEnvironmentId: () => 'env-9' }))
expect(decided(managed)).toMatchObject({ outcome: 'managed', reason: 'connected' })
const deployed = await eventsOf(deps({ isEmptyHost: () => true }))
expect(decided(deployed)).toMatchObject({ outcome: 'deployed', reason: 'deployed' })
const converted = await eventsOf(deps())
expect(decided(converted)).toMatchObject({ outcome: 'converted', reason: 'converted' })
expect(converted.filter((event) => event.kind === 'conversion')).toEqual([
{ kind: 'conversion', phase: 'started' },
expect.objectContaining({ kind: 'conversion', phase: 'committed' })
])
})
it('names why a host stays on the relay', async () => {
const live = await eventsOf(
deps({ relayTerminals: async () => ({ verdict: 'live', count: 1 }) })
)
expect(decided(live)).toMatchObject({ outcome: 'relay', reason: 'relay_terminals_live' })
const noTunnel = await eventsOf(
deps({
managedEnvironmentId: () => 'env-9',
isFencedBeforeStaging: () => true,
ensureTunnel: async () => {
throw new OrcadStdioBridgeUnavailableError('no bridge')
}
})
)
expect(decided(noTunnel)).toMatchObject({ outcome: 'relay', reason: 'ssh_tunnel_unavailable' })
const recorded = await eventsOf(deps({ recordedUnavailable: () => 'runtime_self_test' }))
expect(decided(recorded)).toMatchObject({ reason: 'runtime_self_test', recorded: true })
const noTemplate = await eventsOf(deps({ hasTemplate: () => false }))
expect(decided(noTemplate)).toMatchObject({ reason: 'artifacts_unavailable' })
})
it('reports a refused conversion with its blocker code', async () => {
const events = await eventsOf(
deps({
convert: async () => ({
outcome: 'refused',
verdict: 'live',
code: 'orcad_migration_preflight_blocked',
reason: 'An automation still runs on this host.'
})
})
)
expect(decided(events)).toMatchObject({
reason: 'refused',
refusal: 'orcad_migration_preflight_blocked'
})
expect(events).toContainEqual(
expect.objectContaining({ kind: 'conversion', phase: 'failed', failure: 'refused' })
)
})
it('reports deploy failures, deferred or thrown, with their classified code', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const deferred = await eventsOf(
deps({
isEmptyHost: () => true,
deploy: async () => ({
outcome: 'deferred',
candidateVersion: '1.0.0',
code: 'orcad_activation_no_readiness',
reason: 'No readiness line.\nLast lines of orcad.log:\nboom'
})
})
)
expect(deferred).toContainEqual(
expect.objectContaining({
kind: 'deploy_failed',
context: 'deploy',
failure: 'deferred',
code: 'orcad_activation_no_readiness'
})
)
expect(decided(deferred)).toMatchObject({ outcome: 'relay', reason: 'deferred' })
const thrown = await eventsOf(
deps({
isEmptyHost: () => true,
deploy: async () => {
throw new OrcadHostUnsupportedError('no build for this host')
}
})
)
expect(thrown).toContainEqual(
expect.objectContaining({ kind: 'deploy_failed', failure: 'error', code: 'unsupported_host' })
)
expect(decided(thrown)).toMatchObject({ reason: 'unsupported_host' })
const conversion = await eventsOf(
deps({
convert: async () => ({
outcome: 'deferred',
candidateVersion: '1.0.0',
code: 'orcad_update_terminals_running',
reason: 'Terminals are running.'
})
})
)
expect(conversion).toContainEqual(
expect.objectContaining({ kind: 'conversion', phase: 'failed', failure: 'deferred' })
)
expect(conversion).toContainEqual(
expect.objectContaining({ kind: 'deploy_failed', context: 'conversion' })
)
warn.mockRestore()
})
it('shows a deferral reason, log tail included, under the SSH host', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const reason = 'No readiness line.\nLast lines of orcad.log:\nError: EADDRINUSE'
const d = deps({
isEmptyHost: () => true,
deploy: async () => ({
outcome: 'deferred',
candidateVersion: '1.0.0',
code: 'orcad_activation_no_readiness',
reason
})
})
await expect(resolveHostServerOnConnect(target, d)).resolves.toEqual({
route: 'relay',
reason: 'deferred',
detail: reason
})
expect(warn).toHaveBeenCalledWith(expect.stringContaining('deferred'), reason)
warn.mockRestore()
})
it('still reports, as failed, a decision that throws, and a throwing reporter never fails a connect', async () => {
const d = deps({
managedEnvironmentId: () => 'env-9',
ensureTunnel: async () => {
throw new Error('tunnel failed')
}
})
expect(decided(await eventsOf(d))).toMatchObject({ outcome: 'relay', reason: 'failed' })
const throwing = deps({
report: () => {
throw new Error('telemetry broke')
}
})
await expect(resolveHostServerOnConnect(target, throwing)).resolves.toMatchObject({
route: 'managed'
})
})
})
@@ -30,6 +30,7 @@ function deps(overrides: Partial<HostServerOnConnectDeps> = {}): HostServerOnCon
progress: vi.fn(),
isFencedBeforeStaging: () => false,
releaseUnreachableSetup: vi.fn(async () => undefined),
report: vi.fn(),
...overrides
}
}
@@ -187,7 +188,8 @@ describe('which server an SSH host runs on connect', () => {
})
await expect(resolveHostServerOnConnect(target, d)).resolves.toEqual({
route: 'relay',
reason: 'failed'
reason: 'failed',
detail: 'Connection lost'
})
expect(d.recordUnavailable).not.toHaveBeenCalled()
warn.mockRestore()
+63 -9
View File
@@ -17,6 +17,16 @@ import {
classifyOrcadHostUnavailable,
ORCAD_TUNNEL_UNAVAILABLE_REASON
} from './orcad-host-unavailable'
import {
decidedEvent,
reportHostServerConversion,
reportHostServerDeploy,
reportHostServerEvent,
reportHostServerSetupError,
startHostServerDecisionTrace,
type HostServerDecisionTrace,
type HostServerReport
} from './ssh-host-server-connect-events'
export type HostServerPhase = 'deploying' | 'converting' | 'connecting'
@@ -60,17 +70,38 @@ export type HostServerOnConnectDeps = {
isFencedBeforeStaging: (target: SshTarget) => boolean
/** Unregisters that unreachable server and releases its fence, so the relay serves again. */
releaseUnreachableSetup: (target: SshTarget) => Promise<void>
/** Receives each decision, conversion and deploy failure as raw codes for telemetry. */
report: HostServerReport
}
// Bounds a failure's detail; the host log tail inside it is already capped.
const MAX_FAILURE_DETAIL_CHARS = 16_000
export async function resolveHostServerOnConnect(
target: SshTarget,
deps: HostServerOnConnectDeps
): Promise<HostServerOnConnectResult> {
const trace = startHostServerDecisionTrace()
let result: HostServerOnConnectResult | null = null
try {
result = await decide(target, deps, trace)
return result
} finally {
reportHostServerEvent(deps.report, target, decidedEvent(result, trace))
}
}
async function decide(
target: SshTarget,
deps: HostServerOnConnectDeps,
trace: HostServerDecisionTrace
): Promise<HostServerOnConnectResult> {
if (target.orcadFence?.sourceChangedAt) {
return { route: 'relay', reason: 'source_changed' }
}
const existing = deps.managedEnvironmentId(target)
if (existing) {
trace.path = 'existing'
deps.progress(target, 'connecting')
try {
await deps.ensureTunnel(existing)
@@ -97,6 +128,7 @@ export async function resolveHostServerOnConnect(
}
const recorded = deps.recordedUnavailable(target)
if (recorded) {
trace.recorded = true
return { route: 'relay', reason: 'orcad_unavailable', detail: recorded }
}
if (!deps.hasTemplate()) {
@@ -105,8 +137,11 @@ export async function resolveHostServerOnConnect(
const empty = deps.isEmptyHost(target)
try {
if (empty) {
trace.path = 'deploy'
deps.progress(target, 'deploying')
return await afterDeploy(target, deps, await deps.deploy(target))
const deployed = await deps.deploy(target)
reportHostServerDeploy(deps.report, target, trace, deployed)
return await afterDeploy(target, deps, deployed)
}
const terminals = await deps.relayTerminals(target)
if (terminals.verdict !== 'exited') {
@@ -117,11 +152,22 @@ export async function resolveHostServerOnConnect(
terminals: terminals.count
}
}
trace.path = 'convert'
trace.conversionStartedAt = Date.now()
reportHostServerEvent(deps.report, target, { kind: 'conversion', phase: 'started' })
deps.progress(target, 'converting')
return await afterConversion(target, deps, await deps.convert(target))
const converted = await deps.convert(target)
if (converted.outcome === 'refused') {
trace.refusal = converted.code
}
reportHostServerConversion(deps.report, target, trace, converted)
return await afterConversion(target, deps, converted)
} catch (error) {
console.warn('[ssh] Managed Orca server setup failed; using the relay this session:', error)
return unfinished(target, deps, classifyOrcadHostUnavailable(error), empty, 'failed')
reportHostServerSetupError(deps.report, target, trace, error)
const unavailable = classifyOrcadHostUnavailable(error)
const detail = error instanceof Error ? error.message : String(error)
return unfinished(target, deps, unavailable, empty, 'failed', detail)
}
}
@@ -135,7 +181,7 @@ async function afterDeploy(
result: OrcadManagedDeployResult
): Promise<HostServerOnConnectResult> {
if (result.outcome === 'deferred') {
return deferred(target, deps, result.code, true)
return deferred(target, deps, result, true)
}
return { route: 'managed', environmentId: result.environment.id }
}
@@ -149,7 +195,7 @@ async function afterConversion(
case 'converted':
return { route: 'managed', environmentId: result.environment.id }
case 'deferred':
return deferred(target, deps, result.code, false)
return deferred(target, deps, result, false)
case 'refused':
if (result.code === 'orcad_migration_terminals') {
return { route: 'relay', reason: terminalReason(result.verdict) }
@@ -161,10 +207,12 @@ async function afterConversion(
function deferred(
target: SshTarget,
deps: HostServerOnConnectDeps,
code: string,
result: { code: string; reason: string },
empty: boolean
): Promise<HostServerOnConnectResult> {
return unfinished(target, deps, classifyOrcadHostUnavailable({ code }), empty, 'deferred')
console.warn(`[ssh] Managed Orca server setup deferred (${result.code}):`, result.reason)
const unavailable = classifyOrcadHostUnavailable({ code: result.code })
return unfinished(target, deps, unavailable, empty, 'deferred', result.reason)
}
/**
@@ -176,7 +224,8 @@ async function unfinished(
deps: HostServerOnConnectDeps,
unavailable: string | null,
empty: boolean,
reason: 'deferred' | 'failed'
reason: 'deferred' | 'failed',
failure: string
): Promise<HostServerOnConnectResult> {
try {
await (empty ? deps.abandonDeploy(target) : deps.abandonConversion(target))
@@ -185,8 +234,13 @@ async function unfinished(
console.warn('[ssh] The managed Orca server setup keeps its fence:', error)
}
if (!unavailable) {
return { route: 'relay', reason }
// Shown under the SSH host, with the host's orcad.log tail when the failure carries one.
return { route: 'relay', reason, detail: boundedDetail(failure) }
}
deps.recordUnavailable(target, unavailable)
return { route: 'relay', reason: 'orcad_unavailable', detail: unavailable }
}
function boundedDetail(text: string): string {
return text.length > MAX_FAILURE_DETAIL_CHARS ? `…${text.slice(-MAX_FAILURE_DETAIL_CHARS)}` : text
}
@@ -0,0 +1,184 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { eventSchemas } from '../../shared/telemetry-event-registry'
vi.mock('../telemetry/client', () => ({ track: vi.fn() }))
const { track } = await import('../telemetry/client')
const { hostServerFailure, hostServerRefusal, trackSshHostServerEvent, trackSshHostServerMove } =
await import('./ssh-host-server-telemetry')
const linuxGlibc = { os: 'linux', arch: 'x64', libc: 'glibc' } as const
function tracked() {
const call = vi.mocked(track).mock.calls.at(-1)
if (!call) {
throw new Error('nothing was tracked')
}
return call
}
function expectSchemaValid(): unknown {
const [name, props] = tracked()
expect(eventSchemas[name].safeParse(props).success).toBe(true)
return props
}
describe('ssh host server telemetry', () => {
beforeEach(() => {
vi.mocked(track).mockReset()
})
it('maps a decision onto enum-only props the strict schema accepts', () => {
trackSshHostServerEvent(
{
kind: 'decided',
outcome: 'relay',
reason: 'refused',
refusal: 'orcad_migration_preflight_blocked',
recorded: false,
durationMs: 7_000
},
linuxGlibc
)
expect(tracked()[0]).toBe('ssh_host_server_decided')
expect(expectSchemaValid()).toEqual({
outcome: 'relay',
transport: 'none',
reason: 'refused',
refusal: 'preflight_blocked',
recorded: false,
host_os: 'linux',
host_arch: 'x64',
host_libc: 'glibc',
duration_bucket: '5s_15s'
})
})
it('reports an unprobed host as unknown rather than guessing', () => {
trackSshHostServerEvent(
{
kind: 'decided',
outcome: 'managed',
reason: 'connected',
refusal: null,
recorded: false,
durationMs: 10
},
null
)
expect(expectSchemaValid()).toMatchObject({
host_os: 'unknown',
host_arch: 'unknown',
host_libc: 'unknown'
})
})
it('names the transport a managed connect used, and none for the relay', () => {
const decision = {
kind: 'decided',
outcome: 'managed',
reason: 'connected',
refusal: null,
recorded: false,
durationMs: 10
} as const
trackSshHostServerEvent(decision, linuxGlibc, 'stdio_bridge')
expect(expectSchemaValid()).toMatchObject({ transport: 'stdio_bridge' })
trackSshHostServerEvent(decision, linuxGlibc)
expect(expectSchemaValid()).toMatchObject({ transport: 'unknown' })
trackSshHostServerEvent(
{ ...decision, outcome: 'relay', reason: 'ssh_tunnel_unavailable' },
linuxGlibc,
'tcp_forward'
)
expect(expectSchemaValid()).toMatchObject({
transport: 'none',
reason: 'ssh_tunnel_unavailable'
})
})
it('reports the per-host move offer and each move result', () => {
for (const outcome of ['offered', 'moved', 'refused_live', 'failed'] as const) {
trackSshHostServerMove(outcome, linuxGlibc)
expect(tracked()[0]).toBe('ssh_host_server_move')
expect(expectSchemaValid()).toEqual({
outcome,
host_os: 'linux',
host_arch: 'x64',
host_libc: 'glibc'
})
}
})
it('narrows any unrecognised code, so free text never reaches an event', () => {
expect(hostServerRefusal('orcad_migration_/home/me/secret')).toBe('other')
expect(hostServerFailure('Error: ssh me@host.example failed')).toBe('other')
expect(hostServerFailure('orcad_activation_no_readiness')).toBe('activation_no_readiness')
expect(hostServerFailure('unsupported_host')).toBe('unsupported_host')
trackSshHostServerEvent(
{
kind: 'decided',
outcome: 'relay',
reason: 'host.example.com',
refusal: null,
recorded: false,
durationMs: 0
},
linuxGlibc
)
expect(expectSchemaValid()).toMatchObject({ reason: 'other' })
})
it('maps conversion phases and deploy failures', () => {
trackSshHostServerEvent({ kind: 'conversion', phase: 'started' }, linuxGlibc)
expect(expectSchemaValid()).toMatchObject({ phase: 'started', failure: 'none' })
trackSshHostServerEvent(
{ kind: 'conversion', phase: 'committed', durationMs: 90_000 },
linuxGlibc
)
expect(expectSchemaValid()).toMatchObject({ phase: 'committed', duration_bucket: 'gte_60s' })
trackSshHostServerEvent(
{
kind: 'conversion',
phase: 'failed',
failure: 'refused',
code: 'orcad_migration_terminals',
durationMs: 100
},
linuxGlibc
)
expect(expectSchemaValid()).toMatchObject({ refusal: 'terminals', failure_code: 'none' })
trackSshHostServerEvent(
{
kind: 'deploy_failed',
context: 'deploy',
failure: 'deferred',
code: 'orcad_candidate_preflight_failed',
durationMs: 20_000
},
{ os: 'linux', arch: 'arm64', libc: 'musl' }
)
expect(tracked()[0]).toBe('ssh_host_server_deploy_failed')
expect(expectSchemaValid()).toEqual({
context: 'deploy',
failure: 'deferred',
failure_code: 'candidate_preflight_failed',
host_os: 'linux',
host_arch: 'arm64',
host_libc: 'musl',
duration_bucket: '15s_60s'
})
})
it('never throws into a connect when tracking fails', () => {
vi.mocked(track).mockImplementation(() => {
throw new Error('invalid')
})
trackSshHostServerEvent({ kind: 'conversion', phase: 'started' }, null)
expect(track).toHaveBeenCalledTimes(1)
})
})
+102
View File
@@ -0,0 +1,102 @@
/**
* Connect-decision telemetry: which server each SSH connect chose and why, plus conversion and
* deploy failures. Every raw code is narrowed onto a fixed enum here, so nothing a host or user
* named can reach an event; unrecognised codes become `other`.
*/
import type { EventProps } from '../../shared/telemetry-events'
import {
SSH_HOST_SERVER_FAILURE_VALUES,
SSH_HOST_SERVER_REASON_VALUES,
SSH_HOST_SERVER_REFUSAL_VALUES
} from '../../shared/telemetry-ssh-host-server-event-schemas'
import { track } from '../telemetry/client'
import type { HostServerConnectEvent } from './ssh-host-server-connect-events'
import type { SshHostPlatformFacts } from './ssh-host-platform-memo'
import type { OrcadTunnelTransport } from './orcad-tunnel-transport-memo'
import { durationBucket } from './ssh-remote-runtime-telemetry'
type HostProps = Pick<EventProps<'ssh_host_server_decided'>, 'host_os' | 'host_arch' | 'host_libc'>
type FailureCode = (typeof SSH_HOST_SERVER_FAILURE_VALUES)[number]
type RefusalCode = (typeof SSH_HOST_SERVER_REFUSAL_VALUES)[number]
function pick<T extends string>(values: readonly T[], value: string, fallback: T): T {
return values.find((candidate) => candidate === value) ?? fallback
}
function hostProps(facts: SshHostPlatformFacts | null): HostProps {
return facts
? { host_os: facts.os, host_arch: facts.arch, host_libc: facts.libc }
: { host_os: 'unknown', host_arch: 'unknown', host_libc: 'unknown' }
}
export function hostServerRefusal(code: string | null): RefusalCode {
return code
? pick(SSH_HOST_SERVER_REFUSAL_VALUES, code.replace(/^orcad_migration_/u, ''), 'other')
: 'none'
}
export function hostServerFailure(code: string | null): FailureCode {
return code
? pick(SSH_HOST_SERVER_FAILURE_VALUES, code.replace(/^orcad_/u, ''), 'other')
: 'other'
}
export type SshHostServerMoveOutcome = EventProps<'ssh_host_server_move'>['outcome']
export function trackSshHostServerMove(
outcome: SshHostServerMoveOutcome,
facts: SshHostPlatformFacts | null
): void {
try {
track('ssh_host_server_move', { outcome, ...hostProps(facts) })
} catch {
// Telemetry never costs a move.
}
}
export function trackSshHostServerEvent(
event: HostServerConnectEvent,
facts: SshHostPlatformFacts | null,
transport: OrcadTunnelTransport | null = null
): void {
const host = hostProps(facts)
try {
// track() applies the consent gate and validates against the strict schema.
switch (event.kind) {
case 'decided':
track('ssh_host_server_decided', {
outcome: event.outcome,
transport: event.outcome === 'relay' ? 'none' : (transport ?? 'unknown'),
reason: pick(SSH_HOST_SERVER_REASON_VALUES, event.reason, 'other'),
refusal: hostServerRefusal(event.refusal),
recorded: event.recorded,
...host,
duration_bucket: durationBucket(event.durationMs)
})
return
case 'conversion': {
const failed = event.phase === 'failed'
track('ssh_host_server_conversion', {
phase: event.phase,
failure: failed ? event.failure : 'none',
refusal: failed && event.failure === 'refused' ? hostServerRefusal(event.code) : 'none',
failure_code:
failed && event.failure !== 'refused' ? hostServerFailure(event.code) : 'none',
...host,
duration_bucket: durationBucket(event.phase === 'started' ? 0 : event.durationMs)
})
return
}
case 'deploy_failed':
track('ssh_host_server_deploy_failed', {
context: event.context,
failure: event.failure,
failure_code: hostServerFailure(event.code),
...host,
duration_bucket: durationBucket(event.durationMs)
})
}
} catch {
// Telemetry never costs a connect.
}
}
@@ -23,6 +23,7 @@ import {
type RemoteRuntimeUnavailableReason
} from './ssh-relay-runtime-ladder'
import type { PinnedRuntimeRefusal } from './ssh-relay-runtime-self-test'
import { rememberSshHostPlatform } from './ssh-host-platform-memo'
import type { RemoteHostPlatform } from './ssh-remote-platform'
import type { HostNodeVersion } from './ssh-remote-node-toolchain-probe'
import {
@@ -141,6 +142,7 @@ export class RelayRuntimeLadderRun {
private track(rung: SshRemoteRuntimeRung, outcome: SshRemoteRuntimeOutcome): void {
if (this.host) {
rememberSshHostPlatform(this.targetId, this.host, this.facts?.target ?? null)
trackSshRemoteRuntimeResolved(this.targetId, {
rung,
outcome,
@@ -1,6 +1,7 @@
import { ArrowRightLeft } from 'lucide-react'
import { ArrowRightLeft, ChevronDown } from 'lucide-react'
import { useState } from 'react'
import type { SshTarget } from '../../../../shared/ssh-types'
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible'
import { translate } from '@/i18n/i18n'
import { cn } from '@/lib/utils'
import { canMoveSshHostToManagedServer } from '@/ssh/ssh-managed-server-move'
@@ -47,6 +48,21 @@ export function SshTargetServerStatus({
</Button>
) : null}
</div>
{line.detail ? (
<Collapsible>
<CollapsibleTrigger asChild>
<Button type="button" variant="ghost" size="xs" className="group">
{translate('auto.components.settings.sshHostServer.failureDetails', 'Details')}
<ChevronDown className="transition-transform group-data-[state=open]:rotate-180" />
</Button>
</CollapsibleTrigger>
<CollapsibleContent>
<pre className="scrollbar-sleek max-h-72 overflow-auto whitespace-pre-wrap break-words rounded-md border border-border bg-background px-3 py-2 font-mono text-[11px] leading-relaxed text-muted-foreground">
{line.detail}
</pre>
</CollapsibleContent>
</Collapsible>
) : null}
<SshHostChangedActions target={target} onChanged={onChanged} />
{canMove ? (
<SshManagedServerMoveDialog
@@ -47,6 +47,18 @@ describe('SSH host server status line', () => {
).not.toHaveProperty('action')
})
it('offers the setup failure, log tail included, beside a retry line', () => {
const detail = 'No readiness line.\nLast lines of orcad.log:\nError: EADDRINUSE'
expect(
sshHostServerStatusLine(plain, {
managedServer: { kind: 'relay', reason: 'deferred', detail }
})
).toMatchObject({ tone: 'muted', detail })
expect(
sshHostServerStatusLine(plain, { managedServer: { kind: 'relay', reason: 'failed' } })
).not.toHaveProperty('detail')
})
it('keeps durable reasons visible without a live state', () => {
expect(sshHostServerStatusLine({ orcadFence: { environmentId: 'e' } }, undefined)?.text).toBe(
'Runs a managed Orca server'
@@ -7,6 +7,8 @@ export type SshHostServerStatusLine = {
tone: 'muted' | 'warning' | 'destructive'
/** Offers "Move to managed server", which restarts the live relay terminals. */
action?: 'move'
/** Why setup stopped, with the host's orcad.log tail when there is one; shown on request. */
detail?: string
}
export function sshHostServerStatusLine(
@@ -156,6 +158,6 @@ function relayLine(
}
case 'deferred':
case 'failed':
return retryLine()
return { ...retryLine(), ...(status.detail ? { detail: status.detail } : {}) }
}
}
+2 -1
View File
@@ -12411,7 +12411,8 @@
"terminalsLive": "Runs the relay until its {{count}} open terminals are closed, then moves to a managed server.",
"terminalsUnverifiable": "Runs the relay: Orca couldn’t confirm its terminals are closed. It moves on a later connect.",
"refused": "Not moved to a managed server: {{blocker}}",
"retry": "Runs the relay this session; it moves to a managed server on a later connect."
"retry": "Runs the relay this session; it moves to a managed server on a later connect.",
"failureDetails": "Details"
},
"ZcodePlanAccountsSection": {
"site": {
+10
View File
@@ -97,6 +97,12 @@ import {
workspaceCreateFailedSchema
} from './telemetry-repository-event-schemas'
import { sshRemoteRuntimeResolvedSchema } from './telemetry-ssh-runtime-event-schemas'
import {
sshHostServerConversionSchema,
sshHostServerDecidedSchema,
sshHostServerDeployFailedSchema,
sshHostServerMoveSchema
} from './telemetry-ssh-host-server-event-schemas'
// ── Event registry: the one record the validator consumes ───────────────
// Versioning: breaking changes (rename/re-mean/remove a key) need a new event name; in-place edits blend pre/post rows unmixably. Additive-optional fields are safe.
@@ -199,6 +205,10 @@ export const eventSchemas = {
direct_ssh_reconnect_operation: directSshReconnectOperationSchema,
ssh_remote_runtime_resolved: sshRemoteRuntimeResolvedSchema,
ssh_host_server_decided: sshHostServerDecidedSchema,
ssh_host_server_conversion: sshHostServerConversionSchema,
ssh_host_server_deploy_failed: sshHostServerDeployFailedSchema,
ssh_host_server_move: sshHostServerMoveSchema,
smart_sort_class_distribution: smartSortClassDistributionSchema,
smart_sort_class_1_promotion: smartSortClass1PromotionSchema,
@@ -0,0 +1,142 @@
import { z } from 'zod'
import { SSH_RUNTIME_DURATION_BUCKETS } from './telemetry-ssh-runtime-event-schemas'
// Why enum-only: these rows size how many hosts still need the relay before it can retire. A
// hostname, user, path, log line or raw error would identify a machine, so none may appear.
export const SSH_HOST_SERVER_OUTCOME_VALUES = ['managed', 'deployed', 'converted', 'relay'] as const
/** Why orcad can't run on a host; mirrors main's OrcadHostUnavailableReason. */
const UNAVAILABLE_REASONS = [
// Recorded by builds before the stdio bridge; now retried rather than honoured.
'tcp_forwarding_refused',
// The host refuses port forwarding and can't run the stdio bridge either.
'ssh_tunnel_unavailable',
'unsupported_host',
'artifacts_unavailable',
'libc_unidentified',
'runtime_self_test',
'security_software',
'native_preflight'
] as const
export const SSH_HOST_SERVER_REASON_VALUES = [
'connected',
'deployed',
'converted',
'source_changed',
'relay_terminals_live',
'relay_terminals_unverifiable',
...UNAVAILABLE_REASONS,
'refused',
'deferred',
'failed',
'other'
] as const
/** A conversion refusal's code without its `orcad_migration_` prefix. */
export const SSH_HOST_SERVER_REFUSAL_VALUES = [
'none',
'already_managed',
'terminals',
'target_not_found',
'in_progress',
'direct_ssh_connected',
'preflight_blocked',
'owned_by_deploy',
'fenced_unverifiable',
'stale_journal',
'other'
] as const
/** A deploy's deferral code without its `orcad_` prefix, or why orcad can't run on the host. */
export const SSH_HOST_SERVER_FAILURE_VALUES = [
...UNAVAILABLE_REASONS,
'activation_recovery_required',
'candidate_preflight_failed',
'candidate_launch_failed',
'incumbent_identity_unverifiable',
'outgoing_stop_incomplete',
'activation_build_mismatch',
'activation_daemon_absent',
'activation_daemon_degraded',
'activation_no_health',
'activation_no_persistent_terminals',
'activation_no_readiness',
'activation_not_listening',
'activation_pty_self_test_failed',
'update_daemon_protocol_unverifiable',
'update_strands_live_terminals',
'update_terminal_census_unavailable',
'update_terminals_running',
'windows_launch_refused',
'windows_command_line_unsafe',
'other'
] as const
const hostFields = {
host_os: z.enum(['linux', 'darwin', 'win32', 'unknown']),
host_arch: z.enum(['x64', 'arm64', 'unknown']),
// `unknown` when this session has not yet probed the host; the decision itself never probes.
host_libc: z.enum(['glibc', 'musl', 'none', 'unknown'])
}
/** How the client reached a managed server; `none` when the host stayed on the relay. */
export const SSH_HOST_SERVER_TRANSPORT_VALUES = [
'tcp_forward',
'stdio_bridge',
'none',
'unknown'
] as const
export const SSH_HOST_SERVER_MOVE_OUTCOME_VALUES = [
'offered',
'moved',
'stayed',
'refused_live',
'refused_unverifiable',
'failed'
] as const
export const sshHostServerDecidedSchema = z
.object({
outcome: z.enum(SSH_HOST_SERVER_OUTCOME_VALUES),
transport: z.enum(SSH_HOST_SERVER_TRANSPORT_VALUES),
reason: z.enum(SSH_HOST_SERVER_REASON_VALUES),
refusal: z.enum(SSH_HOST_SERVER_REFUSAL_VALUES),
// True when a recorded "orcad can't run here" sent the host to the relay without a try.
recorded: z.boolean(),
...hostFields,
duration_bucket: z.enum(SSH_RUNTIME_DURATION_BUCKETS)
})
.strict()
export const sshHostServerConversionSchema = z
.object({
phase: z.enum(['started', 'committed', 'failed']),
failure: z.enum(['none', 'deferred', 'refused', 'error']),
refusal: z.enum(SSH_HOST_SERVER_REFUSAL_VALUES),
failure_code: z.enum(['none', ...SSH_HOST_SERVER_FAILURE_VALUES]),
...hostFields,
duration_bucket: z.enum(SSH_RUNTIME_DURATION_BUCKETS)
})
.strict()
export const sshHostServerDeployFailedSchema = z
.object({
// Whether the deploy ran for an empty host or inside a conversion.
context: z.enum(['deploy', 'conversion']),
failure: z.enum(['deferred', 'error']),
failure_code: z.enum(SSH_HOST_SERVER_FAILURE_VALUES),
...hostFields,
duration_bucket: z.enum(SSH_RUNTIME_DURATION_BUCKETS)
})
.strict()
/** The per-host offer to move a host whose open terminals keep it on the relay, and its result. */
export const sshHostServerMoveSchema = z
.object({
outcome: z.enum(SSH_HOST_SERVER_MOVE_OUTCOME_VALUES),
...hostFields
})
.strict()