From 080b9ffdf8e03cf62df118972100bc83584e1d04 Mon Sep 17 00:00:00 2001 From: OrcaWin Date: Sat, 3 Oct 2026 21:16:10 -0700 Subject: [PATCH] feat(ssh): report each connect's server decision, and show orcad.log's last lines when setup fails (#25118) * telemetry: one enum-only event per connect decision (outcome, reason, tunnel transport, host platform, duration), plus conversion start/commit/fail, deploy failures, and the per-host move offer and its result * errors: deploy, launch and rollback failures carry the last 40 redacted lines of the host's orcad.log, read over SSH (Windows through the node.exe host script) * SSH Hosts: a deferred or failed setup offers its reason, log tail included, under Details Co-authored-by: m4air --- src/main/ipc/ssh-host-server-connect.ts | 3 + .../ipc/ssh-host-server-on-connect-wiring.ts | 10 + src/main/ipc/ssh-managed-server-move.test.ts | 8 +- src/main/ipc/ssh-managed-server-move.ts | 29 ++- src/main/ipc/telemetry.ts | 6 +- src/main/ssh/orcad-active-readiness.ts | 11 +- src/main/ssh/orcad-installed-activation.ts | 8 +- .../orcad-managed-tunnel-transport.test.ts | 3 + .../ssh/orcad-managed-tunnel-transport.ts | 9 +- src/main/ssh/orcad-remote-context.ts | 2 + src/main/ssh/orcad-remote-deploy.test.ts | 11 +- src/main/ssh/orcad-remote-log-tail.test.ts | 111 ++++++++++ src/main/ssh/orcad-remote-log-tail.ts | 71 ++++++ src/main/ssh/orcad-remote-rollback.test.ts | 6 + src/main/ssh/orcad-rollback-transition.ts | 8 +- src/main/ssh/orcad-tunnel-transport-memo.ts | 20 ++ src/main/ssh/orcad-windows-host-script.ts | 17 ++ src/main/ssh/ssh-host-platform-memo.test.ts | 24 +++ src/main/ssh/ssh-host-platform-memo.ts | 50 +++++ .../ssh/ssh-host-server-connect-events.ts | 186 ++++++++++++++++ ...h-host-server-on-connect-telemetry.test.ts | 204 ++++++++++++++++++ .../ssh/ssh-host-server-on-connect.test.ts | 4 +- src/main/ssh/ssh-host-server-on-connect.ts | 72 ++++++- .../ssh/ssh-host-server-telemetry.test.ts | 184 ++++++++++++++++ src/main/ssh/ssh-host-server-telemetry.ts | 102 +++++++++ src/main/ssh/ssh-relay-runtime-resolution.ts | 2 + .../settings/SshTargetServerStatus.tsx | 18 +- .../ssh-host-server-status-copy.test.ts | 12 ++ .../settings/ssh-host-server-status-copy.ts | 4 +- src/renderer/src/i18n/locales/en.json | 3 +- src/shared/telemetry-event-registry.ts | 10 + ...telemetry-ssh-host-server-event-schemas.ts | 142 ++++++++++++ 32 files changed, 1324 insertions(+), 26 deletions(-) create mode 100644 src/main/ssh/orcad-remote-log-tail.test.ts create mode 100644 src/main/ssh/orcad-remote-log-tail.ts create mode 100644 src/main/ssh/orcad-tunnel-transport-memo.ts create mode 100644 src/main/ssh/ssh-host-platform-memo.test.ts create mode 100644 src/main/ssh/ssh-host-platform-memo.ts create mode 100644 src/main/ssh/ssh-host-server-connect-events.ts create mode 100644 src/main/ssh/ssh-host-server-on-connect-telemetry.test.ts create mode 100644 src/main/ssh/ssh-host-server-telemetry.test.ts create mode 100644 src/main/ssh/ssh-host-server-telemetry.ts create mode 100644 src/shared/telemetry-ssh-host-server-event-schemas.ts diff --git a/src/main/ipc/ssh-host-server-connect.ts b/src/main/ipc/ssh-host-server-connect.ts index 2122c47197d..e07b5b22576 100644 --- a/src/main/ipc/ssh-host-server-connect.ts +++ b/src/main/ipc/ssh-host-server-connect.ts @@ -4,6 +4,8 @@ import type { SshConnectionState, SshTarget } from '../../shared/ssh-types' import type { HostServerOnConnectResult } from '../ssh/ssh-host-server-on-connect' import { relayServerStatus, shouldToastManagedServerMove } from '../ssh/ssh-host-server-move-offer' import { setSshHostServerStatus } from '../ssh/ssh-host-server-status' +import { trackSshHostServerMove } from '../ssh/ssh-host-server-telemetry' +import { knownSshHostPlatform } from '../ssh/ssh-host-platform-memo' import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry' import { connectionManager, getCurrentMainWindow } from './ssh-ipc-context' import { broadcastSshState, getPublicSshState } from './ssh-renderer-broadcast' @@ -57,6 +59,7 @@ export function recordRelayDecision( getSshTargetRegistryStore()!.updateTarget(target.id, { managedServerMoveOffered: { appVersion } }) + trackSshHostServerMove('offered', knownSshHostPlatform(target.id)) } setSshHostServerStatus(target.id, relayServerStatus(decision, offerMove)) } diff --git a/src/main/ipc/ssh-host-server-on-connect-wiring.ts b/src/main/ipc/ssh-host-server-on-connect-wiring.ts index 590d506bf79..dd88a5b71fa 100644 --- a/src/main/ipc/ssh-host-server-on-connect-wiring.ts +++ b/src/main/ipc/ssh-host-server-on-connect-wiring.ts @@ -15,6 +15,9 @@ import { orcadMigrationDestinationFor } from '../ssh/orcad-runtime-conversion-wi import { createManagedOrcadEnvironment } from '../ssh/orcad-runtime-deployment' import type { HostServerOnConnectDeps } from '../ssh/ssh-host-server-on-connect' import { setSshHostServerStatus } from '../ssh/ssh-host-server-status' +import { trackSshHostServerEvent } from '../ssh/ssh-host-server-telemetry' +import { knownSshHostPlatform } from '../ssh/ssh-host-platform-memo' +import { knownOrcadTunnelTransport } from '../ssh/orcad-tunnel-transport-memo' import { getSshTargetRegistryStore, hasRegisteredDirectSshAuthority @@ -117,6 +120,13 @@ export function hostServerOnConnectDeps(userDataPath: string): HostServerOnConne findOrcadMigrationSourceCutoverForTarget(userDataPath, target.id)?.phase === 'source-fenced', releaseUnreachableSetup: (target) => releaseUnreachableOrcadSetup({ userDataPath, claims, targetId: target.id }), + // Read at report time: a deploy or conversion learns the platform while the decision runs. + report: (target, event) => + trackSshHostServerEvent( + event, + knownSshHostPlatform(target.id), + knownOrcadTunnelTransport(target.id) + ), progress: (target, phase) => { setSshHostServerStatus(target.id, { kind: 'setting-up', phase }) broadcastSshState(getCurrentMainWindow, target.id, { diff --git a/src/main/ipc/ssh-managed-server-move.test.ts b/src/main/ipc/ssh-managed-server-move.test.ts index 3986ffb116e..7867887e38d 100644 --- a/src/main/ipc/ssh-managed-server-move.test.ts +++ b/src/main/ipc/ssh-managed-server-move.test.ts @@ -38,7 +38,8 @@ function deps( calls.push('connect') status = options.afterConnect ?? { kind: 'managed', environmentId: 'env-1' } }), - serverStatus: vi.fn(() => status) + serverStatus: vi.fn(() => status), + report: vi.fn() } } @@ -51,6 +52,7 @@ describe('moving an SSH host to its managed server on request', () => { }) expect(move.calls).toEqual(['terminate', 'census', 'connect']) expect(move.relayTerminals).toHaveBeenCalledWith(target) + expect(move.report).toHaveBeenCalledWith('ssh-1', 'moved') }) it('refuses without converting when the stop could not reach every terminal', async () => { @@ -61,6 +63,7 @@ describe('moving an SSH host to its managed server on request', () => { terminals: 1 }) expect(move.calls).toEqual(['terminate']) + expect(move.report).toHaveBeenCalledWith('ssh-1', 'refused_unverifiable') }) it('refuses without converting when the census is unverifiable or still live', async () => { @@ -78,11 +81,13 @@ describe('moving an SSH host to its managed server on request', () => { verdict: 'live' }) expect(live.connect).not.toHaveBeenCalled() + expect(live.report).toHaveBeenCalledWith('ssh-1', 'refused_live') }) it('reports a connect that kept the relay for another reason', async () => { const move = deps({ afterConnect: { kind: 'relay', reason: 'refused', detail: 'blocked' } }) await expect(moveSshHostToManagedServer('ssh-1', move)).resolves.toEqual({ outcome: 'stayed' }) + expect(move.report).toHaveBeenCalledWith('ssh-1', 'stayed') }) it('reattaches the relay first when preserved terminals need one to be stopped', async () => { @@ -101,5 +106,6 @@ describe('moving an SSH host to its managed server on request', () => { move.terminate.mockRejectedValueOnce(new Error('Failed to terminate SSH host sessions')) await expect(moveSshHostToManagedServer('ssh-1', move)).rejects.toThrow('Failed to terminate') expect(move.connect).not.toHaveBeenCalled() + expect(move.report).toHaveBeenCalledWith('ssh-1', 'failed') }) }) diff --git a/src/main/ipc/ssh-managed-server-move.ts b/src/main/ipc/ssh-managed-server-move.ts index 953e4145b7a..9494da5c069 100644 --- a/src/main/ipc/ssh-managed-server-move.ts +++ b/src/main/ipc/ssh-managed-server-move.ts @@ -12,6 +12,11 @@ import type { } from '../../shared/ssh-types' import type { HostServerTerminalVerdict } from '../ssh/ssh-host-server-on-connect' import { getSshHostServerStatus } from '../ssh/ssh-host-server-status' +import { + trackSshHostServerMove, + type SshHostServerMoveOutcome +} from '../ssh/ssh-host-server-telemetry' +import { knownSshHostPlatform } from '../ssh/ssh-host-platform-memo' import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry' import { connectTarget } from './ssh-connect-flow' import { terminateSshTargetSessions } from './ssh-terminate-sessions' @@ -22,11 +27,32 @@ export type SshManagedServerMoveDeps = { relayTerminals: (target: SshTarget) => Promise connect: (targetId: string) => Promise serverStatus: (targetId: string) => SshManagedServerStatus | undefined + report: (targetId: string, outcome: SshHostServerMoveOutcome) => void } export async function moveSshHostToManagedServer( targetId: string, deps: SshManagedServerMoveDeps = defaultMoveDeps() +): Promise { + let result: SshManagedServerMoveResult | null = null + try { + result = await moveHost(targetId, deps) + return result + } finally { + deps.report(targetId, moveOutcome(result)) + } +} + +function moveOutcome(result: SshManagedServerMoveResult | null): SshHostServerMoveOutcome { + if (!result) { + return 'failed' + } + return result.outcome === 'refused' ? `refused_${result.verdict}` : result.outcome +} + +async function moveHost( + targetId: string, + deps: SshManagedServerMoveDeps ): Promise { const target = deps.getTarget(targetId) if (!target) { @@ -74,6 +100,7 @@ function defaultMoveDeps(): SshManagedServerMoveDeps { return hostServerOnConnectDeps(getAppEnvironment().getPath('userData')).relayTerminals(target) }, connect: connectTarget, - serverStatus: getSshHostServerStatus + serverStatus: getSshHostServerStatus, + report: (targetId, outcome) => trackSshHostServerMove(outcome, knownSshHostPlatform(targetId)) } } diff --git a/src/main/ipc/telemetry.ts b/src/main/ipc/telemetry.ts index 83cc6579591..164a1ef2ac9 100644 --- a/src/main/ipc/telemetry.ts +++ b/src/main/ipc/telemetry.ts @@ -31,7 +31,11 @@ const MAIN_OWNED_TELEMETRY_EVENTS = new Set([ 'daemon_pty_cwd_readable', 'star_nag_outcome', 'feature_interaction_usage_bucket_reached', - 'ssh_remote_runtime_resolved' + 'ssh_remote_runtime_resolved', + 'ssh_host_server_decided', + 'ssh_host_server_conversion', + 'ssh_host_server_deploy_failed', + 'ssh_host_server_move' ]) /** diff --git a/src/main/ssh/orcad-active-readiness.ts b/src/main/ssh/orcad-active-readiness.ts index 73e228c29b1..fc5858c85a8 100644 --- a/src/main/ssh/orcad-active-readiness.ts +++ b/src/main/ssh/orcad-active-readiness.ts @@ -16,6 +16,7 @@ import { readOrcadReadinessNowCommand } from './orcad-remote-readiness-wait' import type { ServeReadiness } from '../server/serve-readiness' +import { withOrcadLogTail } from './orcad-remote-log-tail' /** `exited` is proven absence; `unverifiable` means the host could not say, which is not death. */ export type OrcadReadinessFailureVerdict = 'exited' | 'unverifiable' | 'rejected' @@ -116,5 +117,13 @@ export async function launchOrcadSlotAndAwaitReadiness( expectation: OrcadActivationExpectation ): Promise { const parsed = await launchOrcadAndAwaitReadiness(target, spec) - return gatedReadiness(parsed, expectation, `orcad ${spec.fullVersion}`) + try { + return gatedReadiness(parsed, expectation, `orcad ${spec.fullVersion}`) + } catch (error) { + if (!(error instanceof OrcadActiveReadinessError)) { + throw error + } + const message = await withOrcadLogTail(target, spec.remoteInstallDir, error.message) + throw new OrcadActiveReadinessError(error.verdict, message) + } } diff --git a/src/main/ssh/orcad-installed-activation.ts b/src/main/ssh/orcad-installed-activation.ts index 429b5d9519f..5e0deaeccd0 100644 --- a/src/main/ssh/orcad-installed-activation.ts +++ b/src/main/ssh/orcad-installed-activation.ts @@ -57,6 +57,7 @@ import { type OrcadSlotIdentity } from './orcad-recovery-slot' import { orcadSnapshotPath, recoverOrcadIncumbent } from './orcad-incumbent-recovery' +import { withOrcadLogTail } from './orcad-remote-log-tail' type Outcome = Extract @@ -242,11 +243,10 @@ export async function activateInstalledOrcad( `The candidate failed while starting: ${errorMessage(launchError)}` ] const restored = await restoreAfterRejectedCandidate(options, transaction, incumbent, lock) - return notActivated( - code, + const located = `${reason} Candidate stderr is at ` + - `${joinRemotePath(options.host, remoteDir, ORCAD_LOG_FILENAME)}. ${restored}` - ) + `${joinRemotePath(options.host, remoteDir, ORCAD_LOG_FILENAME)}. ${restored}` + return notActivated(code, await withOrcadLogTail(options, remoteDir, located)) } const recordAfter = withActivatedVersion(record, fullVersion, snapshot, now()) diff --git a/src/main/ssh/orcad-managed-tunnel-transport.test.ts b/src/main/ssh/orcad-managed-tunnel-transport.test.ts index 5b702c0b730..64650eb4660 100644 --- a/src/main/ssh/orcad-managed-tunnel-transport.test.ts +++ b/src/main/ssh/orcad-managed-tunnel-transport.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from 'vitest' import { OrcadManagedTunnelTransportProvider } from './orcad-managed-tunnel-transport' +import { knownOrcadTunnelTransport } from './orcad-tunnel-transport-memo' import type { SshConnection } from './ssh-connection' import type { PortForwardStartOptions, @@ -49,6 +50,7 @@ describe('the managed tunnel’s transport', () => { }) expect(refused.stdio.start).toHaveBeenCalledWith(conn, options) expect(refused.forward.start).not.toHaveBeenCalled() + expect(knownOrcadTunnelTransport('ssh-1')).toBe('stdio_bridge') }) it('keeps the forward when forwarding is allowed or the answer is unverifiable', async () => { @@ -58,6 +60,7 @@ describe('the managed tunnel’s transport', () => { entry: { id: 'forward' } }) expect(chosen.stdio.start).not.toHaveBeenCalled() + expect(knownOrcadTunnelTransport('ssh-1')).toBe('tcp_forward') } }) }) diff --git a/src/main/ssh/orcad-managed-tunnel-transport.ts b/src/main/ssh/orcad-managed-tunnel-transport.ts index 136834f693c..b8f02b495bc 100644 --- a/src/main/ssh/orcad-managed-tunnel-transport.ts +++ b/src/main/ssh/orcad-managed-tunnel-transport.ts @@ -3,6 +3,7 @@ * wherever the host allows it, and the stdio bridge where its sshd refuses forwarding. */ import { OrcadStdioBridgePortForwardProvider } from './orcad-stdio-bridge-provider' +import { rememberOrcadTunnelTransport } from './orcad-tunnel-transport-memo' import type { SshConnection } from './ssh-connection' import type { PortForwardStartOptions, @@ -36,12 +37,16 @@ export class OrcadManagedTunnelTransportProvider implements SshPortForwardProvid const { forwards, stdio, probe } = this.dependencies // Why only on refusal: an unverifiable answer keeps the forward, as before this transport. if ((await probe(conn, options.remotePort)) === 'refused' && stdio.canHandle(conn)) { - return stdio.start(conn, options) + const started = await stdio.start(conn, options) + rememberOrcadTunnelTransport(options.connectionId, 'stdio_bridge') + return started } const forward = forwards.find((provider) => provider.canHandle(conn)) if (!forward) { throw new Error('SSH connection is not established') } - return forward.start(conn, options) + const started = await forward.start(conn, options) + rememberOrcadTunnelTransport(options.connectionId, 'tcp_forward') + return started } } diff --git a/src/main/ssh/orcad-remote-context.ts b/src/main/ssh/orcad-remote-context.ts index 98b791ef921..cb0663ce1ea 100644 --- a/src/main/ssh/orcad-remote-context.ts +++ b/src/main/ssh/orcad-remote-context.ts @@ -17,6 +17,7 @@ import { } from './ssh-remote-platform' import { detectRemoteHostPlatform } from './ssh-remote-platform-detection' import { OrcadHostUnsupportedError } from './orcad-host-unavailable' +import { rememberSshHostPlatform } from './ssh-host-platform-memo' export type OrcadRemoteContext = { activationRecord: OrcadActivationRecord @@ -47,6 +48,7 @@ export async function resolveOrcadRemoteContext( throw new Error(`Remote home is not a valid path: ${remoteHome.slice(0, 100)}`) } const serverTarget = await resolveOrcadDeploymentTarget({ conn: connection, host, signal }) + rememberSshHostPlatform(target.id, host, serverTarget) if (isWindowsRemoteHost(host)) { // Every Windows host op, the activation record read included, runs on the pinned node.exe. await prepareWindowsOrcadHost({ conn: connection, host, remoteHome, serverTarget, signal }) diff --git a/src/main/ssh/orcad-remote-deploy.test.ts b/src/main/ssh/orcad-remote-deploy.test.ts index 5d1a52647fe..606fa14a7df 100644 --- a/src/main/ssh/orcad-remote-deploy.test.ts +++ b/src/main/ssh/orcad-remote-deploy.test.ts @@ -109,11 +109,15 @@ type HostScript = { comparisonResult?: string candidateStopResult?: string readinessAtMs?: number + orcadLog?: string } function scriptHost(script: HostScript): void { mockExec.mockImplementation(async (_conn, command: string) => { const text = String(command) + if (text.startsWith('tail -c')) { + return script.orcadLog ?? '' + } if (text.includes('__ORCAD_RECORD_PRESENT__') && text.includes('orcad-active.json')) { return script.activationRecord ? `__ORCAD_RECORD_PRESENT__\n${script.activationRecord}` @@ -504,11 +508,16 @@ describe('deployOrcad', () => { const script: HostScript = { activationRecord: ACTIVE_OLD, readiness: { [NEW_VERSION]: readyLine({ daemonState: 'degraded' }) }, - log: [] + log: [], + orcadLog: 'daemon: starting\ndaemon: socket bind failed: EACCES\n' } scriptHost(script) const result = await deployOrcad(options()) expect(result).toMatchObject({ code: 'orcad_activation_daemon_degraded' }) + // The error carries why the candidate failed, not only where its log is. + expect(result).toMatchObject({ + reason: expect.stringMatching(/Last lines of orcad\.log:\ndaemon: starting\n.*EACCES$/u) + }) expect( vi .mocked(writeAtomicOrcadRemoteRecord) diff --git a/src/main/ssh/orcad-remote-log-tail.test.ts b/src/main/ssh/orcad-remote-log-tail.test.ts new file mode 100644 index 00000000000..61b8fa3f4ac --- /dev/null +++ b/src/main/ssh/orcad-remote-log-tail.test.ts @@ -0,0 +1,111 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { runProcess } from '../../shared/child-process/run-process' +import { + ORCAD_LOG_TAIL_LINES, + ORCAD_LOG_TAIL_MAX_BYTES, + orcadLogTailCommand, + parseOrcadLogTail, + withOrcadLogTail +} from './orcad-remote-log-tail' +import { ORCAD_WINDOWS_HOST_SCRIPT } from './orcad-windows-host-script' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const execCommand = vi.hoisted(() => vi.fn()) +vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand })) + +const linux = getRemoteHostPlatform('linux-x64') +const windows = getRemoteHostPlatform('win32-x64') +const SLOT = '/home/me/.orca-remote/orcad-1.0.0' +const WINDOWS_SLOT = 'C:\\Users\\me\\.orca-remote\\orcad-1.0.0' +// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand is mocked; the connection is never used. +const conn = {} as never + +let dir = '' +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'orcad-log-tail-')) + execCommand.mockReset() +}) +afterEach(() => rmSync(dir, { recursive: true, force: true })) + +function numberedLines(count: number): string { + return Array.from({ length: count }, (_, index) => `line ${index + 1}`).join('\n') +} + +describe('orcad.log tail', () => { + it('keeps only the last lines and bytes, redacting secrets', () => { + const tail = parseOrcadLogTail(linux, `${numberedLines(100)}\ntoken=abc123secret\n`) + const lines = (tail ?? '').split('\n') + expect(lines).toHaveLength(ORCAD_LOG_TAIL_LINES) + expect(lines[0]).toBe('line 62') + expect(tail).not.toContain('abc123secret') + expect(tail).toContain('[redacted') + + const huge = parseOrcadLogTail(linux, 'x'.repeat(ORCAD_LOG_TAIL_MAX_BYTES * 4)) + expect(Buffer.byteLength(huge ?? '', 'utf8')).toBeLessThanOrEqual(ORCAD_LOG_TAIL_MAX_BYTES) + expect(parseOrcadLogTail(linux, ' \n')).toBeNull() + }) + + it('reads the log on Windows through the node.exe host script, never an encoded command', () => { + const command = orcadLogTailCommand(windows, WINDOWS_SLOT) + expect(command).toContain('node.exe') + expect(command).toContain(' log-tail ') + expect(command).not.toMatch(/EncodedCommand/iu) + }) + + it.skipIf(process.platform === 'win32')( + 'runs the POSIX command against a real file', + async () => { + const slot = join(dir, 'slot') + mkdirSync(slot) + writeFileSync(join(slot, 'orcad.log'), `${numberedLines(60)}\n`) + const run = await runProcess({ + program: '/bin/sh', + args: ['-c', orcadLogTailCommand(linux, slot)], + timeoutMs: 5_000 + }) + expect(parseOrcadLogTail(linux, run.stdout)?.split('\n').at(-1)).toBe('line 60') + const missing = await runProcess({ + program: '/bin/sh', + args: ['-c', orcadLogTailCommand(linux, join(dir, 'absent'))], + timeoutMs: 5_000 + }) + expect(parseOrcadLogTail(linux, missing.stdout)).toBeNull() + } + ) + + it('reads the end of the log through the real Windows host script op', async () => { + const script = join(dir, 'host-script.js') + writeFileSync(script, ORCAD_WINDOWS_HOST_SCRIPT) + const log = join(dir, 'orcad.log') + writeFileSync(log, `${'y'.repeat(ORCAD_LOG_TAIL_MAX_BYTES * 2)}\nfatal: bind failed\n`) + const run = await runProcess({ + program: process.execPath, + args: [script, 'log-tail', log, String(ORCAD_LOG_TAIL_MAX_BYTES)], + timeoutMs: 15_000 + }) + const tail = parseOrcadLogTail(windows, run.stdout) + expect(tail?.endsWith('fatal: bind failed')).toBe(true) + const absent = await runProcess({ + program: process.execPath, + args: [script, 'log-tail', join(dir, 'absent.log'), '100'], + timeoutMs: 15_000 + }) + expect(parseOrcadLogTail(windows, absent.stdout)).toBeNull() + }) + + it('appends the tail to a failure message, and leaves it alone when the host cannot answer', async () => { + execCommand.mockResolvedValueOnce('starting\nError: EADDRINUSE\n') + await expect(withOrcadLogTail({ conn, host: linux }, SLOT, 'Launch failed.')).resolves.toBe( + 'Launch failed.\nLast lines of orcad.log:\nstarting\nError: EADDRINUSE' + ) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + execCommand.mockRejectedValueOnce(new Error('Connection lost')) + await expect(withOrcadLogTail({ conn, host: linux }, SLOT, 'Launch failed.')).resolves.toBe( + 'Launch failed.' + ) + warn.mockRestore() + }) +}) diff --git a/src/main/ssh/orcad-remote-log-tail.ts b/src/main/ssh/orcad-remote-log-tail.ts new file mode 100644 index 00000000000..d588e0d35db --- /dev/null +++ b/src/main/ssh/orcad-remote-log-tail.ts @@ -0,0 +1,71 @@ +/** + * The end of a slot's orcad.log, appended to a failed deploy, launch or rollback so the error + * says why orcad stopped rather than only where its log is. Errors and the main log only; never + * telemetry. Best-effort: a host that can't answer leaves the error as it was. + */ +import { redactString } from '../observability/redactor' +import { shellEscape } from './ssh-connection-utils' +import { ORCAD_LOG_FILENAME } from './orcad-remote-host-support' +import { execCommand } from './ssh-relay-deploy-helpers' +import { isWindowsRemoteHost, joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { + orcadWindowsBaseDir, + orcadWindowsHostOpCommand, + readOrcadWindowsEncodedAnswer +} from './orcad-remote-windows-node' +import { ORCAD_WINDOWS_LOG_TAIL_MARKER } from './orcad-windows-host-script' +import type { OrcadRemoteExecTarget } from './orcad-remote-runtime-control' + +export const ORCAD_LOG_TAIL_LINES = 40 +export const ORCAD_LOG_TAIL_MAX_BYTES = 8 * 1024 +// Short, because it only decorates an error the caller already has. +const ORCAD_LOG_TAIL_TIMEOUT_MS = 10_000 + +export function orcadLogTailCommand(host: RemoteHostPlatform, slotDir: string): string { + const log = joinRemotePath(host, slotDir, ORCAD_LOG_FILENAME) + if (isWindowsRemoteHost(host)) { + return orcadWindowsHostOpCommand(host, orcadWindowsBaseDir(host, slotDir), 'log-tail', [ + log, + String(ORCAD_LOG_TAIL_MAX_BYTES) + ]) + } + return `tail -c ${ORCAD_LOG_TAIL_MAX_BYTES} ${shellEscape(log)} 2>/dev/null || true` +} + +/** The last lines of what the host printed, redacted; null when the log is empty or absent. */ +export function parseOrcadLogTail(host: RemoteHostPlatform, output: string): string | null { + const raw = isWindowsRemoteHost(host) + ? (readOrcadWindowsEncodedAnswer(output, ORCAD_WINDOWS_LOG_TAIL_MARKER) ?? '') + : output + const bounded = Buffer.from(raw, 'utf8').subarray(-ORCAD_LOG_TAIL_MAX_BYTES).toString('utf8') + const lines = bounded.replace(/\s+$/u, '').split(/\r?\n/u).slice(-ORCAD_LOG_TAIL_LINES) + const tail = redactString(lines.join('\n')) + return tail.trim() ? tail : null +} + +export async function readOrcadLogTail( + target: OrcadRemoteExecTarget, + slotDir: string +): Promise { + try { + // No abort signal: the failure being reported may be the cancellation itself. + const output = await execCommand(target.conn, orcadLogTailCommand(target.host, slotDir), { + wrapCommand: target.host.commandDialect !== 'powershell', + timeoutMs: ORCAD_LOG_TAIL_TIMEOUT_MS + }) + return parseOrcadLogTail(target.host, output) + } catch (error) { + console.warn('[ssh] Could not read the orcad.log tail:', error) + return null + } +} + +/** `message` followed by the log tail, or `message` alone when the host had none to give. */ +export async function withOrcadLogTail( + target: OrcadRemoteExecTarget, + slotDir: string, + message: string +): Promise { + const tail = await readOrcadLogTail(target, slotDir) + return tail ? `${message}\nLast lines of orcad.log:\n${tail}` : message +} diff --git a/src/main/ssh/orcad-remote-rollback.test.ts b/src/main/ssh/orcad-remote-rollback.test.ts index 57a926a16fe..35b939a45d8 100644 --- a/src/main/ssh/orcad-remote-rollback.test.ts +++ b/src/main/ssh/orcad-remote-rollback.test.ts @@ -91,6 +91,9 @@ function scriptHost(log: string[], overrides: HostOverrides = {}): void { const restores = [...(overrides.restores ?? [])] mockExec.mockImplementation(async (_conn, command: string) => { const text = String(command) + if (text.startsWith('tail -c')) { + return text.includes(TARGET) ? 'orcad: listen EADDRINUSE 127.0.0.1\n' : '' + } if (text.includes('__ORCAD_RECORD_PRESENT__')) { return text.includes('transaction.json') ? '__ORCAD_RECORD_ABSENT__\n' @@ -249,6 +252,9 @@ describe('rollbackOrcad', () => { const result = await rollbackOrcad(options()) expect(result).toMatchObject({ outcome: 'failed', code: 'orcad_activation_no_readiness' }) expect(result.outcome === 'failed' && result.reason).toContain('fresh host terminal census') + expect(result.outcome === 'failed' && result.reason).toContain( + 'Last lines of orcad.log:\norcad: listen EADDRINUSE' + ) expect(log).toEqual([ `stop:${ACTIVE}`, 'rescue', diff --git a/src/main/ssh/orcad-rollback-transition.ts b/src/main/ssh/orcad-rollback-transition.ts index 96e08cfb2f1..a3d653affe5 100644 --- a/src/main/ssh/orcad-rollback-transition.ts +++ b/src/main/ssh/orcad-rollback-transition.ts @@ -42,6 +42,7 @@ import { type OrcadSlotIdentity } from './orcad-recovery-slot' import { orcadSnapshotPath, recoverOrcadIncumbent } from './orcad-incumbent-recovery' +import { withOrcadLogTail } from './orcad-remote-log-tail' /** A confirmed failure reads as no answer; an unconfirmed one propagates and keeps the fence. */ async function execOrEmpty(options: OrcadRollbackOptions, command: string): Promise { @@ -235,9 +236,12 @@ export async function rollbackOrcadLocked( return { outcome: 'failed', code: launchError === undefined ? verdict.code : 'orcad_rollback_target_launch_failed', - reason: + reason: await withOrcadLogTail( + options, + targetDir, `The rollback target ${safety.target} did not come up healthy: ${reason} ${recovered} ` + - `Its stderr is at ${joinRemotePath(options.host, targetDir, ORCAD_LOG_FILENAME)}.` + `Its stderr is at ${joinRemotePath(options.host, targetDir, ORCAD_LOG_FILENAME)}.` + ) } } diff --git a/src/main/ssh/orcad-tunnel-transport-memo.ts b/src/main/ssh/orcad-tunnel-transport-memo.ts new file mode 100644 index 00000000000..529146c9f59 --- /dev/null +++ b/src/main/ssh/orcad-tunnel-transport-memo.ts @@ -0,0 +1,20 @@ +/** Which transport each host's managed tunnel last started on, for connect telemetry. */ +export type OrcadTunnelTransport = 'tcp_forward' | 'stdio_bridge' + +// Keyed by the local target id, which never leaves this process. +const transports = new Map() + +export function rememberOrcadTunnelTransport( + targetId: string, + transport: OrcadTunnelTransport +): void { + transports.set(targetId, transport) +} + +export function knownOrcadTunnelTransport(targetId: string): OrcadTunnelTransport | null { + return transports.get(targetId) ?? null +} + +export function resetOrcadTunnelTransportMemoForTests(): void { + transports.clear() +} diff --git a/src/main/ssh/orcad-windows-host-script.ts b/src/main/ssh/orcad-windows-host-script.ts index 7c232daf4cd..4332c85cd44 100644 --- a/src/main/ssh/orcad-windows-host-script.ts +++ b/src/main/ssh/orcad-windows-host-script.ts @@ -40,6 +40,7 @@ export const ORCAD_BUILD_HASH_MARKER = '__ORCAD_BUILD_HASH__' export const ORCAD_WINDOWS_READINESS_MARKER = '__ORCAD_READINESS__' export const ORCAD_WINDOWS_RUNTIME_MARKER = '__ORCAD_RUNTIME__' export const ORCAD_WINDOWS_LIVENESS_MANY_MARKER = '__ORCAD_LIVENESS__' +export const ORCAD_WINDOWS_LOG_TAIL_MARKER = '__ORCAD_LOG_TAIL__' /** A slot whose marker names no usable runtime; the POSIX selector exits the same way. */ export const ORCAD_WINDOWS_RUNTIME_MISSING_EXIT = 78 /** The record exists but cannot be read within its bound. */ @@ -50,6 +51,7 @@ export type OrcadWindowsHostOp = | 'liveness-many' | 'stop' | 'readiness-wait' + | 'log-tail' | 'build-hash' | 'record-read' | 'record-publish' @@ -163,6 +165,21 @@ const ops = { tick() }, + // The last bytes of a log, as POSIX \`tail -c\` reads them; a missing log is empty. + 'log-tail'(file, capArg) { + const cap = Number(capArg) + let fd + try { fd = fs.openSync(file, 'r') } catch { return encoded(${text(ORCAD_WINDOWS_LOG_TAIL_MARKER)}, Buffer.alloc(0)) } + let buffer = Buffer.alloc(0) + try { + const size = fs.fstatSync(fd).size + const length = Math.min(cap, size) + buffer = Buffer.alloc(length) + buffer = buffer.subarray(0, fs.readSync(fd, buffer, 0, length, size - length)) + } catch { buffer = Buffer.alloc(0) } finally { fs.closeSync(fd) } + encoded(${text(ORCAD_WINDOWS_LOG_TAIL_MARKER)}, buffer) + }, + 'build-hash'(file) { const digest = crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex') answer(${text(`${ORCAD_BUILD_HASH_MARKER} `)} + digest.slice(0, 16) + '\\n') diff --git a/src/main/ssh/ssh-host-platform-memo.test.ts b/src/main/ssh/ssh-host-platform-memo.test.ts new file mode 100644 index 00000000000..4249cb1144d --- /dev/null +++ b/src/main/ssh/ssh-host-platform-memo.test.ts @@ -0,0 +1,24 @@ +import { beforeEach, describe, expect, it } from 'vitest' +import { + knownSshHostPlatform, + rememberSshHostPlatform, + resetSshHostPlatformMemoForTests +} from './ssh-host-platform-memo' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +describe('ssh host platform memo', () => { + beforeEach(() => { + resetSshHostPlatformMemoForTests() + }) + + it('records the libc flavor, and keeps a probed one when a later report has none', () => { + expect(knownSshHostPlatform('ssh-1')).toBeNull() + rememberSshHostPlatform('ssh-1', getRemoteHostPlatform('linux-arm64'), 'linux-arm64-musl') + expect(knownSshHostPlatform('ssh-1')).toEqual({ os: 'linux', arch: 'arm64', libc: 'musl' }) + rememberSshHostPlatform('ssh-1', getRemoteHostPlatform('linux-arm64'), null) + expect(knownSshHostPlatform('ssh-1')?.libc).toBe('musl') + + rememberSshHostPlatform('ssh-2', getRemoteHostPlatform('win32-x64'), 'win32-x64') + expect(knownSshHostPlatform('ssh-2')).toEqual({ os: 'win32', arch: 'x64', libc: 'none' }) + }) +}) diff --git a/src/main/ssh/ssh-host-platform-memo.ts b/src/main/ssh/ssh-host-platform-memo.ts new file mode 100644 index 00000000000..eaedbc647df --- /dev/null +++ b/src/main/ssh/ssh-host-platform-memo.ts @@ -0,0 +1,50 @@ +/** + * The platform each SSH host last reported this app session, so connect telemetry can name it + * without a probe of its own. Keyed by the local target id, which never leaves this process. + */ +import type { ServerTarget } from '../../shared/node-runtime-pin' +import type { RemoteHostPlatform } from './ssh-remote-platform' + +export type SshHostPlatformFacts = { + os: RemoteHostPlatform['os'] + arch: RemoteHostPlatform['arch'] + libc: 'glibc' | 'musl' | 'none' | 'unknown' +} + +const known = new Map() + +function libcOf( + host: RemoteHostPlatform, + target: ServerTarget | null +): SshHostPlatformFacts['libc'] { + if (host.os !== 'linux') { + return 'none' + } + if (!target) { + return 'unknown' + } + return target.endsWith('-musl') ? 'musl' : 'glibc' +} + +export function rememberSshHostPlatform( + targetId: string, + host: RemoteHostPlatform, + target: ServerTarget | null +): void { + const libc = libcOf(host, target) + const previous = known.get(targetId) + // A libc probe that did not run must not erase one that did. + known.set(targetId, { + os: host.os, + arch: host.arch, + libc: libc === 'unknown' && previous ? previous.libc : libc + }) +} + +export function knownSshHostPlatform(targetId: string): SshHostPlatformFacts | null { + return known.get(targetId) ?? null +} + +export function resetSshHostPlatformMemoForTests(): void { + known.clear() +} diff --git a/src/main/ssh/ssh-host-server-connect-events.ts b/src/main/ssh/ssh-host-server-connect-events.ts new file mode 100644 index 00000000000..8ca45e87199 --- /dev/null +++ b/src/main/ssh/ssh-host-server-connect-events.ts @@ -0,0 +1,186 @@ +/** + * What one connect's server decision did, in raw codes; ssh-host-server-telemetry maps these onto + * enum-only events. Nothing here carries a message, path or host name. + */ +import type { + OrcadManagedConversionResult, + OrcadManagedDeployResult +} from '../../shared/orcad-managed-runtime' +import type { SshTarget } from '../../shared/ssh-types' +import type { HostServerOnConnectResult } from './ssh-host-server-on-connect' +import { classifyOrcadHostUnavailable } from './orcad-host-unavailable' + +export type HostServerConnectEvent = + | { + kind: 'decided' + outcome: 'managed' | 'deployed' | 'converted' | 'relay' + reason: string + refusal: string | null + recorded: boolean + durationMs: number + } + | { kind: 'conversion'; phase: 'started' } + | { kind: 'conversion'; phase: 'committed'; durationMs: number } + | { + kind: 'conversion' + phase: 'failed' + failure: 'deferred' | 'refused' | 'error' + code: string | null + durationMs: number + } + | { + kind: 'deploy_failed' + context: 'deploy' | 'conversion' + failure: 'deferred' | 'error' + code: string | null + durationMs: number + } + +/** How far one decision got, filled in as it runs. */ +export type HostServerDecisionTrace = { + startedAt: number + path: 'existing' | 'deploy' | 'convert' | null + conversionStartedAt: number | null + recorded: boolean + refusal: string | null +} + +export function startHostServerDecisionTrace(now = Date.now()): HostServerDecisionTrace { + return { startedAt: now, path: null, conversionStartedAt: null, recorded: false, refusal: null } +} + +export function decidedEvent( + result: HostServerOnConnectResult | null, + trace: HostServerDecisionTrace, + now = Date.now() +): HostServerConnectEvent { + const base = { refusal: null, recorded: trace.recorded, durationMs: now - trace.startedAt } + if (!result) { + return { kind: 'decided', outcome: 'relay', reason: 'failed', ...base } + } + if (result.route === 'managed') { + if (trace.path === 'deploy') { + return { kind: 'decided', outcome: 'deployed', reason: 'deployed', ...base } + } + if (trace.path === 'convert') { + return { kind: 'decided', outcome: 'converted', reason: 'converted', ...base } + } + return { kind: 'decided', outcome: 'managed', reason: 'connected', ...base } + } + if (result.reason === 'orcad_unavailable') { + // Why the detail: for this reason it is always a classified code, never a message. + return { kind: 'decided', outcome: 'relay', reason: result.detail ?? 'other', ...base } + } + return { + kind: 'decided', + outcome: 'relay', + reason: result.reason, + ...base, + refusal: result.reason === 'refused' ? trace.refusal : null + } +} + +/** A classified code for a thrown setup failure; null when it carries none. */ +export function hostServerFailureCode(error: unknown): string | null { + const unavailable = classifyOrcadHostUnavailable(error) + if (unavailable) { + return unavailable + } + if (error && typeof error === 'object' && 'code' in error && typeof error.code === 'string') { + return error.code + } + return null +} + +export type HostServerReport = (target: SshTarget, event: HostServerConnectEvent) => void + +export function reportHostServerEvent( + report: HostServerReport, + target: SshTarget, + event: HostServerConnectEvent +): void { + try { + report(target, event) + } catch { + // Telemetry never costs a connect. + } +} + +function sinceMs(startedAt: number | null): number { + return startedAt === null ? 0 : Date.now() - startedAt +} + +export function reportHostServerSetupError( + report: HostServerReport, + target: SshTarget, + trace: HostServerDecisionTrace, + error: unknown +): void { + const code = hostServerFailureCode(error) + if (trace.path === 'deploy') { + reportHostServerEvent(report, target, { + kind: 'deploy_failed', + context: 'deploy', + failure: 'error', + code, + durationMs: sinceMs(trace.startedAt) + }) + } else if (trace.path === 'convert') { + reportHostServerEvent(report, target, { + kind: 'conversion', + phase: 'failed', + failure: 'error', + code, + durationMs: sinceMs(trace.conversionStartedAt) + }) + } +} + +export function reportHostServerDeploy( + report: HostServerReport, + target: SshTarget, + trace: HostServerDecisionTrace, + result: OrcadManagedDeployResult +): void { + if (result.outcome === 'deferred') { + reportHostServerEvent(report, target, { + kind: 'deploy_failed', + context: 'deploy', + failure: 'deferred', + code: result.code, + durationMs: sinceMs(trace.startedAt) + }) + } +} + +export function reportHostServerConversion( + report: HostServerReport, + target: SshTarget, + trace: HostServerDecisionTrace, + result: OrcadManagedConversionResult +): void { + const durationMs = sinceMs(trace.conversionStartedAt) + if (result.outcome === 'converted') { + reportHostServerEvent(report, target, { kind: 'conversion', phase: 'committed', durationMs }) + return + } + const failure = result.outcome === 'deferred' ? 'deferred' : 'refused' + const code = result.code + reportHostServerEvent(report, target, { + kind: 'conversion', + phase: 'failed', + failure, + code, + durationMs + }) + if (failure === 'deferred') { + // The conversion's own deploy is what deferred. + reportHostServerEvent(report, target, { + kind: 'deploy_failed', + context: 'conversion', + failure, + code, + durationMs + }) + } +} diff --git a/src/main/ssh/ssh-host-server-on-connect-telemetry.test.ts b/src/main/ssh/ssh-host-server-on-connect-telemetry.test.ts new file mode 100644 index 00000000000..f55a9d37463 --- /dev/null +++ b/src/main/ssh/ssh-host-server-on-connect-telemetry.test.ts @@ -0,0 +1,204 @@ +import { describe, expect, it, vi } from 'vitest' +import type { SshTarget } from '../../shared/ssh-types' +import { + OrcadHostUnsupportedError, + OrcadStdioBridgeUnavailableError +} from './orcad-host-unavailable' +import type { HostServerConnectEvent } from './ssh-host-server-connect-events' +import { + resolveHostServerOnConnect, + type HostServerOnConnectDeps +} from './ssh-host-server-on-connect' + +const target: SshTarget = { id: 'ssh-1', label: 'Box', host: 'box', port: 22, username: 'me' } +// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the orchestrator reads only the environment id. +const environment = { id: 'env-1' } as never + +function deps(overrides: Partial = {}): HostServerOnConnectDeps { + return { + managedEnvironmentId: () => null, + ensureTunnel: vi.fn(async () => undefined), + retireRetainedSource: vi.fn(async () => undefined), + abandonConversion: vi.fn(async () => undefined), + abandonDeploy: vi.fn(async () => undefined), + hasTemplate: () => true, + recordedUnavailable: () => null, + recordUnavailable: vi.fn(), + isEmptyHost: () => false, + relayTerminals: vi.fn(async () => ({ verdict: 'exited' as const, count: 0 })), + deploy: vi.fn(async () => ({ outcome: 'created' as const, environment, activeVersion: '1' })), + convert: vi.fn(async () => ({ outcome: 'converted' as const, environment, migrationId: 'm' })), + progress: vi.fn(), + isFencedBeforeStaging: () => false, + releaseUnreachableSetup: vi.fn(async () => undefined), + report: vi.fn(), + ...overrides + } +} + +async function eventsOf(d: HostServerOnConnectDeps): Promise { + await resolveHostServerOnConnect(target, d).catch(() => undefined) + return vi.mocked(d.report).mock.calls.map(([, event]) => event) +} + +function decided(events: HostServerConnectEvent[]) { + const found = events.filter((event) => event.kind === 'decided') + expect(found).toHaveLength(1) + return found[0] +} + +describe('connect-decision telemetry', () => { + it('reports exactly one decision per connect, naming the outcome', async () => { + const managed = await eventsOf(deps({ managedEnvironmentId: () => 'env-9' })) + expect(decided(managed)).toMatchObject({ outcome: 'managed', reason: 'connected' }) + + const deployed = await eventsOf(deps({ isEmptyHost: () => true })) + expect(decided(deployed)).toMatchObject({ outcome: 'deployed', reason: 'deployed' }) + + const converted = await eventsOf(deps()) + expect(decided(converted)).toMatchObject({ outcome: 'converted', reason: 'converted' }) + expect(converted.filter((event) => event.kind === 'conversion')).toEqual([ + { kind: 'conversion', phase: 'started' }, + expect.objectContaining({ kind: 'conversion', phase: 'committed' }) + ]) + }) + + it('names why a host stays on the relay', async () => { + const live = await eventsOf( + deps({ relayTerminals: async () => ({ verdict: 'live', count: 1 }) }) + ) + expect(decided(live)).toMatchObject({ outcome: 'relay', reason: 'relay_terminals_live' }) + + const noTunnel = await eventsOf( + deps({ + managedEnvironmentId: () => 'env-9', + isFencedBeforeStaging: () => true, + ensureTunnel: async () => { + throw new OrcadStdioBridgeUnavailableError('no bridge') + } + }) + ) + expect(decided(noTunnel)).toMatchObject({ outcome: 'relay', reason: 'ssh_tunnel_unavailable' }) + + const recorded = await eventsOf(deps({ recordedUnavailable: () => 'runtime_self_test' })) + expect(decided(recorded)).toMatchObject({ reason: 'runtime_self_test', recorded: true }) + + const noTemplate = await eventsOf(deps({ hasTemplate: () => false })) + expect(decided(noTemplate)).toMatchObject({ reason: 'artifacts_unavailable' }) + }) + + it('reports a refused conversion with its blocker code', async () => { + const events = await eventsOf( + deps({ + convert: async () => ({ + outcome: 'refused', + verdict: 'live', + code: 'orcad_migration_preflight_blocked', + reason: 'An automation still runs on this host.' + }) + }) + ) + expect(decided(events)).toMatchObject({ + reason: 'refused', + refusal: 'orcad_migration_preflight_blocked' + }) + expect(events).toContainEqual( + expect.objectContaining({ kind: 'conversion', phase: 'failed', failure: 'refused' }) + ) + }) + + it('reports deploy failures, deferred or thrown, with their classified code', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const deferred = await eventsOf( + deps({ + isEmptyHost: () => true, + deploy: async () => ({ + outcome: 'deferred', + candidateVersion: '1.0.0', + code: 'orcad_activation_no_readiness', + reason: 'No readiness line.\nLast lines of orcad.log:\nboom' + }) + }) + ) + expect(deferred).toContainEqual( + expect.objectContaining({ + kind: 'deploy_failed', + context: 'deploy', + failure: 'deferred', + code: 'orcad_activation_no_readiness' + }) + ) + expect(decided(deferred)).toMatchObject({ outcome: 'relay', reason: 'deferred' }) + + const thrown = await eventsOf( + deps({ + isEmptyHost: () => true, + deploy: async () => { + throw new OrcadHostUnsupportedError('no build for this host') + } + }) + ) + expect(thrown).toContainEqual( + expect.objectContaining({ kind: 'deploy_failed', failure: 'error', code: 'unsupported_host' }) + ) + expect(decided(thrown)).toMatchObject({ reason: 'unsupported_host' }) + + const conversion = await eventsOf( + deps({ + convert: async () => ({ + outcome: 'deferred', + candidateVersion: '1.0.0', + code: 'orcad_update_terminals_running', + reason: 'Terminals are running.' + }) + }) + ) + expect(conversion).toContainEqual( + expect.objectContaining({ kind: 'conversion', phase: 'failed', failure: 'deferred' }) + ) + expect(conversion).toContainEqual( + expect.objectContaining({ kind: 'deploy_failed', context: 'conversion' }) + ) + warn.mockRestore() + }) + + it('shows a deferral reason, log tail included, under the SSH host', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const reason = 'No readiness line.\nLast lines of orcad.log:\nError: EADDRINUSE' + const d = deps({ + isEmptyHost: () => true, + deploy: async () => ({ + outcome: 'deferred', + candidateVersion: '1.0.0', + code: 'orcad_activation_no_readiness', + reason + }) + }) + await expect(resolveHostServerOnConnect(target, d)).resolves.toEqual({ + route: 'relay', + reason: 'deferred', + detail: reason + }) + expect(warn).toHaveBeenCalledWith(expect.stringContaining('deferred'), reason) + warn.mockRestore() + }) + + it('still reports, as failed, a decision that throws, and a throwing reporter never fails a connect', async () => { + const d = deps({ + managedEnvironmentId: () => 'env-9', + ensureTunnel: async () => { + throw new Error('tunnel failed') + } + }) + expect(decided(await eventsOf(d))).toMatchObject({ outcome: 'relay', reason: 'failed' }) + + const throwing = deps({ + report: () => { + throw new Error('telemetry broke') + } + }) + await expect(resolveHostServerOnConnect(target, throwing)).resolves.toMatchObject({ + route: 'managed' + }) + }) +}) diff --git a/src/main/ssh/ssh-host-server-on-connect.test.ts b/src/main/ssh/ssh-host-server-on-connect.test.ts index fb79a47a780..e7392a7c9aa 100644 --- a/src/main/ssh/ssh-host-server-on-connect.test.ts +++ b/src/main/ssh/ssh-host-server-on-connect.test.ts @@ -30,6 +30,7 @@ function deps(overrides: Partial = {}): HostServerOnCon progress: vi.fn(), isFencedBeforeStaging: () => false, releaseUnreachableSetup: vi.fn(async () => undefined), + report: vi.fn(), ...overrides } } @@ -187,7 +188,8 @@ describe('which server an SSH host runs on connect', () => { }) await expect(resolveHostServerOnConnect(target, d)).resolves.toEqual({ route: 'relay', - reason: 'failed' + reason: 'failed', + detail: 'Connection lost' }) expect(d.recordUnavailable).not.toHaveBeenCalled() warn.mockRestore() diff --git a/src/main/ssh/ssh-host-server-on-connect.ts b/src/main/ssh/ssh-host-server-on-connect.ts index 17e64662aff..18ba024a7d1 100644 --- a/src/main/ssh/ssh-host-server-on-connect.ts +++ b/src/main/ssh/ssh-host-server-on-connect.ts @@ -17,6 +17,16 @@ import { classifyOrcadHostUnavailable, ORCAD_TUNNEL_UNAVAILABLE_REASON } from './orcad-host-unavailable' +import { + decidedEvent, + reportHostServerConversion, + reportHostServerDeploy, + reportHostServerEvent, + reportHostServerSetupError, + startHostServerDecisionTrace, + type HostServerDecisionTrace, + type HostServerReport +} from './ssh-host-server-connect-events' export type HostServerPhase = 'deploying' | 'converting' | 'connecting' @@ -60,17 +70,38 @@ export type HostServerOnConnectDeps = { isFencedBeforeStaging: (target: SshTarget) => boolean /** Unregisters that unreachable server and releases its fence, so the relay serves again. */ releaseUnreachableSetup: (target: SshTarget) => Promise + /** Receives each decision, conversion and deploy failure as raw codes for telemetry. */ + report: HostServerReport } +// Bounds a failure's detail; the host log tail inside it is already capped. +const MAX_FAILURE_DETAIL_CHARS = 16_000 + export async function resolveHostServerOnConnect( target: SshTarget, deps: HostServerOnConnectDeps +): Promise { + const trace = startHostServerDecisionTrace() + let result: HostServerOnConnectResult | null = null + try { + result = await decide(target, deps, trace) + return result + } finally { + reportHostServerEvent(deps.report, target, decidedEvent(result, trace)) + } +} + +async function decide( + target: SshTarget, + deps: HostServerOnConnectDeps, + trace: HostServerDecisionTrace ): Promise { if (target.orcadFence?.sourceChangedAt) { return { route: 'relay', reason: 'source_changed' } } const existing = deps.managedEnvironmentId(target) if (existing) { + trace.path = 'existing' deps.progress(target, 'connecting') try { await deps.ensureTunnel(existing) @@ -97,6 +128,7 @@ export async function resolveHostServerOnConnect( } const recorded = deps.recordedUnavailable(target) if (recorded) { + trace.recorded = true return { route: 'relay', reason: 'orcad_unavailable', detail: recorded } } if (!deps.hasTemplate()) { @@ -105,8 +137,11 @@ export async function resolveHostServerOnConnect( const empty = deps.isEmptyHost(target) try { if (empty) { + trace.path = 'deploy' deps.progress(target, 'deploying') - return await afterDeploy(target, deps, await deps.deploy(target)) + const deployed = await deps.deploy(target) + reportHostServerDeploy(deps.report, target, trace, deployed) + return await afterDeploy(target, deps, deployed) } const terminals = await deps.relayTerminals(target) if (terminals.verdict !== 'exited') { @@ -117,11 +152,22 @@ export async function resolveHostServerOnConnect( terminals: terminals.count } } + trace.path = 'convert' + trace.conversionStartedAt = Date.now() + reportHostServerEvent(deps.report, target, { kind: 'conversion', phase: 'started' }) deps.progress(target, 'converting') - return await afterConversion(target, deps, await deps.convert(target)) + const converted = await deps.convert(target) + if (converted.outcome === 'refused') { + trace.refusal = converted.code + } + reportHostServerConversion(deps.report, target, trace, converted) + return await afterConversion(target, deps, converted) } catch (error) { console.warn('[ssh] Managed Orca server setup failed; using the relay this session:', error) - return unfinished(target, deps, classifyOrcadHostUnavailable(error), empty, 'failed') + reportHostServerSetupError(deps.report, target, trace, error) + const unavailable = classifyOrcadHostUnavailable(error) + const detail = error instanceof Error ? error.message : String(error) + return unfinished(target, deps, unavailable, empty, 'failed', detail) } } @@ -135,7 +181,7 @@ async function afterDeploy( result: OrcadManagedDeployResult ): Promise { if (result.outcome === 'deferred') { - return deferred(target, deps, result.code, true) + return deferred(target, deps, result, true) } return { route: 'managed', environmentId: result.environment.id } } @@ -149,7 +195,7 @@ async function afterConversion( case 'converted': return { route: 'managed', environmentId: result.environment.id } case 'deferred': - return deferred(target, deps, result.code, false) + return deferred(target, deps, result, false) case 'refused': if (result.code === 'orcad_migration_terminals') { return { route: 'relay', reason: terminalReason(result.verdict) } @@ -161,10 +207,12 @@ async function afterConversion( function deferred( target: SshTarget, deps: HostServerOnConnectDeps, - code: string, + result: { code: string; reason: string }, empty: boolean ): Promise { - return unfinished(target, deps, classifyOrcadHostUnavailable({ code }), empty, 'deferred') + console.warn(`[ssh] Managed Orca server setup deferred (${result.code}):`, result.reason) + const unavailable = classifyOrcadHostUnavailable({ code: result.code }) + return unfinished(target, deps, unavailable, empty, 'deferred', result.reason) } /** @@ -176,7 +224,8 @@ async function unfinished( deps: HostServerOnConnectDeps, unavailable: string | null, empty: boolean, - reason: 'deferred' | 'failed' + reason: 'deferred' | 'failed', + failure: string ): Promise { try { await (empty ? deps.abandonDeploy(target) : deps.abandonConversion(target)) @@ -185,8 +234,13 @@ async function unfinished( console.warn('[ssh] The managed Orca server setup keeps its fence:', error) } if (!unavailable) { - return { route: 'relay', reason } + // Shown under the SSH host, with the host's orcad.log tail when the failure carries one. + return { route: 'relay', reason, detail: boundedDetail(failure) } } deps.recordUnavailable(target, unavailable) return { route: 'relay', reason: 'orcad_unavailable', detail: unavailable } } + +function boundedDetail(text: string): string { + return text.length > MAX_FAILURE_DETAIL_CHARS ? `…${text.slice(-MAX_FAILURE_DETAIL_CHARS)}` : text +} diff --git a/src/main/ssh/ssh-host-server-telemetry.test.ts b/src/main/ssh/ssh-host-server-telemetry.test.ts new file mode 100644 index 00000000000..045b75c39cf --- /dev/null +++ b/src/main/ssh/ssh-host-server-telemetry.test.ts @@ -0,0 +1,184 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { eventSchemas } from '../../shared/telemetry-event-registry' + +vi.mock('../telemetry/client', () => ({ track: vi.fn() })) + +const { track } = await import('../telemetry/client') +const { hostServerFailure, hostServerRefusal, trackSshHostServerEvent, trackSshHostServerMove } = + await import('./ssh-host-server-telemetry') + +const linuxGlibc = { os: 'linux', arch: 'x64', libc: 'glibc' } as const + +function tracked() { + const call = vi.mocked(track).mock.calls.at(-1) + if (!call) { + throw new Error('nothing was tracked') + } + return call +} + +function expectSchemaValid(): unknown { + const [name, props] = tracked() + expect(eventSchemas[name].safeParse(props).success).toBe(true) + return props +} + +describe('ssh host server telemetry', () => { + beforeEach(() => { + vi.mocked(track).mockReset() + }) + + it('maps a decision onto enum-only props the strict schema accepts', () => { + trackSshHostServerEvent( + { + kind: 'decided', + outcome: 'relay', + reason: 'refused', + refusal: 'orcad_migration_preflight_blocked', + recorded: false, + durationMs: 7_000 + }, + linuxGlibc + ) + expect(tracked()[0]).toBe('ssh_host_server_decided') + expect(expectSchemaValid()).toEqual({ + outcome: 'relay', + transport: 'none', + reason: 'refused', + refusal: 'preflight_blocked', + recorded: false, + host_os: 'linux', + host_arch: 'x64', + host_libc: 'glibc', + duration_bucket: '5s_15s' + }) + }) + + it('reports an unprobed host as unknown rather than guessing', () => { + trackSshHostServerEvent( + { + kind: 'decided', + outcome: 'managed', + reason: 'connected', + refusal: null, + recorded: false, + durationMs: 10 + }, + null + ) + expect(expectSchemaValid()).toMatchObject({ + host_os: 'unknown', + host_arch: 'unknown', + host_libc: 'unknown' + }) + }) + + it('names the transport a managed connect used, and none for the relay', () => { + const decision = { + kind: 'decided', + outcome: 'managed', + reason: 'connected', + refusal: null, + recorded: false, + durationMs: 10 + } as const + trackSshHostServerEvent(decision, linuxGlibc, 'stdio_bridge') + expect(expectSchemaValid()).toMatchObject({ transport: 'stdio_bridge' }) + trackSshHostServerEvent(decision, linuxGlibc) + expect(expectSchemaValid()).toMatchObject({ transport: 'unknown' }) + trackSshHostServerEvent( + { ...decision, outcome: 'relay', reason: 'ssh_tunnel_unavailable' }, + linuxGlibc, + 'tcp_forward' + ) + expect(expectSchemaValid()).toMatchObject({ + transport: 'none', + reason: 'ssh_tunnel_unavailable' + }) + }) + + it('reports the per-host move offer and each move result', () => { + for (const outcome of ['offered', 'moved', 'refused_live', 'failed'] as const) { + trackSshHostServerMove(outcome, linuxGlibc) + expect(tracked()[0]).toBe('ssh_host_server_move') + expect(expectSchemaValid()).toEqual({ + outcome, + host_os: 'linux', + host_arch: 'x64', + host_libc: 'glibc' + }) + } + }) + + it('narrows any unrecognised code, so free text never reaches an event', () => { + expect(hostServerRefusal('orcad_migration_/home/me/secret')).toBe('other') + expect(hostServerFailure('Error: ssh me@host.example failed')).toBe('other') + expect(hostServerFailure('orcad_activation_no_readiness')).toBe('activation_no_readiness') + expect(hostServerFailure('unsupported_host')).toBe('unsupported_host') + + trackSshHostServerEvent( + { + kind: 'decided', + outcome: 'relay', + reason: 'host.example.com', + refusal: null, + recorded: false, + durationMs: 0 + }, + linuxGlibc + ) + expect(expectSchemaValid()).toMatchObject({ reason: 'other' }) + }) + + it('maps conversion phases and deploy failures', () => { + trackSshHostServerEvent({ kind: 'conversion', phase: 'started' }, linuxGlibc) + expect(expectSchemaValid()).toMatchObject({ phase: 'started', failure: 'none' }) + + trackSshHostServerEvent( + { kind: 'conversion', phase: 'committed', durationMs: 90_000 }, + linuxGlibc + ) + expect(expectSchemaValid()).toMatchObject({ phase: 'committed', duration_bucket: 'gte_60s' }) + + trackSshHostServerEvent( + { + kind: 'conversion', + phase: 'failed', + failure: 'refused', + code: 'orcad_migration_terminals', + durationMs: 100 + }, + linuxGlibc + ) + expect(expectSchemaValid()).toMatchObject({ refusal: 'terminals', failure_code: 'none' }) + + trackSshHostServerEvent( + { + kind: 'deploy_failed', + context: 'deploy', + failure: 'deferred', + code: 'orcad_candidate_preflight_failed', + durationMs: 20_000 + }, + { os: 'linux', arch: 'arm64', libc: 'musl' } + ) + expect(tracked()[0]).toBe('ssh_host_server_deploy_failed') + expect(expectSchemaValid()).toEqual({ + context: 'deploy', + failure: 'deferred', + failure_code: 'candidate_preflight_failed', + host_os: 'linux', + host_arch: 'arm64', + host_libc: 'musl', + duration_bucket: '15s_60s' + }) + }) + + it('never throws into a connect when tracking fails', () => { + vi.mocked(track).mockImplementation(() => { + throw new Error('invalid') + }) + trackSshHostServerEvent({ kind: 'conversion', phase: 'started' }, null) + expect(track).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/main/ssh/ssh-host-server-telemetry.ts b/src/main/ssh/ssh-host-server-telemetry.ts new file mode 100644 index 00000000000..f3774f63ed9 --- /dev/null +++ b/src/main/ssh/ssh-host-server-telemetry.ts @@ -0,0 +1,102 @@ +/** + * Connect-decision telemetry: which server each SSH connect chose and why, plus conversion and + * deploy failures. Every raw code is narrowed onto a fixed enum here, so nothing a host or user + * named can reach an event; unrecognised codes become `other`. + */ +import type { EventProps } from '../../shared/telemetry-events' +import { + SSH_HOST_SERVER_FAILURE_VALUES, + SSH_HOST_SERVER_REASON_VALUES, + SSH_HOST_SERVER_REFUSAL_VALUES +} from '../../shared/telemetry-ssh-host-server-event-schemas' +import { track } from '../telemetry/client' +import type { HostServerConnectEvent } from './ssh-host-server-connect-events' +import type { SshHostPlatformFacts } from './ssh-host-platform-memo' +import type { OrcadTunnelTransport } from './orcad-tunnel-transport-memo' +import { durationBucket } from './ssh-remote-runtime-telemetry' + +type HostProps = Pick, 'host_os' | 'host_arch' | 'host_libc'> +type FailureCode = (typeof SSH_HOST_SERVER_FAILURE_VALUES)[number] +type RefusalCode = (typeof SSH_HOST_SERVER_REFUSAL_VALUES)[number] + +function pick(values: readonly T[], value: string, fallback: T): T { + return values.find((candidate) => candidate === value) ?? fallback +} + +function hostProps(facts: SshHostPlatformFacts | null): HostProps { + return facts + ? { host_os: facts.os, host_arch: facts.arch, host_libc: facts.libc } + : { host_os: 'unknown', host_arch: 'unknown', host_libc: 'unknown' } +} + +export function hostServerRefusal(code: string | null): RefusalCode { + return code + ? pick(SSH_HOST_SERVER_REFUSAL_VALUES, code.replace(/^orcad_migration_/u, ''), 'other') + : 'none' +} + +export function hostServerFailure(code: string | null): FailureCode { + return code + ? pick(SSH_HOST_SERVER_FAILURE_VALUES, code.replace(/^orcad_/u, ''), 'other') + : 'other' +} + +export type SshHostServerMoveOutcome = EventProps<'ssh_host_server_move'>['outcome'] + +export function trackSshHostServerMove( + outcome: SshHostServerMoveOutcome, + facts: SshHostPlatformFacts | null +): void { + try { + track('ssh_host_server_move', { outcome, ...hostProps(facts) }) + } catch { + // Telemetry never costs a move. + } +} + +export function trackSshHostServerEvent( + event: HostServerConnectEvent, + facts: SshHostPlatformFacts | null, + transport: OrcadTunnelTransport | null = null +): void { + const host = hostProps(facts) + try { + // track() applies the consent gate and validates against the strict schema. + switch (event.kind) { + case 'decided': + track('ssh_host_server_decided', { + outcome: event.outcome, + transport: event.outcome === 'relay' ? 'none' : (transport ?? 'unknown'), + reason: pick(SSH_HOST_SERVER_REASON_VALUES, event.reason, 'other'), + refusal: hostServerRefusal(event.refusal), + recorded: event.recorded, + ...host, + duration_bucket: durationBucket(event.durationMs) + }) + return + case 'conversion': { + const failed = event.phase === 'failed' + track('ssh_host_server_conversion', { + phase: event.phase, + failure: failed ? event.failure : 'none', + refusal: failed && event.failure === 'refused' ? hostServerRefusal(event.code) : 'none', + failure_code: + failed && event.failure !== 'refused' ? hostServerFailure(event.code) : 'none', + ...host, + duration_bucket: durationBucket(event.phase === 'started' ? 0 : event.durationMs) + }) + return + } + case 'deploy_failed': + track('ssh_host_server_deploy_failed', { + context: event.context, + failure: event.failure, + failure_code: hostServerFailure(event.code), + ...host, + duration_bucket: durationBucket(event.durationMs) + }) + } + } catch { + // Telemetry never costs a connect. + } +} diff --git a/src/main/ssh/ssh-relay-runtime-resolution.ts b/src/main/ssh/ssh-relay-runtime-resolution.ts index e5c6e37b230..19fcf28c4a9 100644 --- a/src/main/ssh/ssh-relay-runtime-resolution.ts +++ b/src/main/ssh/ssh-relay-runtime-resolution.ts @@ -23,6 +23,7 @@ import { type RemoteRuntimeUnavailableReason } from './ssh-relay-runtime-ladder' import type { PinnedRuntimeRefusal } from './ssh-relay-runtime-self-test' +import { rememberSshHostPlatform } from './ssh-host-platform-memo' import type { RemoteHostPlatform } from './ssh-remote-platform' import type { HostNodeVersion } from './ssh-remote-node-toolchain-probe' import { @@ -141,6 +142,7 @@ export class RelayRuntimeLadderRun { private track(rung: SshRemoteRuntimeRung, outcome: SshRemoteRuntimeOutcome): void { if (this.host) { + rememberSshHostPlatform(this.targetId, this.host, this.facts?.target ?? null) trackSshRemoteRuntimeResolved(this.targetId, { rung, outcome, diff --git a/src/renderer/src/components/settings/SshTargetServerStatus.tsx b/src/renderer/src/components/settings/SshTargetServerStatus.tsx index 861c62de4f2..dc992aa0e1e 100644 --- a/src/renderer/src/components/settings/SshTargetServerStatus.tsx +++ b/src/renderer/src/components/settings/SshTargetServerStatus.tsx @@ -1,6 +1,7 @@ -import { ArrowRightLeft } from 'lucide-react' +import { ArrowRightLeft, ChevronDown } from 'lucide-react' import { useState } from 'react' import type { SshTarget } from '../../../../shared/ssh-types' +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible' import { translate } from '@/i18n/i18n' import { cn } from '@/lib/utils' import { canMoveSshHostToManagedServer } from '@/ssh/ssh-managed-server-move' @@ -47,6 +48,21 @@ export function SshTargetServerStatus({ ) : null} + {line.detail ? ( + + + + + + 
+              {line.detail}
+
+ + + ) : null} {canMove ? ( { ).not.toHaveProperty('action') }) + it('offers the setup failure, log tail included, beside a retry line', () => { + const detail = 'No readiness line.\nLast lines of orcad.log:\nError: EADDRINUSE' + expect( + sshHostServerStatusLine(plain, { + managedServer: { kind: 'relay', reason: 'deferred', detail } + }) + ).toMatchObject({ tone: 'muted', detail }) + expect( + sshHostServerStatusLine(plain, { managedServer: { kind: 'relay', reason: 'failed' } }) + ).not.toHaveProperty('detail') + }) + it('keeps durable reasons visible without a live state', () => { expect(sshHostServerStatusLine({ orcadFence: { environmentId: 'e' } }, undefined)?.text).toBe( 'Runs a managed Orca server' diff --git a/src/renderer/src/components/settings/ssh-host-server-status-copy.ts b/src/renderer/src/components/settings/ssh-host-server-status-copy.ts index 523c7684598..002ddc3b2ea 100644 --- a/src/renderer/src/components/settings/ssh-host-server-status-copy.ts +++ b/src/renderer/src/components/settings/ssh-host-server-status-copy.ts @@ -7,6 +7,8 @@ export type SshHostServerStatusLine = { tone: 'muted' | 'warning' | 'destructive' /** Offers "Move to managed server", which restarts the live relay terminals. */ action?: 'move' + /** Why setup stopped, with the host's orcad.log tail when there is one; shown on request. */ + detail?: string } export function sshHostServerStatusLine( @@ -156,6 +158,6 @@ function relayLine( } case 'deferred': case 'failed': - return retryLine() + return { ...retryLine(), ...(status.detail ? { detail: status.detail } : {}) } } } diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index af410f0d54e..9bb5b968d8c 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -12411,7 +12411,8 @@ "terminalsLive": "Runs the relay until its {{count}} open terminals are closed, then moves to a managed server.", "terminalsUnverifiable": "Runs the relay: Orca couldn’t confirm its terminals are closed. It moves on a later connect.", "refused": "Not moved to a managed server: {{blocker}}", - "retry": "Runs the relay this session; it moves to a managed server on a later connect." + "retry": "Runs the relay this session; it moves to a managed server on a later connect.", + "failureDetails": "Details" }, "ZcodePlanAccountsSection": { "site": { diff --git a/src/shared/telemetry-event-registry.ts b/src/shared/telemetry-event-registry.ts index f2d1a6e7c52..4cfa4ac49b5 100644 --- a/src/shared/telemetry-event-registry.ts +++ b/src/shared/telemetry-event-registry.ts @@ -97,6 +97,12 @@ import { workspaceCreateFailedSchema } from './telemetry-repository-event-schemas' import { sshRemoteRuntimeResolvedSchema } from './telemetry-ssh-runtime-event-schemas' +import { + sshHostServerConversionSchema, + sshHostServerDecidedSchema, + sshHostServerDeployFailedSchema, + sshHostServerMoveSchema +} from './telemetry-ssh-host-server-event-schemas' // ── Event registry: the one record the validator consumes ─────────────── // Versioning: breaking changes (rename/re-mean/remove a key) need a new event name; in-place edits blend pre/post rows unmixably. Additive-optional fields are safe. @@ -199,6 +205,10 @@ export const eventSchemas = { direct_ssh_reconnect_operation: directSshReconnectOperationSchema, ssh_remote_runtime_resolved: sshRemoteRuntimeResolvedSchema, + ssh_host_server_decided: sshHostServerDecidedSchema, + ssh_host_server_conversion: sshHostServerConversionSchema, + ssh_host_server_deploy_failed: sshHostServerDeployFailedSchema, + ssh_host_server_move: sshHostServerMoveSchema, smart_sort_class_distribution: smartSortClassDistributionSchema, smart_sort_class_1_promotion: smartSortClass1PromotionSchema, diff --git a/src/shared/telemetry-ssh-host-server-event-schemas.ts b/src/shared/telemetry-ssh-host-server-event-schemas.ts new file mode 100644 index 00000000000..721a1f032c0 --- /dev/null +++ b/src/shared/telemetry-ssh-host-server-event-schemas.ts @@ -0,0 +1,142 @@ +import { z } from 'zod' +import { SSH_RUNTIME_DURATION_BUCKETS } from './telemetry-ssh-runtime-event-schemas' + +// Why enum-only: these rows size how many hosts still need the relay before it can retire. A +// hostname, user, path, log line or raw error would identify a machine, so none may appear. + +export const SSH_HOST_SERVER_OUTCOME_VALUES = ['managed', 'deployed', 'converted', 'relay'] as const + +/** Why orcad can't run on a host; mirrors main's OrcadHostUnavailableReason. */ +const UNAVAILABLE_REASONS = [ + // Recorded by builds before the stdio bridge; now retried rather than honoured. + 'tcp_forwarding_refused', + // The host refuses port forwarding and can't run the stdio bridge either. + 'ssh_tunnel_unavailable', + 'unsupported_host', + 'artifacts_unavailable', + 'libc_unidentified', + 'runtime_self_test', + 'security_software', + 'native_preflight' +] as const + +export const SSH_HOST_SERVER_REASON_VALUES = [ + 'connected', + 'deployed', + 'converted', + 'source_changed', + 'relay_terminals_live', + 'relay_terminals_unverifiable', + ...UNAVAILABLE_REASONS, + 'refused', + 'deferred', + 'failed', + 'other' +] as const + +/** A conversion refusal's code without its `orcad_migration_` prefix. */ +export const SSH_HOST_SERVER_REFUSAL_VALUES = [ + 'none', + 'already_managed', + 'terminals', + 'target_not_found', + 'in_progress', + 'direct_ssh_connected', + 'preflight_blocked', + 'owned_by_deploy', + 'fenced_unverifiable', + 'stale_journal', + 'other' +] as const + +/** A deploy's deferral code without its `orcad_` prefix, or why orcad can't run on the host. */ +export const SSH_HOST_SERVER_FAILURE_VALUES = [ + ...UNAVAILABLE_REASONS, + 'activation_recovery_required', + 'candidate_preflight_failed', + 'candidate_launch_failed', + 'incumbent_identity_unverifiable', + 'outgoing_stop_incomplete', + 'activation_build_mismatch', + 'activation_daemon_absent', + 'activation_daemon_degraded', + 'activation_no_health', + 'activation_no_persistent_terminals', + 'activation_no_readiness', + 'activation_not_listening', + 'activation_pty_self_test_failed', + 'update_daemon_protocol_unverifiable', + 'update_strands_live_terminals', + 'update_terminal_census_unavailable', + 'update_terminals_running', + 'windows_launch_refused', + 'windows_command_line_unsafe', + 'other' +] as const + +const hostFields = { + host_os: z.enum(['linux', 'darwin', 'win32', 'unknown']), + host_arch: z.enum(['x64', 'arm64', 'unknown']), + // `unknown` when this session has not yet probed the host; the decision itself never probes. + host_libc: z.enum(['glibc', 'musl', 'none', 'unknown']) +} + +/** How the client reached a managed server; `none` when the host stayed on the relay. */ +export const SSH_HOST_SERVER_TRANSPORT_VALUES = [ + 'tcp_forward', + 'stdio_bridge', + 'none', + 'unknown' +] as const + +export const SSH_HOST_SERVER_MOVE_OUTCOME_VALUES = [ + 'offered', + 'moved', + 'stayed', + 'refused_live', + 'refused_unverifiable', + 'failed' +] as const + +export const sshHostServerDecidedSchema = z + .object({ + outcome: z.enum(SSH_HOST_SERVER_OUTCOME_VALUES), + transport: z.enum(SSH_HOST_SERVER_TRANSPORT_VALUES), + reason: z.enum(SSH_HOST_SERVER_REASON_VALUES), + refusal: z.enum(SSH_HOST_SERVER_REFUSAL_VALUES), + // True when a recorded "orcad can't run here" sent the host to the relay without a try. + recorded: z.boolean(), + ...hostFields, + duration_bucket: z.enum(SSH_RUNTIME_DURATION_BUCKETS) + }) + .strict() + +export const sshHostServerConversionSchema = z + .object({ + phase: z.enum(['started', 'committed', 'failed']), + failure: z.enum(['none', 'deferred', 'refused', 'error']), + refusal: z.enum(SSH_HOST_SERVER_REFUSAL_VALUES), + failure_code: z.enum(['none', ...SSH_HOST_SERVER_FAILURE_VALUES]), + ...hostFields, + duration_bucket: z.enum(SSH_RUNTIME_DURATION_BUCKETS) + }) + .strict() + +export const sshHostServerDeployFailedSchema = z + .object({ + // Whether the deploy ran for an empty host or inside a conversion. + context: z.enum(['deploy', 'conversion']), + failure: z.enum(['deferred', 'error']), + failure_code: z.enum(SSH_HOST_SERVER_FAILURE_VALUES), + ...hostFields, + duration_bucket: z.enum(SSH_RUNTIME_DURATION_BUCKETS) + }) + .strict() + +/** The per-host offer to move a host whose open terminals keep it on the relay, and its result. */ +export const sshHostServerMoveSchema = z + .object({ + outcome: z.enum(SSH_HOST_SERVER_MOVE_OUTCOME_VALUES), + ...hostFields + }) + .strict()