mirror of
https://github.com/stablyai/orca.git
synced 2026-10-13 00:02:54 +00:00
feat(ssh): report each connect's server decision, and show orcad.log's last lines when setup fails (#25118)
* telemetry: one enum-only event per connect decision (outcome, reason, tunnel transport, host platform, duration), plus conversion start/commit/fail, deploy failures, and the per-host move offer and its result * errors: deploy, launch and rollback failures carry the last 40 redacted lines of the host's orcad.log, read over SSH (Windows through the node.exe host script) * SSH Hosts: a deferred or failed setup offers its reason, log tail included, under Details Co-authored-by: m4air <m4air@Mac.localdomain>
This commit is contained in:
@@ -4,6 +4,8 @@ import type { SshConnectionState, SshTarget } from '../../shared/ssh-types'
|
||||
import type { HostServerOnConnectResult } from '../ssh/ssh-host-server-on-connect'
|
||||
import { relayServerStatus, shouldToastManagedServerMove } from '../ssh/ssh-host-server-move-offer'
|
||||
import { setSshHostServerStatus } from '../ssh/ssh-host-server-status'
|
||||
import { trackSshHostServerMove } from '../ssh/ssh-host-server-telemetry'
|
||||
import { knownSshHostPlatform } from '../ssh/ssh-host-platform-memo'
|
||||
import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry'
|
||||
import { connectionManager, getCurrentMainWindow } from './ssh-ipc-context'
|
||||
import { broadcastSshState, getPublicSshState } from './ssh-renderer-broadcast'
|
||||
@@ -57,6 +59,7 @@ export function recordRelayDecision(
|
||||
getSshTargetRegistryStore()!.updateTarget(target.id, {
|
||||
managedServerMoveOffered: { appVersion }
|
||||
})
|
||||
trackSshHostServerMove('offered', knownSshHostPlatform(target.id))
|
||||
}
|
||||
setSshHostServerStatus(target.id, relayServerStatus(decision, offerMove))
|
||||
}
|
||||
|
||||
@@ -15,6 +15,9 @@ import { orcadMigrationDestinationFor } from '../ssh/orcad-runtime-conversion-wi
|
||||
import { createManagedOrcadEnvironment } from '../ssh/orcad-runtime-deployment'
|
||||
import type { HostServerOnConnectDeps } from '../ssh/ssh-host-server-on-connect'
|
||||
import { setSshHostServerStatus } from '../ssh/ssh-host-server-status'
|
||||
import { trackSshHostServerEvent } from '../ssh/ssh-host-server-telemetry'
|
||||
import { knownSshHostPlatform } from '../ssh/ssh-host-platform-memo'
|
||||
import { knownOrcadTunnelTransport } from '../ssh/orcad-tunnel-transport-memo'
|
||||
import {
|
||||
getSshTargetRegistryStore,
|
||||
hasRegisteredDirectSshAuthority
|
||||
@@ -117,6 +120,13 @@ export function hostServerOnConnectDeps(userDataPath: string): HostServerOnConne
|
||||
findOrcadMigrationSourceCutoverForTarget(userDataPath, target.id)?.phase === 'source-fenced',
|
||||
releaseUnreachableSetup: (target) =>
|
||||
releaseUnreachableOrcadSetup({ userDataPath, claims, targetId: target.id }),
|
||||
// Read at report time: a deploy or conversion learns the platform while the decision runs.
|
||||
report: (target, event) =>
|
||||
trackSshHostServerEvent(
|
||||
event,
|
||||
knownSshHostPlatform(target.id),
|
||||
knownOrcadTunnelTransport(target.id)
|
||||
),
|
||||
progress: (target, phase) => {
|
||||
setSshHostServerStatus(target.id, { kind: 'setting-up', phase })
|
||||
broadcastSshState(getCurrentMainWindow, target.id, {
|
||||
|
||||
@@ -38,7 +38,8 @@ function deps(
|
||||
calls.push('connect')
|
||||
status = options.afterConnect ?? { kind: 'managed', environmentId: 'env-1' }
|
||||
}),
|
||||
serverStatus: vi.fn(() => status)
|
||||
serverStatus: vi.fn(() => status),
|
||||
report: vi.fn()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,6 +52,7 @@ describe('moving an SSH host to its managed server on request', () => {
|
||||
})
|
||||
expect(move.calls).toEqual(['terminate', 'census', 'connect'])
|
||||
expect(move.relayTerminals).toHaveBeenCalledWith(target)
|
||||
expect(move.report).toHaveBeenCalledWith('ssh-1', 'moved')
|
||||
})
|
||||
|
||||
it('refuses without converting when the stop could not reach every terminal', async () => {
|
||||
@@ -61,6 +63,7 @@ describe('moving an SSH host to its managed server on request', () => {
|
||||
terminals: 1
|
||||
})
|
||||
expect(move.calls).toEqual(['terminate'])
|
||||
expect(move.report).toHaveBeenCalledWith('ssh-1', 'refused_unverifiable')
|
||||
})
|
||||
|
||||
it('refuses without converting when the census is unverifiable or still live', async () => {
|
||||
@@ -78,11 +81,13 @@ describe('moving an SSH host to its managed server on request', () => {
|
||||
verdict: 'live'
|
||||
})
|
||||
expect(live.connect).not.toHaveBeenCalled()
|
||||
expect(live.report).toHaveBeenCalledWith('ssh-1', 'refused_live')
|
||||
})
|
||||
|
||||
it('reports a connect that kept the relay for another reason', async () => {
|
||||
const move = deps({ afterConnect: { kind: 'relay', reason: 'refused', detail: 'blocked' } })
|
||||
await expect(moveSshHostToManagedServer('ssh-1', move)).resolves.toEqual({ outcome: 'stayed' })
|
||||
expect(move.report).toHaveBeenCalledWith('ssh-1', 'stayed')
|
||||
})
|
||||
|
||||
it('reattaches the relay first when preserved terminals need one to be stopped', async () => {
|
||||
@@ -101,5 +106,6 @@ describe('moving an SSH host to its managed server on request', () => {
|
||||
move.terminate.mockRejectedValueOnce(new Error('Failed to terminate SSH host sessions'))
|
||||
await expect(moveSshHostToManagedServer('ssh-1', move)).rejects.toThrow('Failed to terminate')
|
||||
expect(move.connect).not.toHaveBeenCalled()
|
||||
expect(move.report).toHaveBeenCalledWith('ssh-1', 'failed')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -12,6 +12,11 @@ import type {
|
||||
} from '../../shared/ssh-types'
|
||||
import type { HostServerTerminalVerdict } from '../ssh/ssh-host-server-on-connect'
|
||||
import { getSshHostServerStatus } from '../ssh/ssh-host-server-status'
|
||||
import {
|
||||
trackSshHostServerMove,
|
||||
type SshHostServerMoveOutcome
|
||||
} from '../ssh/ssh-host-server-telemetry'
|
||||
import { knownSshHostPlatform } from '../ssh/ssh-host-platform-memo'
|
||||
import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry'
|
||||
import { connectTarget } from './ssh-connect-flow'
|
||||
import { terminateSshTargetSessions } from './ssh-terminate-sessions'
|
||||
@@ -22,11 +27,32 @@ export type SshManagedServerMoveDeps = {
|
||||
relayTerminals: (target: SshTarget) => Promise<HostServerTerminalVerdict>
|
||||
connect: (targetId: string) => Promise<unknown>
|
||||
serverStatus: (targetId: string) => SshManagedServerStatus | undefined
|
||||
report: (targetId: string, outcome: SshHostServerMoveOutcome) => void
|
||||
}
|
||||
|
||||
export async function moveSshHostToManagedServer(
|
||||
targetId: string,
|
||||
deps: SshManagedServerMoveDeps = defaultMoveDeps()
|
||||
): Promise<SshManagedServerMoveResult> {
|
||||
let result: SshManagedServerMoveResult | null = null
|
||||
try {
|
||||
result = await moveHost(targetId, deps)
|
||||
return result
|
||||
} finally {
|
||||
deps.report(targetId, moveOutcome(result))
|
||||
}
|
||||
}
|
||||
|
||||
function moveOutcome(result: SshManagedServerMoveResult | null): SshHostServerMoveOutcome {
|
||||
if (!result) {
|
||||
return 'failed'
|
||||
}
|
||||
return result.outcome === 'refused' ? `refused_${result.verdict}` : result.outcome
|
||||
}
|
||||
|
||||
async function moveHost(
|
||||
targetId: string,
|
||||
deps: SshManagedServerMoveDeps
|
||||
): Promise<SshManagedServerMoveResult> {
|
||||
const target = deps.getTarget(targetId)
|
||||
if (!target) {
|
||||
@@ -74,6 +100,7 @@ function defaultMoveDeps(): SshManagedServerMoveDeps {
|
||||
return hostServerOnConnectDeps(getAppEnvironment().getPath('userData')).relayTerminals(target)
|
||||
},
|
||||
connect: connectTarget,
|
||||
serverStatus: getSshHostServerStatus
|
||||
serverStatus: getSshHostServerStatus,
|
||||
report: (targetId, outcome) => trackSshHostServerMove(outcome, knownSshHostPlatform(targetId))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,7 +31,11 @@ const MAIN_OWNED_TELEMETRY_EVENTS = new Set<EventName>([
|
||||
'daemon_pty_cwd_readable',
|
||||
'star_nag_outcome',
|
||||
'feature_interaction_usage_bucket_reached',
|
||||
'ssh_remote_runtime_resolved'
|
||||
'ssh_remote_runtime_resolved',
|
||||
'ssh_host_server_decided',
|
||||
'ssh_host_server_conversion',
|
||||
'ssh_host_server_deploy_failed',
|
||||
'ssh_host_server_move'
|
||||
])
|
||||
|
||||
/**
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
readOrcadReadinessNowCommand
|
||||
} from './orcad-remote-readiness-wait'
|
||||
import type { ServeReadiness } from '../server/serve-readiness'
|
||||
import { withOrcadLogTail } from './orcad-remote-log-tail'
|
||||
|
||||
/** `exited` is proven absence; `unverifiable` means the host could not say, which is not death. */
|
||||
export type OrcadReadinessFailureVerdict = 'exited' | 'unverifiable' | 'rejected'
|
||||
@@ -116,5 +117,13 @@ export async function launchOrcadSlotAndAwaitReadiness(
|
||||
expectation: OrcadActivationExpectation
|
||||
): Promise<ServeReadiness> {
|
||||
const parsed = await launchOrcadAndAwaitReadiness(target, spec)
|
||||
return gatedReadiness(parsed, expectation, `orcad ${spec.fullVersion}`)
|
||||
try {
|
||||
return gatedReadiness(parsed, expectation, `orcad ${spec.fullVersion}`)
|
||||
} catch (error) {
|
||||
if (!(error instanceof OrcadActiveReadinessError)) {
|
||||
throw error
|
||||
}
|
||||
const message = await withOrcadLogTail(target, spec.remoteInstallDir, error.message)
|
||||
throw new OrcadActiveReadinessError(error.verdict, message)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,6 +57,7 @@ import {
|
||||
type OrcadSlotIdentity
|
||||
} from './orcad-recovery-slot'
|
||||
import { orcadSnapshotPath, recoverOrcadIncumbent } from './orcad-incumbent-recovery'
|
||||
import { withOrcadLogTail } from './orcad-remote-log-tail'
|
||||
|
||||
type Outcome = Extract<OrcadDeployResult, { outcome: 'installed-not-activated' }>
|
||||
|
||||
@@ -242,11 +243,10 @@ export async function activateInstalledOrcad(
|
||||
`The candidate failed while starting: ${errorMessage(launchError)}`
|
||||
]
|
||||
const restored = await restoreAfterRejectedCandidate(options, transaction, incumbent, lock)
|
||||
return notActivated(
|
||||
code,
|
||||
const located =
|
||||
`${reason} Candidate stderr is at ` +
|
||||
`${joinRemotePath(options.host, remoteDir, ORCAD_LOG_FILENAME)}. ${restored}`
|
||||
)
|
||||
`${joinRemotePath(options.host, remoteDir, ORCAD_LOG_FILENAME)}. ${restored}`
|
||||
return notActivated(code, await withOrcadLogTail(options, remoteDir, located))
|
||||
}
|
||||
|
||||
const recordAfter = withActivatedVersion(record, fullVersion, snapshot, now())
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { OrcadManagedTunnelTransportProvider } from './orcad-managed-tunnel-transport'
|
||||
import { knownOrcadTunnelTransport } from './orcad-tunnel-transport-memo'
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import type {
|
||||
PortForwardStartOptions,
|
||||
@@ -49,6 +50,7 @@ describe('the managed tunnel’s transport', () => {
|
||||
})
|
||||
expect(refused.stdio.start).toHaveBeenCalledWith(conn, options)
|
||||
expect(refused.forward.start).not.toHaveBeenCalled()
|
||||
expect(knownOrcadTunnelTransport('ssh-1')).toBe('stdio_bridge')
|
||||
})
|
||||
|
||||
it('keeps the forward when forwarding is allowed or the answer is unverifiable', async () => {
|
||||
@@ -58,6 +60,7 @@ describe('the managed tunnel’s transport', () => {
|
||||
entry: { id: 'forward' }
|
||||
})
|
||||
expect(chosen.stdio.start).not.toHaveBeenCalled()
|
||||
expect(knownOrcadTunnelTransport('ssh-1')).toBe('tcp_forward')
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
* wherever the host allows it, and the stdio bridge where its sshd refuses forwarding.
|
||||
*/
|
||||
import { OrcadStdioBridgePortForwardProvider } from './orcad-stdio-bridge-provider'
|
||||
import { rememberOrcadTunnelTransport } from './orcad-tunnel-transport-memo'
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import type {
|
||||
PortForwardStartOptions,
|
||||
@@ -36,12 +37,16 @@ export class OrcadManagedTunnelTransportProvider implements SshPortForwardProvid
|
||||
const { forwards, stdio, probe } = this.dependencies
|
||||
// Why only on refusal: an unverifiable answer keeps the forward, as before this transport.
|
||||
if ((await probe(conn, options.remotePort)) === 'refused' && stdio.canHandle(conn)) {
|
||||
return stdio.start(conn, options)
|
||||
const started = await stdio.start(conn, options)
|
||||
rememberOrcadTunnelTransport(options.connectionId, 'stdio_bridge')
|
||||
return started
|
||||
}
|
||||
const forward = forwards.find((provider) => provider.canHandle(conn))
|
||||
if (!forward) {
|
||||
throw new Error('SSH connection is not established')
|
||||
}
|
||||
return forward.start(conn, options)
|
||||
const started = await forward.start(conn, options)
|
||||
rememberOrcadTunnelTransport(options.connectionId, 'tcp_forward')
|
||||
return started
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
} from './ssh-remote-platform'
|
||||
import { detectRemoteHostPlatform } from './ssh-remote-platform-detection'
|
||||
import { OrcadHostUnsupportedError } from './orcad-host-unavailable'
|
||||
import { rememberSshHostPlatform } from './ssh-host-platform-memo'
|
||||
|
||||
export type OrcadRemoteContext = {
|
||||
activationRecord: OrcadActivationRecord
|
||||
@@ -47,6 +48,7 @@ export async function resolveOrcadRemoteContext(
|
||||
throw new Error(`Remote home is not a valid path: ${remoteHome.slice(0, 100)}`)
|
||||
}
|
||||
const serverTarget = await resolveOrcadDeploymentTarget({ conn: connection, host, signal })
|
||||
rememberSshHostPlatform(target.id, host, serverTarget)
|
||||
if (isWindowsRemoteHost(host)) {
|
||||
// Every Windows host op, the activation record read included, runs on the pinned node.exe.
|
||||
await prepareWindowsOrcadHost({ conn: connection, host, remoteHome, serverTarget, signal })
|
||||
|
||||
@@ -109,11 +109,15 @@ type HostScript = {
|
||||
comparisonResult?: string
|
||||
candidateStopResult?: string
|
||||
readinessAtMs?: number
|
||||
orcadLog?: string
|
||||
}
|
||||
|
||||
function scriptHost(script: HostScript): void {
|
||||
mockExec.mockImplementation(async (_conn, command: string) => {
|
||||
const text = String(command)
|
||||
if (text.startsWith('tail -c')) {
|
||||
return script.orcadLog ?? ''
|
||||
}
|
||||
if (text.includes('__ORCAD_RECORD_PRESENT__') && text.includes('orcad-active.json')) {
|
||||
return script.activationRecord
|
||||
? `__ORCAD_RECORD_PRESENT__\n${script.activationRecord}`
|
||||
@@ -504,11 +508,16 @@ describe('deployOrcad', () => {
|
||||
const script: HostScript = {
|
||||
activationRecord: ACTIVE_OLD,
|
||||
readiness: { [NEW_VERSION]: readyLine({ daemonState: 'degraded' }) },
|
||||
log: []
|
||||
log: [],
|
||||
orcadLog: 'daemon: starting\ndaemon: socket bind failed: EACCES\n'
|
||||
}
|
||||
scriptHost(script)
|
||||
const result = await deployOrcad(options())
|
||||
expect(result).toMatchObject({ code: 'orcad_activation_daemon_degraded' })
|
||||
// The error carries why the candidate failed, not only where its log is.
|
||||
expect(result).toMatchObject({
|
||||
reason: expect.stringMatching(/Last lines of orcad\.log:\ndaemon: starting\n.*EACCES$/u)
|
||||
})
|
||||
expect(
|
||||
vi
|
||||
.mocked(writeAtomicOrcadRemoteRecord)
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { runProcess } from '../../shared/child-process/run-process'
|
||||
import {
|
||||
ORCAD_LOG_TAIL_LINES,
|
||||
ORCAD_LOG_TAIL_MAX_BYTES,
|
||||
orcadLogTailCommand,
|
||||
parseOrcadLogTail,
|
||||
withOrcadLogTail
|
||||
} from './orcad-remote-log-tail'
|
||||
import { ORCAD_WINDOWS_HOST_SCRIPT } from './orcad-windows-host-script'
|
||||
import { getRemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
const execCommand = vi.hoisted(() => vi.fn())
|
||||
vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand }))
|
||||
|
||||
const linux = getRemoteHostPlatform('linux-x64')
|
||||
const windows = getRemoteHostPlatform('win32-x64')
|
||||
const SLOT = '/home/me/.orca-remote/orcad-1.0.0'
|
||||
const WINDOWS_SLOT = 'C:\\Users\\me\\.orca-remote\\orcad-1.0.0'
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand is mocked; the connection is never used.
|
||||
const conn = {} as never
|
||||
|
||||
let dir = ''
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), 'orcad-log-tail-'))
|
||||
execCommand.mockReset()
|
||||
})
|
||||
afterEach(() => rmSync(dir, { recursive: true, force: true }))
|
||||
|
||||
function numberedLines(count: number): string {
|
||||
return Array.from({ length: count }, (_, index) => `line ${index + 1}`).join('\n')
|
||||
}
|
||||
|
||||
describe('orcad.log tail', () => {
|
||||
it('keeps only the last lines and bytes, redacting secrets', () => {
|
||||
const tail = parseOrcadLogTail(linux, `${numberedLines(100)}\ntoken=abc123secret\n`)
|
||||
const lines = (tail ?? '').split('\n')
|
||||
expect(lines).toHaveLength(ORCAD_LOG_TAIL_LINES)
|
||||
expect(lines[0]).toBe('line 62')
|
||||
expect(tail).not.toContain('abc123secret')
|
||||
expect(tail).toContain('[redacted')
|
||||
|
||||
const huge = parseOrcadLogTail(linux, 'x'.repeat(ORCAD_LOG_TAIL_MAX_BYTES * 4))
|
||||
expect(Buffer.byteLength(huge ?? '', 'utf8')).toBeLessThanOrEqual(ORCAD_LOG_TAIL_MAX_BYTES)
|
||||
expect(parseOrcadLogTail(linux, ' \n')).toBeNull()
|
||||
})
|
||||
|
||||
it('reads the log on Windows through the node.exe host script, never an encoded command', () => {
|
||||
const command = orcadLogTailCommand(windows, WINDOWS_SLOT)
|
||||
expect(command).toContain('node.exe')
|
||||
expect(command).toContain(' log-tail ')
|
||||
expect(command).not.toMatch(/EncodedCommand/iu)
|
||||
})
|
||||
|
||||
it.skipIf(process.platform === 'win32')(
|
||||
'runs the POSIX command against a real file',
|
||||
async () => {
|
||||
const slot = join(dir, 'slot')
|
||||
mkdirSync(slot)
|
||||
writeFileSync(join(slot, 'orcad.log'), `${numberedLines(60)}\n`)
|
||||
const run = await runProcess({
|
||||
program: '/bin/sh',
|
||||
args: ['-c', orcadLogTailCommand(linux, slot)],
|
||||
timeoutMs: 5_000
|
||||
})
|
||||
expect(parseOrcadLogTail(linux, run.stdout)?.split('\n').at(-1)).toBe('line 60')
|
||||
const missing = await runProcess({
|
||||
program: '/bin/sh',
|
||||
args: ['-c', orcadLogTailCommand(linux, join(dir, 'absent'))],
|
||||
timeoutMs: 5_000
|
||||
})
|
||||
expect(parseOrcadLogTail(linux, missing.stdout)).toBeNull()
|
||||
}
|
||||
)
|
||||
|
||||
it('reads the end of the log through the real Windows host script op', async () => {
|
||||
const script = join(dir, 'host-script.js')
|
||||
writeFileSync(script, ORCAD_WINDOWS_HOST_SCRIPT)
|
||||
const log = join(dir, 'orcad.log')
|
||||
writeFileSync(log, `${'y'.repeat(ORCAD_LOG_TAIL_MAX_BYTES * 2)}\nfatal: bind failed\n`)
|
||||
const run = await runProcess({
|
||||
program: process.execPath,
|
||||
args: [script, 'log-tail', log, String(ORCAD_LOG_TAIL_MAX_BYTES)],
|
||||
timeoutMs: 15_000
|
||||
})
|
||||
const tail = parseOrcadLogTail(windows, run.stdout)
|
||||
expect(tail?.endsWith('fatal: bind failed')).toBe(true)
|
||||
const absent = await runProcess({
|
||||
program: process.execPath,
|
||||
args: [script, 'log-tail', join(dir, 'absent.log'), '100'],
|
||||
timeoutMs: 15_000
|
||||
})
|
||||
expect(parseOrcadLogTail(windows, absent.stdout)).toBeNull()
|
||||
})
|
||||
|
||||
it('appends the tail to a failure message, and leaves it alone when the host cannot answer', async () => {
|
||||
execCommand.mockResolvedValueOnce('starting\nError: EADDRINUSE\n')
|
||||
await expect(withOrcadLogTail({ conn, host: linux }, SLOT, 'Launch failed.')).resolves.toBe(
|
||||
'Launch failed.\nLast lines of orcad.log:\nstarting\nError: EADDRINUSE'
|
||||
)
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
execCommand.mockRejectedValueOnce(new Error('Connection lost'))
|
||||
await expect(withOrcadLogTail({ conn, host: linux }, SLOT, 'Launch failed.')).resolves.toBe(
|
||||
'Launch failed.'
|
||||
)
|
||||
warn.mockRestore()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,71 @@
|
||||
/**
|
||||
* The end of a slot's orcad.log, appended to a failed deploy, launch or rollback so the error
|
||||
* says why orcad stopped rather than only where its log is. Errors and the main log only; never
|
||||
* telemetry. Best-effort: a host that can't answer leaves the error as it was.
|
||||
*/
|
||||
import { redactString } from '../observability/redactor'
|
||||
import { shellEscape } from './ssh-connection-utils'
|
||||
import { ORCAD_LOG_FILENAME } from './orcad-remote-host-support'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import { isWindowsRemoteHost, joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform'
|
||||
import {
|
||||
orcadWindowsBaseDir,
|
||||
orcadWindowsHostOpCommand,
|
||||
readOrcadWindowsEncodedAnswer
|
||||
} from './orcad-remote-windows-node'
|
||||
import { ORCAD_WINDOWS_LOG_TAIL_MARKER } from './orcad-windows-host-script'
|
||||
import type { OrcadRemoteExecTarget } from './orcad-remote-runtime-control'
|
||||
|
||||
export const ORCAD_LOG_TAIL_LINES = 40
|
||||
export const ORCAD_LOG_TAIL_MAX_BYTES = 8 * 1024
|
||||
// Short, because it only decorates an error the caller already has.
|
||||
const ORCAD_LOG_TAIL_TIMEOUT_MS = 10_000
|
||||
|
||||
export function orcadLogTailCommand(host: RemoteHostPlatform, slotDir: string): string {
|
||||
const log = joinRemotePath(host, slotDir, ORCAD_LOG_FILENAME)
|
||||
if (isWindowsRemoteHost(host)) {
|
||||
return orcadWindowsHostOpCommand(host, orcadWindowsBaseDir(host, slotDir), 'log-tail', [
|
||||
log,
|
||||
String(ORCAD_LOG_TAIL_MAX_BYTES)
|
||||
])
|
||||
}
|
||||
return `tail -c ${ORCAD_LOG_TAIL_MAX_BYTES} ${shellEscape(log)} 2>/dev/null || true`
|
||||
}
|
||||
|
||||
/** The last lines of what the host printed, redacted; null when the log is empty or absent. */
|
||||
export function parseOrcadLogTail(host: RemoteHostPlatform, output: string): string | null {
|
||||
const raw = isWindowsRemoteHost(host)
|
||||
? (readOrcadWindowsEncodedAnswer(output, ORCAD_WINDOWS_LOG_TAIL_MARKER) ?? '')
|
||||
: output
|
||||
const bounded = Buffer.from(raw, 'utf8').subarray(-ORCAD_LOG_TAIL_MAX_BYTES).toString('utf8')
|
||||
const lines = bounded.replace(/\s+$/u, '').split(/\r?\n/u).slice(-ORCAD_LOG_TAIL_LINES)
|
||||
const tail = redactString(lines.join('\n'))
|
||||
return tail.trim() ? tail : null
|
||||
}
|
||||
|
||||
export async function readOrcadLogTail(
|
||||
target: OrcadRemoteExecTarget,
|
||||
slotDir: string
|
||||
): Promise<string | null> {
|
||||
try {
|
||||
// No abort signal: the failure being reported may be the cancellation itself.
|
||||
const output = await execCommand(target.conn, orcadLogTailCommand(target.host, slotDir), {
|
||||
wrapCommand: target.host.commandDialect !== 'powershell',
|
||||
timeoutMs: ORCAD_LOG_TAIL_TIMEOUT_MS
|
||||
})
|
||||
return parseOrcadLogTail(target.host, output)
|
||||
} catch (error) {
|
||||
console.warn('[ssh] Could not read the orcad.log tail:', error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/** `message` followed by the log tail, or `message` alone when the host had none to give. */
|
||||
export async function withOrcadLogTail(
|
||||
target: OrcadRemoteExecTarget,
|
||||
slotDir: string,
|
||||
message: string
|
||||
): Promise<string> {
|
||||
const tail = await readOrcadLogTail(target, slotDir)
|
||||
return tail ? `${message}\nLast lines of orcad.log:\n${tail}` : message
|
||||
}
|
||||
@@ -91,6 +91,9 @@ function scriptHost(log: string[], overrides: HostOverrides = {}): void {
|
||||
const restores = [...(overrides.restores ?? [])]
|
||||
mockExec.mockImplementation(async (_conn, command: string) => {
|
||||
const text = String(command)
|
||||
if (text.startsWith('tail -c')) {
|
||||
return text.includes(TARGET) ? 'orcad: listen EADDRINUSE 127.0.0.1\n' : ''
|
||||
}
|
||||
if (text.includes('__ORCAD_RECORD_PRESENT__')) {
|
||||
return text.includes('transaction.json')
|
||||
? '__ORCAD_RECORD_ABSENT__\n'
|
||||
@@ -249,6 +252,9 @@ describe('rollbackOrcad', () => {
|
||||
const result = await rollbackOrcad(options())
|
||||
expect(result).toMatchObject({ outcome: 'failed', code: 'orcad_activation_no_readiness' })
|
||||
expect(result.outcome === 'failed' && result.reason).toContain('fresh host terminal census')
|
||||
expect(result.outcome === 'failed' && result.reason).toContain(
|
||||
'Last lines of orcad.log:\norcad: listen EADDRINUSE'
|
||||
)
|
||||
expect(log).toEqual([
|
||||
`stop:${ACTIVE}`,
|
||||
'rescue',
|
||||
|
||||
@@ -42,6 +42,7 @@ import {
|
||||
type OrcadSlotIdentity
|
||||
} from './orcad-recovery-slot'
|
||||
import { orcadSnapshotPath, recoverOrcadIncumbent } from './orcad-incumbent-recovery'
|
||||
import { withOrcadLogTail } from './orcad-remote-log-tail'
|
||||
|
||||
/** A confirmed failure reads as no answer; an unconfirmed one propagates and keeps the fence. */
|
||||
async function execOrEmpty(options: OrcadRollbackOptions, command: string): Promise<string> {
|
||||
@@ -235,9 +236,12 @@ export async function rollbackOrcadLocked(
|
||||
return {
|
||||
outcome: 'failed',
|
||||
code: launchError === undefined ? verdict.code : 'orcad_rollback_target_launch_failed',
|
||||
reason:
|
||||
reason: await withOrcadLogTail(
|
||||
options,
|
||||
targetDir,
|
||||
`The rollback target ${safety.target} did not come up healthy: ${reason} ${recovered} ` +
|
||||
`Its stderr is at ${joinRemotePath(options.host, targetDir, ORCAD_LOG_FILENAME)}.`
|
||||
`Its stderr is at ${joinRemotePath(options.host, targetDir, ORCAD_LOG_FILENAME)}.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
/** Which transport each host's managed tunnel last started on, for connect telemetry. */
|
||||
export type OrcadTunnelTransport = 'tcp_forward' | 'stdio_bridge'
|
||||
|
||||
// Keyed by the local target id, which never leaves this process.
|
||||
const transports = new Map<string, OrcadTunnelTransport>()
|
||||
|
||||
export function rememberOrcadTunnelTransport(
|
||||
targetId: string,
|
||||
transport: OrcadTunnelTransport
|
||||
): void {
|
||||
transports.set(targetId, transport)
|
||||
}
|
||||
|
||||
export function knownOrcadTunnelTransport(targetId: string): OrcadTunnelTransport | null {
|
||||
return transports.get(targetId) ?? null
|
||||
}
|
||||
|
||||
export function resetOrcadTunnelTransportMemoForTests(): void {
|
||||
transports.clear()
|
||||
}
|
||||
@@ -40,6 +40,7 @@ export const ORCAD_BUILD_HASH_MARKER = '__ORCAD_BUILD_HASH__'
|
||||
export const ORCAD_WINDOWS_READINESS_MARKER = '__ORCAD_READINESS__'
|
||||
export const ORCAD_WINDOWS_RUNTIME_MARKER = '__ORCAD_RUNTIME__'
|
||||
export const ORCAD_WINDOWS_LIVENESS_MANY_MARKER = '__ORCAD_LIVENESS__'
|
||||
export const ORCAD_WINDOWS_LOG_TAIL_MARKER = '__ORCAD_LOG_TAIL__'
|
||||
/** A slot whose marker names no usable runtime; the POSIX selector exits the same way. */
|
||||
export const ORCAD_WINDOWS_RUNTIME_MISSING_EXIT = 78
|
||||
/** The record exists but cannot be read within its bound. */
|
||||
@@ -50,6 +51,7 @@ export type OrcadWindowsHostOp =
|
||||
| 'liveness-many'
|
||||
| 'stop'
|
||||
| 'readiness-wait'
|
||||
| 'log-tail'
|
||||
| 'build-hash'
|
||||
| 'record-read'
|
||||
| 'record-publish'
|
||||
@@ -163,6 +165,21 @@ const ops = {
|
||||
tick()
|
||||
},
|
||||
|
||||
// The last bytes of a log, as POSIX \`tail -c\` reads them; a missing log is empty.
|
||||
'log-tail'(file, capArg) {
|
||||
const cap = Number(capArg)
|
||||
let fd
|
||||
try { fd = fs.openSync(file, 'r') } catch { return encoded(${text(ORCAD_WINDOWS_LOG_TAIL_MARKER)}, Buffer.alloc(0)) }
|
||||
let buffer = Buffer.alloc(0)
|
||||
try {
|
||||
const size = fs.fstatSync(fd).size
|
||||
const length = Math.min(cap, size)
|
||||
buffer = Buffer.alloc(length)
|
||||
buffer = buffer.subarray(0, fs.readSync(fd, buffer, 0, length, size - length))
|
||||
} catch { buffer = Buffer.alloc(0) } finally { fs.closeSync(fd) }
|
||||
encoded(${text(ORCAD_WINDOWS_LOG_TAIL_MARKER)}, buffer)
|
||||
},
|
||||
|
||||
'build-hash'(file) {
|
||||
const digest = crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex')
|
||||
answer(${text(`${ORCAD_BUILD_HASH_MARKER} `)} + digest.slice(0, 16) + '\\n')
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import { beforeEach, describe, expect, it } from 'vitest'
|
||||
import {
|
||||
knownSshHostPlatform,
|
||||
rememberSshHostPlatform,
|
||||
resetSshHostPlatformMemoForTests
|
||||
} from './ssh-host-platform-memo'
|
||||
import { getRemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
describe('ssh host platform memo', () => {
|
||||
beforeEach(() => {
|
||||
resetSshHostPlatformMemoForTests()
|
||||
})
|
||||
|
||||
it('records the libc flavor, and keeps a probed one when a later report has none', () => {
|
||||
expect(knownSshHostPlatform('ssh-1')).toBeNull()
|
||||
rememberSshHostPlatform('ssh-1', getRemoteHostPlatform('linux-arm64'), 'linux-arm64-musl')
|
||||
expect(knownSshHostPlatform('ssh-1')).toEqual({ os: 'linux', arch: 'arm64', libc: 'musl' })
|
||||
rememberSshHostPlatform('ssh-1', getRemoteHostPlatform('linux-arm64'), null)
|
||||
expect(knownSshHostPlatform('ssh-1')?.libc).toBe('musl')
|
||||
|
||||
rememberSshHostPlatform('ssh-2', getRemoteHostPlatform('win32-x64'), 'win32-x64')
|
||||
expect(knownSshHostPlatform('ssh-2')).toEqual({ os: 'win32', arch: 'x64', libc: 'none' })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,50 @@
|
||||
/**
|
||||
* The platform each SSH host last reported this app session, so connect telemetry can name it
|
||||
* without a probe of its own. Keyed by the local target id, which never leaves this process.
|
||||
*/
|
||||
import type { ServerTarget } from '../../shared/node-runtime-pin'
|
||||
import type { RemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
export type SshHostPlatformFacts = {
|
||||
os: RemoteHostPlatform['os']
|
||||
arch: RemoteHostPlatform['arch']
|
||||
libc: 'glibc' | 'musl' | 'none' | 'unknown'
|
||||
}
|
||||
|
||||
const known = new Map<string, SshHostPlatformFacts>()
|
||||
|
||||
function libcOf(
|
||||
host: RemoteHostPlatform,
|
||||
target: ServerTarget | null
|
||||
): SshHostPlatformFacts['libc'] {
|
||||
if (host.os !== 'linux') {
|
||||
return 'none'
|
||||
}
|
||||
if (!target) {
|
||||
return 'unknown'
|
||||
}
|
||||
return target.endsWith('-musl') ? 'musl' : 'glibc'
|
||||
}
|
||||
|
||||
export function rememberSshHostPlatform(
|
||||
targetId: string,
|
||||
host: RemoteHostPlatform,
|
||||
target: ServerTarget | null
|
||||
): void {
|
||||
const libc = libcOf(host, target)
|
||||
const previous = known.get(targetId)
|
||||
// A libc probe that did not run must not erase one that did.
|
||||
known.set(targetId, {
|
||||
os: host.os,
|
||||
arch: host.arch,
|
||||
libc: libc === 'unknown' && previous ? previous.libc : libc
|
||||
})
|
||||
}
|
||||
|
||||
export function knownSshHostPlatform(targetId: string): SshHostPlatformFacts | null {
|
||||
return known.get(targetId) ?? null
|
||||
}
|
||||
|
||||
export function resetSshHostPlatformMemoForTests(): void {
|
||||
known.clear()
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
/**
|
||||
* What one connect's server decision did, in raw codes; ssh-host-server-telemetry maps these onto
|
||||
* enum-only events. Nothing here carries a message, path or host name.
|
||||
*/
|
||||
import type {
|
||||
OrcadManagedConversionResult,
|
||||
OrcadManagedDeployResult
|
||||
} from '../../shared/orcad-managed-runtime'
|
||||
import type { SshTarget } from '../../shared/ssh-types'
|
||||
import type { HostServerOnConnectResult } from './ssh-host-server-on-connect'
|
||||
import { classifyOrcadHostUnavailable } from './orcad-host-unavailable'
|
||||
|
||||
export type HostServerConnectEvent =
|
||||
| {
|
||||
kind: 'decided'
|
||||
outcome: 'managed' | 'deployed' | 'converted' | 'relay'
|
||||
reason: string
|
||||
refusal: string | null
|
||||
recorded: boolean
|
||||
durationMs: number
|
||||
}
|
||||
| { kind: 'conversion'; phase: 'started' }
|
||||
| { kind: 'conversion'; phase: 'committed'; durationMs: number }
|
||||
| {
|
||||
kind: 'conversion'
|
||||
phase: 'failed'
|
||||
failure: 'deferred' | 'refused' | 'error'
|
||||
code: string | null
|
||||
durationMs: number
|
||||
}
|
||||
| {
|
||||
kind: 'deploy_failed'
|
||||
context: 'deploy' | 'conversion'
|
||||
failure: 'deferred' | 'error'
|
||||
code: string | null
|
||||
durationMs: number
|
||||
}
|
||||
|
||||
/** How far one decision got, filled in as it runs. */
|
||||
export type HostServerDecisionTrace = {
|
||||
startedAt: number
|
||||
path: 'existing' | 'deploy' | 'convert' | null
|
||||
conversionStartedAt: number | null
|
||||
recorded: boolean
|
||||
refusal: string | null
|
||||
}
|
||||
|
||||
export function startHostServerDecisionTrace(now = Date.now()): HostServerDecisionTrace {
|
||||
return { startedAt: now, path: null, conversionStartedAt: null, recorded: false, refusal: null }
|
||||
}
|
||||
|
||||
export function decidedEvent(
|
||||
result: HostServerOnConnectResult | null,
|
||||
trace: HostServerDecisionTrace,
|
||||
now = Date.now()
|
||||
): HostServerConnectEvent {
|
||||
const base = { refusal: null, recorded: trace.recorded, durationMs: now - trace.startedAt }
|
||||
if (!result) {
|
||||
return { kind: 'decided', outcome: 'relay', reason: 'failed', ...base }
|
||||
}
|
||||
if (result.route === 'managed') {
|
||||
if (trace.path === 'deploy') {
|
||||
return { kind: 'decided', outcome: 'deployed', reason: 'deployed', ...base }
|
||||
}
|
||||
if (trace.path === 'convert') {
|
||||
return { kind: 'decided', outcome: 'converted', reason: 'converted', ...base }
|
||||
}
|
||||
return { kind: 'decided', outcome: 'managed', reason: 'connected', ...base }
|
||||
}
|
||||
if (result.reason === 'orcad_unavailable') {
|
||||
// Why the detail: for this reason it is always a classified code, never a message.
|
||||
return { kind: 'decided', outcome: 'relay', reason: result.detail ?? 'other', ...base }
|
||||
}
|
||||
return {
|
||||
kind: 'decided',
|
||||
outcome: 'relay',
|
||||
reason: result.reason,
|
||||
...base,
|
||||
refusal: result.reason === 'refused' ? trace.refusal : null
|
||||
}
|
||||
}
|
||||
|
||||
/** A classified code for a thrown setup failure; null when it carries none. */
|
||||
export function hostServerFailureCode(error: unknown): string | null {
|
||||
const unavailable = classifyOrcadHostUnavailable(error)
|
||||
if (unavailable) {
|
||||
return unavailable
|
||||
}
|
||||
if (error && typeof error === 'object' && 'code' in error && typeof error.code === 'string') {
|
||||
return error.code
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
export type HostServerReport = (target: SshTarget, event: HostServerConnectEvent) => void
|
||||
|
||||
export function reportHostServerEvent(
|
||||
report: HostServerReport,
|
||||
target: SshTarget,
|
||||
event: HostServerConnectEvent
|
||||
): void {
|
||||
try {
|
||||
report(target, event)
|
||||
} catch {
|
||||
// Telemetry never costs a connect.
|
||||
}
|
||||
}
|
||||
|
||||
function sinceMs(startedAt: number | null): number {
|
||||
return startedAt === null ? 0 : Date.now() - startedAt
|
||||
}
|
||||
|
||||
export function reportHostServerSetupError(
|
||||
report: HostServerReport,
|
||||
target: SshTarget,
|
||||
trace: HostServerDecisionTrace,
|
||||
error: unknown
|
||||
): void {
|
||||
const code = hostServerFailureCode(error)
|
||||
if (trace.path === 'deploy') {
|
||||
reportHostServerEvent(report, target, {
|
||||
kind: 'deploy_failed',
|
||||
context: 'deploy',
|
||||
failure: 'error',
|
||||
code,
|
||||
durationMs: sinceMs(trace.startedAt)
|
||||
})
|
||||
} else if (trace.path === 'convert') {
|
||||
reportHostServerEvent(report, target, {
|
||||
kind: 'conversion',
|
||||
phase: 'failed',
|
||||
failure: 'error',
|
||||
code,
|
||||
durationMs: sinceMs(trace.conversionStartedAt)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export function reportHostServerDeploy(
|
||||
report: HostServerReport,
|
||||
target: SshTarget,
|
||||
trace: HostServerDecisionTrace,
|
||||
result: OrcadManagedDeployResult
|
||||
): void {
|
||||
if (result.outcome === 'deferred') {
|
||||
reportHostServerEvent(report, target, {
|
||||
kind: 'deploy_failed',
|
||||
context: 'deploy',
|
||||
failure: 'deferred',
|
||||
code: result.code,
|
||||
durationMs: sinceMs(trace.startedAt)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export function reportHostServerConversion(
|
||||
report: HostServerReport,
|
||||
target: SshTarget,
|
||||
trace: HostServerDecisionTrace,
|
||||
result: OrcadManagedConversionResult
|
||||
): void {
|
||||
const durationMs = sinceMs(trace.conversionStartedAt)
|
||||
if (result.outcome === 'converted') {
|
||||
reportHostServerEvent(report, target, { kind: 'conversion', phase: 'committed', durationMs })
|
||||
return
|
||||
}
|
||||
const failure = result.outcome === 'deferred' ? 'deferred' : 'refused'
|
||||
const code = result.code
|
||||
reportHostServerEvent(report, target, {
|
||||
kind: 'conversion',
|
||||
phase: 'failed',
|
||||
failure,
|
||||
code,
|
||||
durationMs
|
||||
})
|
||||
if (failure === 'deferred') {
|
||||
// The conversion's own deploy is what deferred.
|
||||
reportHostServerEvent(report, target, {
|
||||
kind: 'deploy_failed',
|
||||
context: 'conversion',
|
||||
failure,
|
||||
code,
|
||||
durationMs
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { SshTarget } from '../../shared/ssh-types'
|
||||
import {
|
||||
OrcadHostUnsupportedError,
|
||||
OrcadStdioBridgeUnavailableError
|
||||
} from './orcad-host-unavailable'
|
||||
import type { HostServerConnectEvent } from './ssh-host-server-connect-events'
|
||||
import {
|
||||
resolveHostServerOnConnect,
|
||||
type HostServerOnConnectDeps
|
||||
} from './ssh-host-server-on-connect'
|
||||
|
||||
const target: SshTarget = { id: 'ssh-1', label: 'Box', host: 'box', port: 22, username: 'me' }
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the orchestrator reads only the environment id.
|
||||
const environment = { id: 'env-1' } as never
|
||||
|
||||
function deps(overrides: Partial<HostServerOnConnectDeps> = {}): HostServerOnConnectDeps {
|
||||
return {
|
||||
managedEnvironmentId: () => null,
|
||||
ensureTunnel: vi.fn(async () => undefined),
|
||||
retireRetainedSource: vi.fn(async () => undefined),
|
||||
abandonConversion: vi.fn(async () => undefined),
|
||||
abandonDeploy: vi.fn(async () => undefined),
|
||||
hasTemplate: () => true,
|
||||
recordedUnavailable: () => null,
|
||||
recordUnavailable: vi.fn(),
|
||||
isEmptyHost: () => false,
|
||||
relayTerminals: vi.fn(async () => ({ verdict: 'exited' as const, count: 0 })),
|
||||
deploy: vi.fn(async () => ({ outcome: 'created' as const, environment, activeVersion: '1' })),
|
||||
convert: vi.fn(async () => ({ outcome: 'converted' as const, environment, migrationId: 'm' })),
|
||||
progress: vi.fn(),
|
||||
isFencedBeforeStaging: () => false,
|
||||
releaseUnreachableSetup: vi.fn(async () => undefined),
|
||||
report: vi.fn(),
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
async function eventsOf(d: HostServerOnConnectDeps): Promise<HostServerConnectEvent[]> {
|
||||
await resolveHostServerOnConnect(target, d).catch(() => undefined)
|
||||
return vi.mocked(d.report).mock.calls.map(([, event]) => event)
|
||||
}
|
||||
|
||||
function decided(events: HostServerConnectEvent[]) {
|
||||
const found = events.filter((event) => event.kind === 'decided')
|
||||
expect(found).toHaveLength(1)
|
||||
return found[0]
|
||||
}
|
||||
|
||||
describe('connect-decision telemetry', () => {
|
||||
it('reports exactly one decision per connect, naming the outcome', async () => {
|
||||
const managed = await eventsOf(deps({ managedEnvironmentId: () => 'env-9' }))
|
||||
expect(decided(managed)).toMatchObject({ outcome: 'managed', reason: 'connected' })
|
||||
|
||||
const deployed = await eventsOf(deps({ isEmptyHost: () => true }))
|
||||
expect(decided(deployed)).toMatchObject({ outcome: 'deployed', reason: 'deployed' })
|
||||
|
||||
const converted = await eventsOf(deps())
|
||||
expect(decided(converted)).toMatchObject({ outcome: 'converted', reason: 'converted' })
|
||||
expect(converted.filter((event) => event.kind === 'conversion')).toEqual([
|
||||
{ kind: 'conversion', phase: 'started' },
|
||||
expect.objectContaining({ kind: 'conversion', phase: 'committed' })
|
||||
])
|
||||
})
|
||||
|
||||
it('names why a host stays on the relay', async () => {
|
||||
const live = await eventsOf(
|
||||
deps({ relayTerminals: async () => ({ verdict: 'live', count: 1 }) })
|
||||
)
|
||||
expect(decided(live)).toMatchObject({ outcome: 'relay', reason: 'relay_terminals_live' })
|
||||
|
||||
const noTunnel = await eventsOf(
|
||||
deps({
|
||||
managedEnvironmentId: () => 'env-9',
|
||||
isFencedBeforeStaging: () => true,
|
||||
ensureTunnel: async () => {
|
||||
throw new OrcadStdioBridgeUnavailableError('no bridge')
|
||||
}
|
||||
})
|
||||
)
|
||||
expect(decided(noTunnel)).toMatchObject({ outcome: 'relay', reason: 'ssh_tunnel_unavailable' })
|
||||
|
||||
const recorded = await eventsOf(deps({ recordedUnavailable: () => 'runtime_self_test' }))
|
||||
expect(decided(recorded)).toMatchObject({ reason: 'runtime_self_test', recorded: true })
|
||||
|
||||
const noTemplate = await eventsOf(deps({ hasTemplate: () => false }))
|
||||
expect(decided(noTemplate)).toMatchObject({ reason: 'artifacts_unavailable' })
|
||||
})
|
||||
|
||||
it('reports a refused conversion with its blocker code', async () => {
|
||||
const events = await eventsOf(
|
||||
deps({
|
||||
convert: async () => ({
|
||||
outcome: 'refused',
|
||||
verdict: 'live',
|
||||
code: 'orcad_migration_preflight_blocked',
|
||||
reason: 'An automation still runs on this host.'
|
||||
})
|
||||
})
|
||||
)
|
||||
expect(decided(events)).toMatchObject({
|
||||
reason: 'refused',
|
||||
refusal: 'orcad_migration_preflight_blocked'
|
||||
})
|
||||
expect(events).toContainEqual(
|
||||
expect.objectContaining({ kind: 'conversion', phase: 'failed', failure: 'refused' })
|
||||
)
|
||||
})
|
||||
|
||||
it('reports deploy failures, deferred or thrown, with their classified code', async () => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const deferred = await eventsOf(
|
||||
deps({
|
||||
isEmptyHost: () => true,
|
||||
deploy: async () => ({
|
||||
outcome: 'deferred',
|
||||
candidateVersion: '1.0.0',
|
||||
code: 'orcad_activation_no_readiness',
|
||||
reason: 'No readiness line.\nLast lines of orcad.log:\nboom'
|
||||
})
|
||||
})
|
||||
)
|
||||
expect(deferred).toContainEqual(
|
||||
expect.objectContaining({
|
||||
kind: 'deploy_failed',
|
||||
context: 'deploy',
|
||||
failure: 'deferred',
|
||||
code: 'orcad_activation_no_readiness'
|
||||
})
|
||||
)
|
||||
expect(decided(deferred)).toMatchObject({ outcome: 'relay', reason: 'deferred' })
|
||||
|
||||
const thrown = await eventsOf(
|
||||
deps({
|
||||
isEmptyHost: () => true,
|
||||
deploy: async () => {
|
||||
throw new OrcadHostUnsupportedError('no build for this host')
|
||||
}
|
||||
})
|
||||
)
|
||||
expect(thrown).toContainEqual(
|
||||
expect.objectContaining({ kind: 'deploy_failed', failure: 'error', code: 'unsupported_host' })
|
||||
)
|
||||
expect(decided(thrown)).toMatchObject({ reason: 'unsupported_host' })
|
||||
|
||||
const conversion = await eventsOf(
|
||||
deps({
|
||||
convert: async () => ({
|
||||
outcome: 'deferred',
|
||||
candidateVersion: '1.0.0',
|
||||
code: 'orcad_update_terminals_running',
|
||||
reason: 'Terminals are running.'
|
||||
})
|
||||
})
|
||||
)
|
||||
expect(conversion).toContainEqual(
|
||||
expect.objectContaining({ kind: 'conversion', phase: 'failed', failure: 'deferred' })
|
||||
)
|
||||
expect(conversion).toContainEqual(
|
||||
expect.objectContaining({ kind: 'deploy_failed', context: 'conversion' })
|
||||
)
|
||||
warn.mockRestore()
|
||||
})
|
||||
|
||||
it('shows a deferral reason, log tail included, under the SSH host', async () => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const reason = 'No readiness line.\nLast lines of orcad.log:\nError: EADDRINUSE'
|
||||
const d = deps({
|
||||
isEmptyHost: () => true,
|
||||
deploy: async () => ({
|
||||
outcome: 'deferred',
|
||||
candidateVersion: '1.0.0',
|
||||
code: 'orcad_activation_no_readiness',
|
||||
reason
|
||||
})
|
||||
})
|
||||
await expect(resolveHostServerOnConnect(target, d)).resolves.toEqual({
|
||||
route: 'relay',
|
||||
reason: 'deferred',
|
||||
detail: reason
|
||||
})
|
||||
expect(warn).toHaveBeenCalledWith(expect.stringContaining('deferred'), reason)
|
||||
warn.mockRestore()
|
||||
})
|
||||
|
||||
it('still reports, as failed, a decision that throws, and a throwing reporter never fails a connect', async () => {
|
||||
const d = deps({
|
||||
managedEnvironmentId: () => 'env-9',
|
||||
ensureTunnel: async () => {
|
||||
throw new Error('tunnel failed')
|
||||
}
|
||||
})
|
||||
expect(decided(await eventsOf(d))).toMatchObject({ outcome: 'relay', reason: 'failed' })
|
||||
|
||||
const throwing = deps({
|
||||
report: () => {
|
||||
throw new Error('telemetry broke')
|
||||
}
|
||||
})
|
||||
await expect(resolveHostServerOnConnect(target, throwing)).resolves.toMatchObject({
|
||||
route: 'managed'
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -30,6 +30,7 @@ function deps(overrides: Partial<HostServerOnConnectDeps> = {}): HostServerOnCon
|
||||
progress: vi.fn(),
|
||||
isFencedBeforeStaging: () => false,
|
||||
releaseUnreachableSetup: vi.fn(async () => undefined),
|
||||
report: vi.fn(),
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
@@ -187,7 +188,8 @@ describe('which server an SSH host runs on connect', () => {
|
||||
})
|
||||
await expect(resolveHostServerOnConnect(target, d)).resolves.toEqual({
|
||||
route: 'relay',
|
||||
reason: 'failed'
|
||||
reason: 'failed',
|
||||
detail: 'Connection lost'
|
||||
})
|
||||
expect(d.recordUnavailable).not.toHaveBeenCalled()
|
||||
warn.mockRestore()
|
||||
|
||||
@@ -17,6 +17,16 @@ import {
|
||||
classifyOrcadHostUnavailable,
|
||||
ORCAD_TUNNEL_UNAVAILABLE_REASON
|
||||
} from './orcad-host-unavailable'
|
||||
import {
|
||||
decidedEvent,
|
||||
reportHostServerConversion,
|
||||
reportHostServerDeploy,
|
||||
reportHostServerEvent,
|
||||
reportHostServerSetupError,
|
||||
startHostServerDecisionTrace,
|
||||
type HostServerDecisionTrace,
|
||||
type HostServerReport
|
||||
} from './ssh-host-server-connect-events'
|
||||
|
||||
export type HostServerPhase = 'deploying' | 'converting' | 'connecting'
|
||||
|
||||
@@ -60,17 +70,38 @@ export type HostServerOnConnectDeps = {
|
||||
isFencedBeforeStaging: (target: SshTarget) => boolean
|
||||
/** Unregisters that unreachable server and releases its fence, so the relay serves again. */
|
||||
releaseUnreachableSetup: (target: SshTarget) => Promise<void>
|
||||
/** Receives each decision, conversion and deploy failure as raw codes for telemetry. */
|
||||
report: HostServerReport
|
||||
}
|
||||
|
||||
// Bounds a failure's detail; the host log tail inside it is already capped.
|
||||
const MAX_FAILURE_DETAIL_CHARS = 16_000
|
||||
|
||||
export async function resolveHostServerOnConnect(
|
||||
target: SshTarget,
|
||||
deps: HostServerOnConnectDeps
|
||||
): Promise<HostServerOnConnectResult> {
|
||||
const trace = startHostServerDecisionTrace()
|
||||
let result: HostServerOnConnectResult | null = null
|
||||
try {
|
||||
result = await decide(target, deps, trace)
|
||||
return result
|
||||
} finally {
|
||||
reportHostServerEvent(deps.report, target, decidedEvent(result, trace))
|
||||
}
|
||||
}
|
||||
|
||||
async function decide(
|
||||
target: SshTarget,
|
||||
deps: HostServerOnConnectDeps,
|
||||
trace: HostServerDecisionTrace
|
||||
): Promise<HostServerOnConnectResult> {
|
||||
if (target.orcadFence?.sourceChangedAt) {
|
||||
return { route: 'relay', reason: 'source_changed' }
|
||||
}
|
||||
const existing = deps.managedEnvironmentId(target)
|
||||
if (existing) {
|
||||
trace.path = 'existing'
|
||||
deps.progress(target, 'connecting')
|
||||
try {
|
||||
await deps.ensureTunnel(existing)
|
||||
@@ -97,6 +128,7 @@ export async function resolveHostServerOnConnect(
|
||||
}
|
||||
const recorded = deps.recordedUnavailable(target)
|
||||
if (recorded) {
|
||||
trace.recorded = true
|
||||
return { route: 'relay', reason: 'orcad_unavailable', detail: recorded }
|
||||
}
|
||||
if (!deps.hasTemplate()) {
|
||||
@@ -105,8 +137,11 @@ export async function resolveHostServerOnConnect(
|
||||
const empty = deps.isEmptyHost(target)
|
||||
try {
|
||||
if (empty) {
|
||||
trace.path = 'deploy'
|
||||
deps.progress(target, 'deploying')
|
||||
return await afterDeploy(target, deps, await deps.deploy(target))
|
||||
const deployed = await deps.deploy(target)
|
||||
reportHostServerDeploy(deps.report, target, trace, deployed)
|
||||
return await afterDeploy(target, deps, deployed)
|
||||
}
|
||||
const terminals = await deps.relayTerminals(target)
|
||||
if (terminals.verdict !== 'exited') {
|
||||
@@ -117,11 +152,22 @@ export async function resolveHostServerOnConnect(
|
||||
terminals: terminals.count
|
||||
}
|
||||
}
|
||||
trace.path = 'convert'
|
||||
trace.conversionStartedAt = Date.now()
|
||||
reportHostServerEvent(deps.report, target, { kind: 'conversion', phase: 'started' })
|
||||
deps.progress(target, 'converting')
|
||||
return await afterConversion(target, deps, await deps.convert(target))
|
||||
const converted = await deps.convert(target)
|
||||
if (converted.outcome === 'refused') {
|
||||
trace.refusal = converted.code
|
||||
}
|
||||
reportHostServerConversion(deps.report, target, trace, converted)
|
||||
return await afterConversion(target, deps, converted)
|
||||
} catch (error) {
|
||||
console.warn('[ssh] Managed Orca server setup failed; using the relay this session:', error)
|
||||
return unfinished(target, deps, classifyOrcadHostUnavailable(error), empty, 'failed')
|
||||
reportHostServerSetupError(deps.report, target, trace, error)
|
||||
const unavailable = classifyOrcadHostUnavailable(error)
|
||||
const detail = error instanceof Error ? error.message : String(error)
|
||||
return unfinished(target, deps, unavailable, empty, 'failed', detail)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,7 +181,7 @@ async function afterDeploy(
|
||||
result: OrcadManagedDeployResult
|
||||
): Promise<HostServerOnConnectResult> {
|
||||
if (result.outcome === 'deferred') {
|
||||
return deferred(target, deps, result.code, true)
|
||||
return deferred(target, deps, result, true)
|
||||
}
|
||||
return { route: 'managed', environmentId: result.environment.id }
|
||||
}
|
||||
@@ -149,7 +195,7 @@ async function afterConversion(
|
||||
case 'converted':
|
||||
return { route: 'managed', environmentId: result.environment.id }
|
||||
case 'deferred':
|
||||
return deferred(target, deps, result.code, false)
|
||||
return deferred(target, deps, result, false)
|
||||
case 'refused':
|
||||
if (result.code === 'orcad_migration_terminals') {
|
||||
return { route: 'relay', reason: terminalReason(result.verdict) }
|
||||
@@ -161,10 +207,12 @@ async function afterConversion(
|
||||
function deferred(
|
||||
target: SshTarget,
|
||||
deps: HostServerOnConnectDeps,
|
||||
code: string,
|
||||
result: { code: string; reason: string },
|
||||
empty: boolean
|
||||
): Promise<HostServerOnConnectResult> {
|
||||
return unfinished(target, deps, classifyOrcadHostUnavailable({ code }), empty, 'deferred')
|
||||
console.warn(`[ssh] Managed Orca server setup deferred (${result.code}):`, result.reason)
|
||||
const unavailable = classifyOrcadHostUnavailable({ code: result.code })
|
||||
return unfinished(target, deps, unavailable, empty, 'deferred', result.reason)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -176,7 +224,8 @@ async function unfinished(
|
||||
deps: HostServerOnConnectDeps,
|
||||
unavailable: string | null,
|
||||
empty: boolean,
|
||||
reason: 'deferred' | 'failed'
|
||||
reason: 'deferred' | 'failed',
|
||||
failure: string
|
||||
): Promise<HostServerOnConnectResult> {
|
||||
try {
|
||||
await (empty ? deps.abandonDeploy(target) : deps.abandonConversion(target))
|
||||
@@ -185,8 +234,13 @@ async function unfinished(
|
||||
console.warn('[ssh] The managed Orca server setup keeps its fence:', error)
|
||||
}
|
||||
if (!unavailable) {
|
||||
return { route: 'relay', reason }
|
||||
// Shown under the SSH host, with the host's orcad.log tail when the failure carries one.
|
||||
return { route: 'relay', reason, detail: boundedDetail(failure) }
|
||||
}
|
||||
deps.recordUnavailable(target, unavailable)
|
||||
return { route: 'relay', reason: 'orcad_unavailable', detail: unavailable }
|
||||
}
|
||||
|
||||
function boundedDetail(text: string): string {
|
||||
return text.length > MAX_FAILURE_DETAIL_CHARS ? `…${text.slice(-MAX_FAILURE_DETAIL_CHARS)}` : text
|
||||
}
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { eventSchemas } from '../../shared/telemetry-event-registry'
|
||||
|
||||
vi.mock('../telemetry/client', () => ({ track: vi.fn() }))
|
||||
|
||||
const { track } = await import('../telemetry/client')
|
||||
const { hostServerFailure, hostServerRefusal, trackSshHostServerEvent, trackSshHostServerMove } =
|
||||
await import('./ssh-host-server-telemetry')
|
||||
|
||||
const linuxGlibc = { os: 'linux', arch: 'x64', libc: 'glibc' } as const
|
||||
|
||||
function tracked() {
|
||||
const call = vi.mocked(track).mock.calls.at(-1)
|
||||
if (!call) {
|
||||
throw new Error('nothing was tracked')
|
||||
}
|
||||
return call
|
||||
}
|
||||
|
||||
function expectSchemaValid(): unknown {
|
||||
const [name, props] = tracked()
|
||||
expect(eventSchemas[name].safeParse(props).success).toBe(true)
|
||||
return props
|
||||
}
|
||||
|
||||
describe('ssh host server telemetry', () => {
|
||||
beforeEach(() => {
|
||||
vi.mocked(track).mockReset()
|
||||
})
|
||||
|
||||
it('maps a decision onto enum-only props the strict schema accepts', () => {
|
||||
trackSshHostServerEvent(
|
||||
{
|
||||
kind: 'decided',
|
||||
outcome: 'relay',
|
||||
reason: 'refused',
|
||||
refusal: 'orcad_migration_preflight_blocked',
|
||||
recorded: false,
|
||||
durationMs: 7_000
|
||||
},
|
||||
linuxGlibc
|
||||
)
|
||||
expect(tracked()[0]).toBe('ssh_host_server_decided')
|
||||
expect(expectSchemaValid()).toEqual({
|
||||
outcome: 'relay',
|
||||
transport: 'none',
|
||||
reason: 'refused',
|
||||
refusal: 'preflight_blocked',
|
||||
recorded: false,
|
||||
host_os: 'linux',
|
||||
host_arch: 'x64',
|
||||
host_libc: 'glibc',
|
||||
duration_bucket: '5s_15s'
|
||||
})
|
||||
})
|
||||
|
||||
it('reports an unprobed host as unknown rather than guessing', () => {
|
||||
trackSshHostServerEvent(
|
||||
{
|
||||
kind: 'decided',
|
||||
outcome: 'managed',
|
||||
reason: 'connected',
|
||||
refusal: null,
|
||||
recorded: false,
|
||||
durationMs: 10
|
||||
},
|
||||
null
|
||||
)
|
||||
expect(expectSchemaValid()).toMatchObject({
|
||||
host_os: 'unknown',
|
||||
host_arch: 'unknown',
|
||||
host_libc: 'unknown'
|
||||
})
|
||||
})
|
||||
|
||||
it('names the transport a managed connect used, and none for the relay', () => {
|
||||
const decision = {
|
||||
kind: 'decided',
|
||||
outcome: 'managed',
|
||||
reason: 'connected',
|
||||
refusal: null,
|
||||
recorded: false,
|
||||
durationMs: 10
|
||||
} as const
|
||||
trackSshHostServerEvent(decision, linuxGlibc, 'stdio_bridge')
|
||||
expect(expectSchemaValid()).toMatchObject({ transport: 'stdio_bridge' })
|
||||
trackSshHostServerEvent(decision, linuxGlibc)
|
||||
expect(expectSchemaValid()).toMatchObject({ transport: 'unknown' })
|
||||
trackSshHostServerEvent(
|
||||
{ ...decision, outcome: 'relay', reason: 'ssh_tunnel_unavailable' },
|
||||
linuxGlibc,
|
||||
'tcp_forward'
|
||||
)
|
||||
expect(expectSchemaValid()).toMatchObject({
|
||||
transport: 'none',
|
||||
reason: 'ssh_tunnel_unavailable'
|
||||
})
|
||||
})
|
||||
|
||||
it('reports the per-host move offer and each move result', () => {
|
||||
for (const outcome of ['offered', 'moved', 'refused_live', 'failed'] as const) {
|
||||
trackSshHostServerMove(outcome, linuxGlibc)
|
||||
expect(tracked()[0]).toBe('ssh_host_server_move')
|
||||
expect(expectSchemaValid()).toEqual({
|
||||
outcome,
|
||||
host_os: 'linux',
|
||||
host_arch: 'x64',
|
||||
host_libc: 'glibc'
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
it('narrows any unrecognised code, so free text never reaches an event', () => {
|
||||
expect(hostServerRefusal('orcad_migration_/home/me/secret')).toBe('other')
|
||||
expect(hostServerFailure('Error: ssh me@host.example failed')).toBe('other')
|
||||
expect(hostServerFailure('orcad_activation_no_readiness')).toBe('activation_no_readiness')
|
||||
expect(hostServerFailure('unsupported_host')).toBe('unsupported_host')
|
||||
|
||||
trackSshHostServerEvent(
|
||||
{
|
||||
kind: 'decided',
|
||||
outcome: 'relay',
|
||||
reason: 'host.example.com',
|
||||
refusal: null,
|
||||
recorded: false,
|
||||
durationMs: 0
|
||||
},
|
||||
linuxGlibc
|
||||
)
|
||||
expect(expectSchemaValid()).toMatchObject({ reason: 'other' })
|
||||
})
|
||||
|
||||
it('maps conversion phases and deploy failures', () => {
|
||||
trackSshHostServerEvent({ kind: 'conversion', phase: 'started' }, linuxGlibc)
|
||||
expect(expectSchemaValid()).toMatchObject({ phase: 'started', failure: 'none' })
|
||||
|
||||
trackSshHostServerEvent(
|
||||
{ kind: 'conversion', phase: 'committed', durationMs: 90_000 },
|
||||
linuxGlibc
|
||||
)
|
||||
expect(expectSchemaValid()).toMatchObject({ phase: 'committed', duration_bucket: 'gte_60s' })
|
||||
|
||||
trackSshHostServerEvent(
|
||||
{
|
||||
kind: 'conversion',
|
||||
phase: 'failed',
|
||||
failure: 'refused',
|
||||
code: 'orcad_migration_terminals',
|
||||
durationMs: 100
|
||||
},
|
||||
linuxGlibc
|
||||
)
|
||||
expect(expectSchemaValid()).toMatchObject({ refusal: 'terminals', failure_code: 'none' })
|
||||
|
||||
trackSshHostServerEvent(
|
||||
{
|
||||
kind: 'deploy_failed',
|
||||
context: 'deploy',
|
||||
failure: 'deferred',
|
||||
code: 'orcad_candidate_preflight_failed',
|
||||
durationMs: 20_000
|
||||
},
|
||||
{ os: 'linux', arch: 'arm64', libc: 'musl' }
|
||||
)
|
||||
expect(tracked()[0]).toBe('ssh_host_server_deploy_failed')
|
||||
expect(expectSchemaValid()).toEqual({
|
||||
context: 'deploy',
|
||||
failure: 'deferred',
|
||||
failure_code: 'candidate_preflight_failed',
|
||||
host_os: 'linux',
|
||||
host_arch: 'arm64',
|
||||
host_libc: 'musl',
|
||||
duration_bucket: '15s_60s'
|
||||
})
|
||||
})
|
||||
|
||||
it('never throws into a connect when tracking fails', () => {
|
||||
vi.mocked(track).mockImplementation(() => {
|
||||
throw new Error('invalid')
|
||||
})
|
||||
trackSshHostServerEvent({ kind: 'conversion', phase: 'started' }, null)
|
||||
expect(track).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* Connect-decision telemetry: which server each SSH connect chose and why, plus conversion and
|
||||
* deploy failures. Every raw code is narrowed onto a fixed enum here, so nothing a host or user
|
||||
* named can reach an event; unrecognised codes become `other`.
|
||||
*/
|
||||
import type { EventProps } from '../../shared/telemetry-events'
|
||||
import {
|
||||
SSH_HOST_SERVER_FAILURE_VALUES,
|
||||
SSH_HOST_SERVER_REASON_VALUES,
|
||||
SSH_HOST_SERVER_REFUSAL_VALUES
|
||||
} from '../../shared/telemetry-ssh-host-server-event-schemas'
|
||||
import { track } from '../telemetry/client'
|
||||
import type { HostServerConnectEvent } from './ssh-host-server-connect-events'
|
||||
import type { SshHostPlatformFacts } from './ssh-host-platform-memo'
|
||||
import type { OrcadTunnelTransport } from './orcad-tunnel-transport-memo'
|
||||
import { durationBucket } from './ssh-remote-runtime-telemetry'
|
||||
|
||||
type HostProps = Pick<EventProps<'ssh_host_server_decided'>, 'host_os' | 'host_arch' | 'host_libc'>
|
||||
type FailureCode = (typeof SSH_HOST_SERVER_FAILURE_VALUES)[number]
|
||||
type RefusalCode = (typeof SSH_HOST_SERVER_REFUSAL_VALUES)[number]
|
||||
|
||||
function pick<T extends string>(values: readonly T[], value: string, fallback: T): T {
|
||||
return values.find((candidate) => candidate === value) ?? fallback
|
||||
}
|
||||
|
||||
function hostProps(facts: SshHostPlatformFacts | null): HostProps {
|
||||
return facts
|
||||
? { host_os: facts.os, host_arch: facts.arch, host_libc: facts.libc }
|
||||
: { host_os: 'unknown', host_arch: 'unknown', host_libc: 'unknown' }
|
||||
}
|
||||
|
||||
export function hostServerRefusal(code: string | null): RefusalCode {
|
||||
return code
|
||||
? pick(SSH_HOST_SERVER_REFUSAL_VALUES, code.replace(/^orcad_migration_/u, ''), 'other')
|
||||
: 'none'
|
||||
}
|
||||
|
||||
export function hostServerFailure(code: string | null): FailureCode {
|
||||
return code
|
||||
? pick(SSH_HOST_SERVER_FAILURE_VALUES, code.replace(/^orcad_/u, ''), 'other')
|
||||
: 'other'
|
||||
}
|
||||
|
||||
export type SshHostServerMoveOutcome = EventProps<'ssh_host_server_move'>['outcome']
|
||||
|
||||
export function trackSshHostServerMove(
|
||||
outcome: SshHostServerMoveOutcome,
|
||||
facts: SshHostPlatformFacts | null
|
||||
): void {
|
||||
try {
|
||||
track('ssh_host_server_move', { outcome, ...hostProps(facts) })
|
||||
} catch {
|
||||
// Telemetry never costs a move.
|
||||
}
|
||||
}
|
||||
|
||||
export function trackSshHostServerEvent(
|
||||
event: HostServerConnectEvent,
|
||||
facts: SshHostPlatformFacts | null,
|
||||
transport: OrcadTunnelTransport | null = null
|
||||
): void {
|
||||
const host = hostProps(facts)
|
||||
try {
|
||||
// track() applies the consent gate and validates against the strict schema.
|
||||
switch (event.kind) {
|
||||
case 'decided':
|
||||
track('ssh_host_server_decided', {
|
||||
outcome: event.outcome,
|
||||
transport: event.outcome === 'relay' ? 'none' : (transport ?? 'unknown'),
|
||||
reason: pick(SSH_HOST_SERVER_REASON_VALUES, event.reason, 'other'),
|
||||
refusal: hostServerRefusal(event.refusal),
|
||||
recorded: event.recorded,
|
||||
...host,
|
||||
duration_bucket: durationBucket(event.durationMs)
|
||||
})
|
||||
return
|
||||
case 'conversion': {
|
||||
const failed = event.phase === 'failed'
|
||||
track('ssh_host_server_conversion', {
|
||||
phase: event.phase,
|
||||
failure: failed ? event.failure : 'none',
|
||||
refusal: failed && event.failure === 'refused' ? hostServerRefusal(event.code) : 'none',
|
||||
failure_code:
|
||||
failed && event.failure !== 'refused' ? hostServerFailure(event.code) : 'none',
|
||||
...host,
|
||||
duration_bucket: durationBucket(event.phase === 'started' ? 0 : event.durationMs)
|
||||
})
|
||||
return
|
||||
}
|
||||
case 'deploy_failed':
|
||||
track('ssh_host_server_deploy_failed', {
|
||||
context: event.context,
|
||||
failure: event.failure,
|
||||
failure_code: hostServerFailure(event.code),
|
||||
...host,
|
||||
duration_bucket: durationBucket(event.durationMs)
|
||||
})
|
||||
}
|
||||
} catch {
|
||||
// Telemetry never costs a connect.
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
type RemoteRuntimeUnavailableReason
|
||||
} from './ssh-relay-runtime-ladder'
|
||||
import type { PinnedRuntimeRefusal } from './ssh-relay-runtime-self-test'
|
||||
import { rememberSshHostPlatform } from './ssh-host-platform-memo'
|
||||
import type { RemoteHostPlatform } from './ssh-remote-platform'
|
||||
import type { HostNodeVersion } from './ssh-remote-node-toolchain-probe'
|
||||
import {
|
||||
@@ -141,6 +142,7 @@ export class RelayRuntimeLadderRun {
|
||||
|
||||
private track(rung: SshRemoteRuntimeRung, outcome: SshRemoteRuntimeOutcome): void {
|
||||
if (this.host) {
|
||||
rememberSshHostPlatform(this.targetId, this.host, this.facts?.target ?? null)
|
||||
trackSshRemoteRuntimeResolved(this.targetId, {
|
||||
rung,
|
||||
outcome,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { ArrowRightLeft } from 'lucide-react'
|
||||
import { ArrowRightLeft, ChevronDown } from 'lucide-react'
|
||||
import { useState } from 'react'
|
||||
import type { SshTarget } from '../../../../shared/ssh-types'
|
||||
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible'
|
||||
import { translate } from '@/i18n/i18n'
|
||||
import { cn } from '@/lib/utils'
|
||||
import { canMoveSshHostToManagedServer } from '@/ssh/ssh-managed-server-move'
|
||||
@@ -47,6 +48,21 @@ export function SshTargetServerStatus({
|
||||
</Button>
|
||||
) : null}
|
||||
</div>
|
||||
{line.detail ? (
|
||||
<Collapsible>
|
||||
<CollapsibleTrigger asChild>
|
||||
<Button type="button" variant="ghost" size="xs" className="group">
|
||||
{translate('auto.components.settings.sshHostServer.failureDetails', 'Details')}
|
||||
<ChevronDown className="transition-transform group-data-[state=open]:rotate-180" />
|
||||
</Button>
|
||||
</CollapsibleTrigger>
|
||||
<CollapsibleContent>
|
||||
<pre className="scrollbar-sleek max-h-72 overflow-auto whitespace-pre-wrap break-words rounded-md border border-border bg-background px-3 py-2 font-mono text-[11px] leading-relaxed text-muted-foreground">
|
||||
{line.detail}
|
||||
</pre>
|
||||
</CollapsibleContent>
|
||||
</Collapsible>
|
||||
) : null}
|
||||
<SshHostChangedActions target={target} onChanged={onChanged} />
|
||||
{canMove ? (
|
||||
<SshManagedServerMoveDialog
|
||||
|
||||
@@ -47,6 +47,18 @@ describe('SSH host server status line', () => {
|
||||
).not.toHaveProperty('action')
|
||||
})
|
||||
|
||||
it('offers the setup failure, log tail included, beside a retry line', () => {
|
||||
const detail = 'No readiness line.\nLast lines of orcad.log:\nError: EADDRINUSE'
|
||||
expect(
|
||||
sshHostServerStatusLine(plain, {
|
||||
managedServer: { kind: 'relay', reason: 'deferred', detail }
|
||||
})
|
||||
).toMatchObject({ tone: 'muted', detail })
|
||||
expect(
|
||||
sshHostServerStatusLine(plain, { managedServer: { kind: 'relay', reason: 'failed' } })
|
||||
).not.toHaveProperty('detail')
|
||||
})
|
||||
|
||||
it('keeps durable reasons visible without a live state', () => {
|
||||
expect(sshHostServerStatusLine({ orcadFence: { environmentId: 'e' } }, undefined)?.text).toBe(
|
||||
'Runs a managed Orca server'
|
||||
|
||||
@@ -7,6 +7,8 @@ export type SshHostServerStatusLine = {
|
||||
tone: 'muted' | 'warning' | 'destructive'
|
||||
/** Offers "Move to managed server", which restarts the live relay terminals. */
|
||||
action?: 'move'
|
||||
/** Why setup stopped, with the host's orcad.log tail when there is one; shown on request. */
|
||||
detail?: string
|
||||
}
|
||||
|
||||
export function sshHostServerStatusLine(
|
||||
@@ -156,6 +158,6 @@ function relayLine(
|
||||
}
|
||||
case 'deferred':
|
||||
case 'failed':
|
||||
return retryLine()
|
||||
return { ...retryLine(), ...(status.detail ? { detail: status.detail } : {}) }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12411,7 +12411,8 @@
|
||||
"terminalsLive": "Runs the relay until its {{count}} open terminals are closed, then moves to a managed server.",
|
||||
"terminalsUnverifiable": "Runs the relay: Orca couldn’t confirm its terminals are closed. It moves on a later connect.",
|
||||
"refused": "Not moved to a managed server: {{blocker}}",
|
||||
"retry": "Runs the relay this session; it moves to a managed server on a later connect."
|
||||
"retry": "Runs the relay this session; it moves to a managed server on a later connect.",
|
||||
"failureDetails": "Details"
|
||||
},
|
||||
"ZcodePlanAccountsSection": {
|
||||
"site": {
|
||||
|
||||
@@ -97,6 +97,12 @@ import {
|
||||
workspaceCreateFailedSchema
|
||||
} from './telemetry-repository-event-schemas'
|
||||
import { sshRemoteRuntimeResolvedSchema } from './telemetry-ssh-runtime-event-schemas'
|
||||
import {
|
||||
sshHostServerConversionSchema,
|
||||
sshHostServerDecidedSchema,
|
||||
sshHostServerDeployFailedSchema,
|
||||
sshHostServerMoveSchema
|
||||
} from './telemetry-ssh-host-server-event-schemas'
|
||||
|
||||
// ── Event registry: the one record the validator consumes ───────────────
|
||||
// Versioning: breaking changes (rename/re-mean/remove a key) need a new event name; in-place edits blend pre/post rows unmixably. Additive-optional fields are safe.
|
||||
@@ -199,6 +205,10 @@ export const eventSchemas = {
|
||||
|
||||
direct_ssh_reconnect_operation: directSshReconnectOperationSchema,
|
||||
ssh_remote_runtime_resolved: sshRemoteRuntimeResolvedSchema,
|
||||
ssh_host_server_decided: sshHostServerDecidedSchema,
|
||||
ssh_host_server_conversion: sshHostServerConversionSchema,
|
||||
ssh_host_server_deploy_failed: sshHostServerDeployFailedSchema,
|
||||
ssh_host_server_move: sshHostServerMoveSchema,
|
||||
|
||||
smart_sort_class_distribution: smartSortClassDistributionSchema,
|
||||
smart_sort_class_1_promotion: smartSortClass1PromotionSchema,
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
import { z } from 'zod'
|
||||
import { SSH_RUNTIME_DURATION_BUCKETS } from './telemetry-ssh-runtime-event-schemas'
|
||||
|
||||
// Why enum-only: these rows size how many hosts still need the relay before it can retire. A
|
||||
// hostname, user, path, log line or raw error would identify a machine, so none may appear.
|
||||
|
||||
export const SSH_HOST_SERVER_OUTCOME_VALUES = ['managed', 'deployed', 'converted', 'relay'] as const
|
||||
|
||||
/** Why orcad can't run on a host; mirrors main's OrcadHostUnavailableReason. */
|
||||
const UNAVAILABLE_REASONS = [
|
||||
// Recorded by builds before the stdio bridge; now retried rather than honoured.
|
||||
'tcp_forwarding_refused',
|
||||
// The host refuses port forwarding and can't run the stdio bridge either.
|
||||
'ssh_tunnel_unavailable',
|
||||
'unsupported_host',
|
||||
'artifacts_unavailable',
|
||||
'libc_unidentified',
|
||||
'runtime_self_test',
|
||||
'security_software',
|
||||
'native_preflight'
|
||||
] as const
|
||||
|
||||
export const SSH_HOST_SERVER_REASON_VALUES = [
|
||||
'connected',
|
||||
'deployed',
|
||||
'converted',
|
||||
'source_changed',
|
||||
'relay_terminals_live',
|
||||
'relay_terminals_unverifiable',
|
||||
...UNAVAILABLE_REASONS,
|
||||
'refused',
|
||||
'deferred',
|
||||
'failed',
|
||||
'other'
|
||||
] as const
|
||||
|
||||
/** A conversion refusal's code without its `orcad_migration_` prefix. */
|
||||
export const SSH_HOST_SERVER_REFUSAL_VALUES = [
|
||||
'none',
|
||||
'already_managed',
|
||||
'terminals',
|
||||
'target_not_found',
|
||||
'in_progress',
|
||||
'direct_ssh_connected',
|
||||
'preflight_blocked',
|
||||
'owned_by_deploy',
|
||||
'fenced_unverifiable',
|
||||
'stale_journal',
|
||||
'other'
|
||||
] as const
|
||||
|
||||
/** A deploy's deferral code without its `orcad_` prefix, or why orcad can't run on the host. */
|
||||
export const SSH_HOST_SERVER_FAILURE_VALUES = [
|
||||
...UNAVAILABLE_REASONS,
|
||||
'activation_recovery_required',
|
||||
'candidate_preflight_failed',
|
||||
'candidate_launch_failed',
|
||||
'incumbent_identity_unverifiable',
|
||||
'outgoing_stop_incomplete',
|
||||
'activation_build_mismatch',
|
||||
'activation_daemon_absent',
|
||||
'activation_daemon_degraded',
|
||||
'activation_no_health',
|
||||
'activation_no_persistent_terminals',
|
||||
'activation_no_readiness',
|
||||
'activation_not_listening',
|
||||
'activation_pty_self_test_failed',
|
||||
'update_daemon_protocol_unverifiable',
|
||||
'update_strands_live_terminals',
|
||||
'update_terminal_census_unavailable',
|
||||
'update_terminals_running',
|
||||
'windows_launch_refused',
|
||||
'windows_command_line_unsafe',
|
||||
'other'
|
||||
] as const
|
||||
|
||||
const hostFields = {
|
||||
host_os: z.enum(['linux', 'darwin', 'win32', 'unknown']),
|
||||
host_arch: z.enum(['x64', 'arm64', 'unknown']),
|
||||
// `unknown` when this session has not yet probed the host; the decision itself never probes.
|
||||
host_libc: z.enum(['glibc', 'musl', 'none', 'unknown'])
|
||||
}
|
||||
|
||||
/** How the client reached a managed server; `none` when the host stayed on the relay. */
|
||||
export const SSH_HOST_SERVER_TRANSPORT_VALUES = [
|
||||
'tcp_forward',
|
||||
'stdio_bridge',
|
||||
'none',
|
||||
'unknown'
|
||||
] as const
|
||||
|
||||
export const SSH_HOST_SERVER_MOVE_OUTCOME_VALUES = [
|
||||
'offered',
|
||||
'moved',
|
||||
'stayed',
|
||||
'refused_live',
|
||||
'refused_unverifiable',
|
||||
'failed'
|
||||
] as const
|
||||
|
||||
export const sshHostServerDecidedSchema = z
|
||||
.object({
|
||||
outcome: z.enum(SSH_HOST_SERVER_OUTCOME_VALUES),
|
||||
transport: z.enum(SSH_HOST_SERVER_TRANSPORT_VALUES),
|
||||
reason: z.enum(SSH_HOST_SERVER_REASON_VALUES),
|
||||
refusal: z.enum(SSH_HOST_SERVER_REFUSAL_VALUES),
|
||||
// True when a recorded "orcad can't run here" sent the host to the relay without a try.
|
||||
recorded: z.boolean(),
|
||||
...hostFields,
|
||||
duration_bucket: z.enum(SSH_RUNTIME_DURATION_BUCKETS)
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const sshHostServerConversionSchema = z
|
||||
.object({
|
||||
phase: z.enum(['started', 'committed', 'failed']),
|
||||
failure: z.enum(['none', 'deferred', 'refused', 'error']),
|
||||
refusal: z.enum(SSH_HOST_SERVER_REFUSAL_VALUES),
|
||||
failure_code: z.enum(['none', ...SSH_HOST_SERVER_FAILURE_VALUES]),
|
||||
...hostFields,
|
||||
duration_bucket: z.enum(SSH_RUNTIME_DURATION_BUCKETS)
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const sshHostServerDeployFailedSchema = z
|
||||
.object({
|
||||
// Whether the deploy ran for an empty host or inside a conversion.
|
||||
context: z.enum(['deploy', 'conversion']),
|
||||
failure: z.enum(['deferred', 'error']),
|
||||
failure_code: z.enum(SSH_HOST_SERVER_FAILURE_VALUES),
|
||||
...hostFields,
|
||||
duration_bucket: z.enum(SSH_RUNTIME_DURATION_BUCKETS)
|
||||
})
|
||||
.strict()
|
||||
|
||||
/** The per-host offer to move a host whose open terminals keep it on the relay, and its result. */
|
||||
export const sshHostServerMoveSchema = z
|
||||
.object({
|
||||
outcome: z.enum(SSH_HOST_SERVER_MOVE_OUTCOME_VALUES),
|
||||
...hostFields
|
||||
})
|
||||
.strict()
|
||||
Reference in New Issue
Block a user