mirror of
https://github.com/stablyai/orca.git
synced 2026-10-31 00:57:45 +00:00
fix(ssh): a network cut during a managed-server install no longer hangs, and says contact was lost (#30732)
* fix(ssh): a network cut during a managed-server install fails promptly and says contact was lost * fix(ssh): settle an SFTP transfer when its session dies, since ssh2 never answers the requests left * fix(ssh): only a closed SFTP session reads as lost contact; a dead session also cancels the abandoned transfer * test(ssh): put the mock's SAFETY note on the cast's own line * fix(ssh): a channel open the host never answered also reads as lost contact
This commit is contained in:
@@ -0,0 +1,78 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
import type { SFTPWrapper } from 'ssh2'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { uploadSshDirectory, type SshFileTransferHost } from './ssh-connection-file-transfers'
|
||||
|
||||
// What ssh2 does once the connection dies: the session closes, and a request sent after that (or a
|
||||
// write stream's CLOSE) is never answered, so the transfer itself never settles.
|
||||
vi.mock('../sftp-upload', () => ({ uploadDirectory: vi.fn(() => new Promise<void>(() => {})) }))
|
||||
|
||||
function hostWith(
|
||||
sftp: EventEmitter,
|
||||
operation = new AbortController().signal
|
||||
): SshFileTransferHost {
|
||||
return {
|
||||
target: { id: 'ssh-1', label: 'Box', host: 'box', port: 22, username: 'me' },
|
||||
usesSystemSshTransport: () => false,
|
||||
systemOperationSignal: () => operation,
|
||||
systemSshBuildArgsOptions: () => ({}),
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the transfer only listens to the session and ends it.
|
||||
sftp: async () => sftp as unknown as SFTPWrapper
|
||||
}
|
||||
}
|
||||
|
||||
function fakeSession(): EventEmitter & { end: () => void; ended: number } {
|
||||
return Object.assign(new EventEmitter(), {
|
||||
ended: 0,
|
||||
end(this: { ended: number }) {
|
||||
this.ended += 1
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
describe('an SFTP transfer whose SSH connection dies', () => {
|
||||
it('fails as lost contact instead of waiting forever', async () => {
|
||||
const sftp = fakeSession()
|
||||
const upload = uploadSshDirectory(hostWith(sftp), '/local/runtime', '/root/.orca-remote/x')
|
||||
await vi.waitFor(async () => {
|
||||
const { uploadDirectory } = await import('../sftp-upload')
|
||||
expect(uploadDirectory).toHaveBeenCalled()
|
||||
})
|
||||
sftp.emit('close')
|
||||
await expect(upload).rejects.toMatchObject({
|
||||
name: 'SftpSessionClosedError',
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
expect(sftp.listenerCount('close')).toBe(0)
|
||||
// The abandoned transfer is cancelled too, so it stops and closes its local files.
|
||||
const { uploadDirectory } = await import('../sftp-upload')
|
||||
const transferSignal = vi.mocked(uploadDirectory).mock.calls[0]?.[4]?.signal
|
||||
expect(transferSignal?.aborted).toBe(true)
|
||||
})
|
||||
|
||||
it('keeps a cancel a cancel when the session closes because Orca ended it', async () => {
|
||||
const sftp = fakeSession()
|
||||
const cancel = new AbortController()
|
||||
const upload = uploadSshDirectory(hostWith(sftp, cancel.signal), '/local/r', '/root/r')
|
||||
const { uploadDirectory } = await import('../sftp-upload')
|
||||
await vi.waitFor(() => expect(uploadDirectory).toHaveBeenCalled())
|
||||
cancel.abort()
|
||||
sftp.emit('close')
|
||||
await expect(upload).rejects.toMatchObject({ name: 'AbortError' })
|
||||
})
|
||||
|
||||
it('ignores a session close after the transfer finished', async () => {
|
||||
const { uploadDirectory } = await import('../sftp-upload')
|
||||
vi.mocked(uploadDirectory).mockResolvedValueOnce(undefined)
|
||||
const sftp = fakeSession()
|
||||
await uploadSshDirectory(hostWith(sftp), '/local/r', '/root/r')
|
||||
expect(sftp.ended).toBe(1)
|
||||
// The close that ending the session produces is expected, so nothing is left listening.
|
||||
sftp.emit('close')
|
||||
expect(sftp.listenerCount('close')).toBe(0)
|
||||
})
|
||||
})
|
||||
@@ -7,7 +7,8 @@ import {
|
||||
} from '../sftp-namespace-resolution'
|
||||
import {
|
||||
createLinkedSshFileTransferSignal,
|
||||
raceSftpFileTransferWithAbort
|
||||
raceSftpFileTransferWithAbort,
|
||||
sftpSessionClosedError
|
||||
} from '../ssh-file-transfer-abort'
|
||||
import type { RemoteHostPlatform } from '../ssh-remote-platform'
|
||||
import {
|
||||
@@ -145,6 +146,13 @@ async function withLinkedSftpTransfer(
|
||||
(signal): signal is AbortSignal => signal !== undefined
|
||||
)
|
||||
)
|
||||
// Why apart from linkedSignal: a dead session is not a cancel, so the abort race must not see it;
|
||||
// only the transfer does, so it stops and releases its local files.
|
||||
const sessionLoss = new AbortController()
|
||||
const transferSignal = createLinkedSshFileTransferSignal([
|
||||
linkedSignal.signal,
|
||||
sessionLoss.signal
|
||||
])
|
||||
try {
|
||||
if (host.usesSystemSshTransport()) {
|
||||
await transfer.system(linkedSignal.signal)
|
||||
@@ -161,22 +169,41 @@ async function withLinkedSftpTransfer(
|
||||
}
|
||||
sftp.on('error', swallowLateSftpError)
|
||||
sftp.once('close', () => sftp.removeListener('error', swallowLateSftpError))
|
||||
// Why: when the connection dies ssh2 ends the session, but a request sent or queued after that
|
||||
// is never answered, and a write stream waits on a CLOSE reply, so the transfer would hang.
|
||||
let sessionLost: (error: Error) => void = () => {}
|
||||
const lostSession = new Promise<never>((_resolve, reject) => {
|
||||
sessionLost = reject
|
||||
})
|
||||
const onSessionClose = (): void => {
|
||||
if (!sftpEndRequested) {
|
||||
const error = sftpSessionClosedError()
|
||||
sessionLost(error)
|
||||
sessionLoss.abort(error)
|
||||
}
|
||||
}
|
||||
sftp.once('close', onSessionClose)
|
||||
try {
|
||||
// Why: resolve on the same session that transfers — a later one is not authoritative for its namespace.
|
||||
const run = (async (): Promise<void> => {
|
||||
const targetPath = await resolveSftpTransferPathIfMapped(sftp, remotePath, options)
|
||||
linkedSignal.signal.throwIfAborted()
|
||||
await transfer.sftp(sftp, targetPath, linkedSignal.signal)
|
||||
transferSignal.signal.throwIfAborted()
|
||||
await transfer.sftp(sftp, targetPath, transferSignal.signal)
|
||||
})()
|
||||
await raceSftpFileTransferWithAbort(run, linkedSignal.signal, (onClose) => {
|
||||
sftp.once('close', onClose)
|
||||
endSftp()
|
||||
return () => sftp.removeListener('close', onClose)
|
||||
})
|
||||
await Promise.race([
|
||||
raceSftpFileTransferWithAbort(run, linkedSignal.signal, (onClose) => {
|
||||
sftp.once('close', onClose)
|
||||
endSftp()
|
||||
return () => sftp.removeListener('close', onClose)
|
||||
}),
|
||||
lostSession
|
||||
])
|
||||
} finally {
|
||||
sftp.removeListener('close', onSessionClose)
|
||||
endSftp()
|
||||
}
|
||||
} finally {
|
||||
transferSignal.dispose()
|
||||
linkedSignal.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,8 @@ function createWritable(): Writable {
|
||||
}
|
||||
|
||||
function createSftpMock(): SFTPWrapper {
|
||||
return {
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these tests call only the methods stubbed here and the session's events.
|
||||
return Object.assign(new EventEmitter(), {
|
||||
mkdir: vi.fn((_path: string, cb: (err?: Error | null) => void) => cb(null)),
|
||||
createWriteStream: vi.fn(() => createWritable()),
|
||||
readdir: vi.fn((_path: string, cb: (err?: Error | null, entries?: unknown[]) => void) =>
|
||||
@@ -32,7 +33,7 @@ function createSftpMock(): SFTPWrapper {
|
||||
),
|
||||
unlink: vi.fn((_path: string, cb: (err?: Error | null) => void) => cb(null)),
|
||||
rmdir: vi.fn((_path: string, cb: (err?: Error | null) => void) => cb(null))
|
||||
} as unknown as SFTPWrapper
|
||||
}) as unknown as SFTPWrapper
|
||||
}
|
||||
|
||||
describe('sftp-upload', () => {
|
||||
@@ -296,7 +297,10 @@ describe('sftp-upload', () => {
|
||||
|
||||
describe('uploadFile over a high-latency link', () => {
|
||||
// ssh2's own WriteStream over a fake session whose every request takes one "round trip".
|
||||
function slowSftp(roundTripMs: number): {
|
||||
function slowSftp(
|
||||
roundTripMs: number,
|
||||
stallAfterWrites = Number.POSITIVE_INFINITY
|
||||
): {
|
||||
sftp: SFTPWrapper
|
||||
written: Map<number, Buffer>
|
||||
maxInFlight: () => number
|
||||
@@ -304,8 +308,11 @@ describe('uploadFile over a high-latency link', () => {
|
||||
let inFlight = 0
|
||||
let peak = 0
|
||||
const written = new Map<number, Buffer>()
|
||||
// A dead link: once stalled, no request (WRITE or the stream's CLOSE) is ever answered.
|
||||
const later = (cb: () => void): void => {
|
||||
setTimeout(cb, roundTripMs)
|
||||
if (written.size <= stallAfterWrites) {
|
||||
setTimeout(cb, roundTripMs)
|
||||
}
|
||||
}
|
||||
const session = Object.assign(new EventEmitter(), {
|
||||
server: false,
|
||||
@@ -358,4 +365,22 @@ describe('uploadFile over a high-latency link', () => {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('fails, as contact lost, when the SSH session closes under a stalled upload', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'orca-sftp-upload-'))
|
||||
await writeFile(join(dir, 'node.tar.gz'), Buffer.alloc(8 * 1024 * 1024))
|
||||
const { sftp, written } = slowSftp(1, 4)
|
||||
try {
|
||||
const upload = uploadFile(sftp, join(dir, 'node.tar.gz'), '/remote/node.tar.gz')
|
||||
await vi.waitFor(() => expect(written.size).toBeGreaterThan(4))
|
||||
sftp.emit('close')
|
||||
await expect(upload).rejects.toMatchObject({
|
||||
name: 'SftpSessionClosedError',
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
expect(sftp.listenerCount('close')).toBe(0)
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -6,12 +6,26 @@ import { finished } from 'node:stream/promises'
|
||||
import { promisify } from 'node:util'
|
||||
import type { SFTPWrapper } from 'ssh2'
|
||||
import { latchLateSftpStreamErrors, type SftpStreamErrorLatch } from './sftp-stream-late-error'
|
||||
import { sftpSessionClosedError } from './ssh-file-transfer-abort'
|
||||
|
||||
// Why: ssh2's WriteStream waits for each chunk's ack before writing the next, so an upload moves
|
||||
// one chunk per round trip. Buffering this much lets its _writev keep that many bytes of WRITE
|
||||
// requests in flight, as fastPut's default 64 x 32 KiB does.
|
||||
export const SFTP_UPLOAD_IN_FLIGHT_BYTES = 2 * 1024 * 1024
|
||||
|
||||
/**
|
||||
* Why: on a dead connection ssh2's write stream waits on a CLOSE reply that never comes; settle on
|
||||
* the session's close so the local file handle is closed, not left to garbage collection.
|
||||
*/
|
||||
function watchSessionClose(sftp: SFTPWrapper): { closed: Promise<'closed'>; release: () => void } {
|
||||
let onClose: () => void = () => {}
|
||||
const closed = new Promise<'closed'>((resolve) => {
|
||||
onClose = () => resolve('closed')
|
||||
})
|
||||
sftp.once('close', onClose)
|
||||
return { closed, release: () => sftp.removeListener('close', onClose) }
|
||||
}
|
||||
|
||||
export function mkdirSftp(
|
||||
sftp: SFTPWrapper,
|
||||
path: string,
|
||||
@@ -43,6 +57,7 @@ export async function uploadFile(
|
||||
let readStream: ReadStream | undefined
|
||||
let writeStream: ReturnType<SFTPWrapper['createWriteStream']> | undefined
|
||||
let writeStreamErrors: SftpStreamErrorLatch | undefined
|
||||
const sessionClosed = watchSessionClose(sftp)
|
||||
const closeHandle = (): Promise<void> => {
|
||||
handleClose ??= handle.close()
|
||||
return handleClose
|
||||
@@ -98,7 +113,13 @@ export async function uploadFile(
|
||||
throw error
|
||||
})
|
||||
readStream.pipe(writeStream)
|
||||
const results = await Promise.allSettled([readDone, writeDone])
|
||||
const results = await Promise.race([
|
||||
Promise.allSettled([readDone, writeDone]),
|
||||
sessionClosed.closed
|
||||
])
|
||||
if (results === 'closed') {
|
||||
throw sftpSessionClosedError()
|
||||
}
|
||||
const failure = results.find((result) => result.status === 'rejected')
|
||||
if (failure?.status === 'rejected') {
|
||||
throw failure.reason
|
||||
@@ -107,6 +128,7 @@ export async function uploadFile(
|
||||
options?.signal?.removeEventListener('abort', abortTransfer)
|
||||
}
|
||||
} finally {
|
||||
sessionClosed.release()
|
||||
writeStreamErrors?.markTransferSettled()
|
||||
readStream?.destroy()
|
||||
writeStream?.destroy()
|
||||
|
||||
@@ -10,6 +10,11 @@ const ABORTED_CHANNEL_CLOSE_GRACE_MS = 5_000
|
||||
const SESSION_LIMIT_OPEN_RETRIES = 4
|
||||
const SESSION_LIMIT_OPEN_RETRY_DELAY_MS = 150
|
||||
|
||||
/** The host never answered a channel open: contact, not the host, is what failed. */
|
||||
export function sshChannelOpenTimeoutError(message: string): Error {
|
||||
return Object.assign(new Error(message), { name: 'SshChannelOpenTimeoutError' })
|
||||
}
|
||||
|
||||
export async function openSshSessionChannelWithRetry<T>(
|
||||
open: () => Promise<T>,
|
||||
signal?: AbortSignal
|
||||
@@ -87,7 +92,9 @@ export function waitForSshChannelOpen<T>(
|
||||
cleanup()
|
||||
reject(
|
||||
markOpenUnconfirmed(
|
||||
abortRequested ? createSshOperationAbortError() : new Error(timeoutMessage)
|
||||
abortRequested
|
||||
? createSshOperationAbortError()
|
||||
: sshChannelOpenTimeoutError(timeoutMessage)
|
||||
)
|
||||
)
|
||||
}, CONNECT_TIMEOUT_MS)
|
||||
|
||||
@@ -24,6 +24,21 @@ export function createLinkedSshFileTransferSignal(signals: readonly AbortSignal[
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A transfer whose SFTP session closed without being asked to: the SSH connection died under it.
|
||||
* Contact was lost, so whether the host kept the bytes is unknown.
|
||||
*/
|
||||
export function sftpSessionClosedError(): Error {
|
||||
return Object.assign(new Error('The SSH connection closed before the transfer finished'), {
|
||||
name: 'SftpSessionClosedError',
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
}
|
||||
|
||||
export function isSftpSessionClosedError(error: unknown): boolean {
|
||||
return error instanceof Error && error.name === 'SftpSessionClosedError'
|
||||
}
|
||||
|
||||
export function raceSftpFileTransferWithAbort<T>(
|
||||
operation: Promise<T>,
|
||||
signal: AbortSignal,
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
/**
|
||||
* A network cut during a managed-server install is lost contact, not a failure on the host: the
|
||||
* connect must say so (live/unverifiable/exited vocabulary), never "couldn't start its server".
|
||||
*/
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { SshTarget } from '../../shared/ssh-types'
|
||||
import { resolveHostServerOnConnect } from './ssh-host-server-on-connect'
|
||||
import { hostServerDepsStub } from './ssh-host-server-on-connect-test-deps'
|
||||
import { unservedHostServerMessage, unservedServerStatus } from './ssh-host-unserved-status'
|
||||
import { sftpSessionClosedError } from './ssh-file-transfer-abort'
|
||||
import { sshChannelOpenTimeoutError } from './ssh-channel-open'
|
||||
import { admitSshManagedServerStatus } from '../../shared/ssh-managed-server-status-admission'
|
||||
|
||||
const target: SshTarget = { id: 'ssh-1', label: 'Box', host: 'box', port: 22, username: 'me' }
|
||||
|
||||
async function deployFailingWith(error: Error) {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
try {
|
||||
return await resolveHostServerOnConnect(
|
||||
target,
|
||||
hostServerDepsStub({
|
||||
isEmptyHost: () => true,
|
||||
deploy: async () => {
|
||||
throw error
|
||||
}
|
||||
})
|
||||
)
|
||||
} finally {
|
||||
warn.mockRestore()
|
||||
}
|
||||
}
|
||||
|
||||
describe('a managed-server install that loses contact with the host', () => {
|
||||
it('says contact was lost when the upload’s SSH session closes', async () => {
|
||||
const result = await deployFailingWith(sftpSessionClosedError())
|
||||
expect(result).toMatchObject({ route: 'unserved', reason: 'failed', contactLost: true })
|
||||
if (result.route !== 'unserved') {
|
||||
throw new Error('expected unserved')
|
||||
}
|
||||
expect(unservedHostServerMessage(result)).toMatch(
|
||||
/lost contact.*Nothing on the host was stopped/
|
||||
)
|
||||
expect(admitSshManagedServerStatus(unservedServerStatus(result))).toMatchObject({
|
||||
managedServer: { kind: 'relay', reason: 'failed', contactLost: true }
|
||||
})
|
||||
})
|
||||
|
||||
it('says contact was lost when the host never answers a channel open', async () => {
|
||||
const unanswered = sshChannelOpenTimeoutError('SSH SFTP channel timed out')
|
||||
await expect(deployFailingWith(unanswered)).resolves.toMatchObject({ contactLost: true })
|
||||
})
|
||||
|
||||
it('reads lost contact through a wrapping install error', async () => {
|
||||
const wrapped = new Error('Uploading the runtime failed', { cause: sftpSessionClosedError() })
|
||||
await expect(deployFailingWith(wrapped)).resolves.toMatchObject({ contactLost: true })
|
||||
})
|
||||
|
||||
it('does not call a host’s own unconfirmed answer lost contact', async () => {
|
||||
const busy = Object.assign(new Error('STATE_MUTATION_DEADLINE'), {
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
await expect(deployFailingWith(busy)).resolves.not.toHaveProperty('contactLost')
|
||||
})
|
||||
|
||||
it('keeps the plain try-again for a failure the host answered', async () => {
|
||||
const result = await deployFailingWith(new Error('tar: Unexpected EOF in archive'))
|
||||
expect(result).not.toHaveProperty('contactLost')
|
||||
if (result.route !== 'unserved') {
|
||||
throw new Error('expected unserved')
|
||||
}
|
||||
expect(unservedHostServerMessage(result)).toContain('Try connecting again')
|
||||
})
|
||||
})
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
import { managedResultAfterUpdate, unservedManagedResult } from './managed-server-serving-gate'
|
||||
import { orcadMigrationConflictGuidance } from './orcad-migration-conflict-guidance'
|
||||
import { keptForBridgedTerminals, relayTerminalReason } from './ssh-host-server-bridged-unserved'
|
||||
import { boundedDetail, setupFailure, type SetupFailure } from './ssh-host-server-setup-failure'
|
||||
import {
|
||||
classifyOrcadHostUnavailable,
|
||||
ORCAD_TUNNEL_UNAVAILABLE_REASON
|
||||
@@ -65,6 +66,8 @@ export type HostServerOnConnectResult =
|
||||
canLeaveBehind?: true
|
||||
/** A move that stopped short kept the host locked; the next connect resumes it. */
|
||||
held?: true
|
||||
/** Setup lost contact with the host, so it is not known to have failed there. */
|
||||
contactLost?: true
|
||||
/** COMPAT(legacy-relay-bridge): the connect stays up so the bridge serves the terminals. */
|
||||
bridged?: true
|
||||
}
|
||||
@@ -115,9 +118,6 @@ export type HostServerOnConnectDeps = {
|
||||
report: HostServerReport
|
||||
} & ManagedServerUpdateDeps
|
||||
|
||||
// Bounds a failure's detail; the host log tail inside it is already capped.
|
||||
const MAX_FAILURE_DETAIL_CHARS = 16_000
|
||||
|
||||
export async function resolveHostServerOnConnect(
|
||||
target: SshTarget,
|
||||
deps: HostServerOnConnectDeps
|
||||
@@ -252,10 +252,9 @@ function setupFailed(
|
||||
const unavailable = classifyOrcadHostUnavailable(error)
|
||||
const conflict = empty ? null : orcadMigrationConflictGuidance(error)
|
||||
if (conflict) {
|
||||
return unfinished(target, deps, unavailable, empty, 'refused', conflict)
|
||||
return unfinished(target, deps, unavailable, empty, 'refused', { detail: conflict })
|
||||
}
|
||||
const detail = error instanceof Error ? error.message : String(error)
|
||||
return unfinished(target, deps, unavailable, empty, 'failed', detail)
|
||||
return unfinished(target, deps, unavailable, empty, 'failed', setupFailure(error))
|
||||
}
|
||||
|
||||
async function afterDeploy(
|
||||
@@ -315,7 +314,7 @@ function deferred(
|
||||
): Promise<HostServerOnConnectResult> {
|
||||
console.warn(`[ssh] Managed Orca server setup deferred (${result.code}):`, result.reason)
|
||||
const unavailable = classifyOrcadHostUnavailable({ code: result.code })
|
||||
return unfinished(target, deps, unavailable, empty, 'deferred', result.reason)
|
||||
return unfinished(target, deps, unavailable, empty, 'deferred', { detail: result.reason })
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -328,7 +327,7 @@ async function unfinished(
|
||||
unavailable: string | null,
|
||||
empty: boolean,
|
||||
reason: 'deferred' | 'failed' | 'refused',
|
||||
failure: string
|
||||
failure: SetupFailure
|
||||
): Promise<HostServerOnConnectResult> {
|
||||
try {
|
||||
await (empty ? deps.abandonDeploy(target) : deps.abandonConversion(target))
|
||||
@@ -343,14 +342,11 @@ async function unfinished(
|
||||
return {
|
||||
route: 'unserved',
|
||||
reason,
|
||||
detail: boundedDetail(failure),
|
||||
detail: boundedDetail(failure.detail),
|
||||
...(failure.contactLost ? { contactLost: true as const } : {}),
|
||||
...(held ? { held: true as const } : {})
|
||||
}
|
||||
}
|
||||
deps.recordUnavailable(target, unavailable)
|
||||
return { route: 'unserved', reason: 'orcad_unavailable', detail: unavailable }
|
||||
}
|
||||
|
||||
function boundedDetail(text: string): string {
|
||||
return text.length > MAX_FAILURE_DETAIL_CHARS ? `…${text.slice(-MAX_FAILURE_DETAIL_CHARS)}` : text
|
||||
}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
/** What a managed-server setup that stopped short shows under its SSH host. */
|
||||
import { isSftpSessionClosedError } from './ssh-file-transfer-abort'
|
||||
|
||||
// Bounds a failure's detail; the host log tail inside it is already capped.
|
||||
const MAX_FAILURE_DETAIL_CHARS = 16_000
|
||||
|
||||
export type SetupFailure = {
|
||||
detail: string
|
||||
/** Contact was lost mid-setup: nothing proves the host failed, so it is never called a failure. */
|
||||
contactLost?: true
|
||||
}
|
||||
|
||||
export function setupFailure(failure: unknown): SetupFailure {
|
||||
return {
|
||||
detail: failure instanceof Error ? failure.message : String(failure),
|
||||
...(lostContact(failure) ? { contactLost: true as const } : {})
|
||||
}
|
||||
}
|
||||
|
||||
export function boundedDetail(text: string): string {
|
||||
return text.length > MAX_FAILURE_DETAIL_CHARS ? `…${text.slice(-MAX_FAILURE_DETAIL_CHARS)}` : text
|
||||
}
|
||||
|
||||
// Why only a closed session or an unanswered channel open: other unconfirmed results (a host's
|
||||
// busy or deadline answer) came from a host that answered. Why the cause chain: install steps wrap
|
||||
// the transport error they hit.
|
||||
function lostContact(failure: unknown, depth = 0): boolean {
|
||||
if (
|
||||
isSftpSessionClosedError(failure) ||
|
||||
(failure instanceof Error && failure.name === 'SshChannelOpenTimeoutError')
|
||||
) {
|
||||
return true
|
||||
}
|
||||
return depth < 4 && failure instanceof Error && lostContact(failure.cause, depth + 1)
|
||||
}
|
||||
@@ -16,6 +16,7 @@ export function unservedServerStatus(
|
||||
...(decision.terminalsElsewhere ? { terminalsElsewhere: true } : {}),
|
||||
...(decision.canLeaveBehind ? { canLeaveBehind: true } : {}),
|
||||
...(decision.held ? { held: true } : {}),
|
||||
...(decision.contactLost ? { contactLost: true } : {}),
|
||||
...(decision.bridged ? { bridged: true } : {})
|
||||
}
|
||||
}
|
||||
@@ -26,6 +27,9 @@ const HELD_MOVE_MESSAGE =
|
||||
const NOEXEC_MESSAGE =
|
||||
'This host doesn’t allow running programs from ~/.orca-remote, so Orca can’t start its server there. This usually means the home folder is mounted with noexec. Ask the host’s administrator to allow running programs there, then connect again.'
|
||||
|
||||
const CONTACT_LOST_MESSAGE =
|
||||
'Orca lost contact with this host while setting up its server. Nothing on the host was stopped. Connect again to continue.'
|
||||
|
||||
/** Untranslated twin of the renderer's status line, for logs and callers without the catalog. */
|
||||
export function unservedHostServerMessage(decision: UnservedDecision): string {
|
||||
if (decision.held && (decision.reason === 'failed' || decision.reason === 'deferred')) {
|
||||
@@ -45,6 +49,8 @@ export function unservedHostServerMessage(decision: UnservedDecision): string {
|
||||
return 'An older version of Orca changed this host’s projects. Move those changes to its managed server in SSH Hosts settings to continue.'
|
||||
case 'deferred':
|
||||
case 'failed':
|
||||
return 'Orca couldn’t start its server on this host. Try connecting again.'
|
||||
return decision.contactLost
|
||||
? CONTACT_LOST_MESSAGE
|
||||
: 'Orca couldn’t start its server on this host. Try connecting again.'
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12820,7 +12820,8 @@
|
||||
"behindNoTemplate": "Runs a managed Orca server this app can’t update; this build has no server for this host.",
|
||||
"behindMigrating": "Runs a managed Orca server that updates after its project move finishes.",
|
||||
"behindCheckFailed": "Runs a managed Orca server; Orca couldn’t check whether it is behind this app.",
|
||||
"runtimeNoexec": "This host doesn’t allow running programs from ~/.orca-remote, so Orca can’t start its server there. This usually means the home folder is mounted with noexec. Ask the host’s administrator to allow running programs there, then connect again."
|
||||
"runtimeNoexec": "This host doesn’t allow running programs from ~/.orca-remote, so Orca can’t start its server there. This usually means the home folder is mounted with noexec. Ask the host’s administrator to allow running programs there, then connect again.",
|
||||
"setupContactLost": "Orca lost contact with this host while setting up its server. Nothing on the host was stopped. Connect again to continue."
|
||||
},
|
||||
"ZcodeCliSetupSection": {
|
||||
"title": "Use GLM with ZCode",
|
||||
|
||||
@@ -230,6 +230,19 @@ describe('SSH host server status line', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('says contact was lost, not that the server failed, when setup lost the host', () => {
|
||||
const line = sshHostServerStatusLine(plain, {
|
||||
managedServer: {
|
||||
kind: 'relay',
|
||||
reason: 'failed',
|
||||
contactLost: true,
|
||||
detail: 'The SSH connection closed before the transfer finished'
|
||||
}
|
||||
})
|
||||
expect(line).toMatchObject({ tone: 'warning', text: expect.stringContaining('lost contact') })
|
||||
expect(line?.text).not.toContain('couldn’t start')
|
||||
})
|
||||
|
||||
it('says a move that stopped short is paused and kept, not a plain try-again', () => {
|
||||
const detail = 'Could not connect to the remote Orca runtime.'
|
||||
const state = {
|
||||
|
||||
@@ -189,6 +189,16 @@ function retryLine(): SshHostServerStatusLine {
|
||||
}
|
||||
}
|
||||
|
||||
function contactLostLine(): SshHostServerStatusLine {
|
||||
return {
|
||||
tone: 'warning',
|
||||
text: translate(
|
||||
'auto.components.settings.sshHostServer.setupContactLost',
|
||||
'Orca lost contact with this host while setting up its server. Nothing on the host was stopped. Connect again to continue.'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
function heldMoveLine(): SshHostServerStatusLine {
|
||||
return {
|
||||
tone: 'warning',
|
||||
@@ -295,6 +305,9 @@ function unservedLine(
|
||||
}
|
||||
case 'deferred':
|
||||
case 'failed':
|
||||
return { ...(status.held ? heldMoveLine() : retryLine()), ...withDetail(status.detail) }
|
||||
return {
|
||||
...(status.held ? heldMoveLine() : status.contactLost ? contactLostLine() : retryLine()),
|
||||
...withDetail(status.detail)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,6 +83,7 @@ export function admitSshManagedServerStatus(value: unknown): {
|
||||
? { canLeaveBehind: true }
|
||||
: {}),
|
||||
...('held' in value && value.held === true ? { held: true } : {}),
|
||||
...('contactLost' in value && value.contactLost === true ? { contactLost: true } : {}),
|
||||
...('bridged' in value && value.bridged === true ? { bridged: true } : {})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -352,6 +352,8 @@ export type SshManagedServerStatus =
|
||||
canLeaveBehind?: true
|
||||
/** A move to the managed server stopped short and keeps the host locked until it resumes. */
|
||||
held?: true
|
||||
/** Setup lost contact with the host, so nothing proves it failed there. */
|
||||
contactLost?: true
|
||||
/** COMPAT(legacy-relay-bridge): the connect stayed up so the bridge serves those terminals. */
|
||||
bridged?: true
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user