fix(orcad): release this app's wake fence once its dropped launch is serving (#35808)

A wake whose launch reply was lost but whose server did start left its
fence held; the next wake saw the slot serving and returned, so deploys
saw the fence busy until it aged out. When the slot is proven LIVE, the
fence names this app's interrupted-wake token and no journal exists,
release it through the owner-guarded release command.
This commit is contained in:
Neil
2026-10-29 01:06:02 -07:00
committed by GitHub
parent 789f53f6a8
commit c0e0246033
2 changed files with 59 additions and 8 deletions
@@ -189,9 +189,53 @@ describe('wakeExpectedOrcadUnderOneFence', () => {
host.fence = false
expect(await wakeExpectedOrcadUnderOneFence(slot, OLD)).toEqual({ outcome: 'serving' })
}
// Still its own fence: the token stays, so only the full wake may reclaim it.
// Still its own fence, but a journal names work under it: the token and the fence both stay.
await dropOneFenceWake()
host.journal = '{}'
expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'serving' })
expect(host.fence).toBe(true)
expect(await wakeExpectedOrcadUnderOneFence(slot, OLD)).toEqual({ outcome: 'fenced' })
})
it('releases its own journal-free wake fence once the dropped launch is proven serving', async () => {
host = stoppedHost()
const lost = Object.assign(new Error('connection lost'), { sshChannelCloseConfirmed: false })
vi.mocked(execCommand).mockImplementation(async (_conn, command) => {
if (command.includes('nohup')) {
// The launch reached the host and started the slot; only the reply was lost.
host.exec(command)
host.alive.add(OLD)
throw lost
}
return host.exec(command)
})
await expect(wakeStoppedManagedOrcad(slot)).rejects.toBe(lost)
vi.mocked(execCommand).mockImplementation(async (_conn, command) => host.exec(command))
expect(host.fence).toBe(true)
expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'serving' })
// Released through the owner-guarded command, so deploys no longer see a busy fence.
expect(host.fence).toBe(false)
expect(host.commands.some((command) => command.includes('echo RELEASED'))).toBe(true)
expect(await wakeExpectedOrcadUnderOneFence(slot, OLD)).toEqual({ outcome: 'serving' })
})
it('never releases a fence another run holds, even with the slot serving', async () => {
host = stoppedHost()
const lost = Object.assign(new Error('connection lost'), { sshChannelCloseConfirmed: false })
vi.mocked(execCommand).mockImplementation(async (_conn, command) => {
if (command.includes('nohup')) {
throw lost
}
return host.exec(command)
})
await expect(wakeStoppedManagedOrcad(slot)).rejects.toBe(lost)
vi.mocked(execCommand).mockImplementation(async (_conn, command) => host.exec(command))
host.alive.add(OLD)
host.wakeOwner = 'another-run'
expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'serving' })
expect(host.fence).toBe(true)
expect(host.wakeOwner).toBe('another-run')
})
})
@@ -113,7 +113,7 @@ async function wakeAfterPrior(
}
const liveness = await slotLiveness(options, orcadSlotDir(options, before.active))
if (!orcadLivenessProvenExited(liveness)) {
await forgetSettledInterruptedWake(options, host)
await forgetSettledInterruptedWake(options, host, liveness === 'LIVE')
return { outcome: liveness === 'LIVE' ? 'serving' : 'unverifiable' }
}
if (!(await orcadActivationFenceExists(options))) {
@@ -192,11 +192,13 @@ function wakeHostKey(options: OrcadSlotOptions): string {
/**
* A serving slot skips the fence branch that clears an interrupted wake's token, which would
* keep the one-fence wake off until restart. Forget it once its fence is gone or another owner's;
* an unanswered probe proves neither, so the token stays.
* an unanswered probe proves neither, so the token stays. A fence still ours whose dropped launch
* did start the slot (proven LIVE, no journal) is released, or deploys would wait for it to age out.
*/
async function forgetSettledInterruptedWake(
options: OrcadSlotOptions,
host: string
host: string,
slotLive: boolean
): Promise<void> {
const token = interruptedWakes.get(host)
if (!token) {
@@ -206,10 +208,15 @@ async function forgetSettledInterruptedWake(
const owner = (await orcadActivationFenceExists(options))
? await readBoundedOrcadRemoteRecord(options, wakeOwnerPath(options), 64)
: null
if (!owner || (owner.state === 'present' && owner.raw.trim() !== token)) {
if (interruptedWakes.get(host) === token) {
interruptedWakes.delete(host)
}
const ours = owner?.state === 'present' && owner.raw.trim() === token
if (ours && slotLive && !(await readOrcadActivationTransaction(options))) {
// The release command re-proves the owner token on the host before removing anything.
await releaseOrcadActivationFence(options, token)
} else if (ours || owner?.state === 'absent') {
return
}
if (interruptedWakes.get(host) === token) {
interruptedWakes.delete(host)
}
} catch {
// Unverifiable: keep the token for the next wake.