mirror of
https://github.com/stablyai/orca.git
synced 2026-11-04 11:07:42 +00:00
fix(orcad): release this app's wake fence once its dropped launch is serving (#35808)
A wake whose launch reply was lost but whose server did start left its fence held; the next wake saw the slot serving and returned, so deploys saw the fence busy until it aged out. When the slot is proven LIVE, the fence names this app's interrupted-wake token and no journal exists, release it through the owner-guarded release command.
This commit is contained in:
@@ -189,9 +189,53 @@ describe('wakeExpectedOrcadUnderOneFence', () => {
|
||||
host.fence = false
|
||||
expect(await wakeExpectedOrcadUnderOneFence(slot, OLD)).toEqual({ outcome: 'serving' })
|
||||
}
|
||||
// Still its own fence: the token stays, so only the full wake may reclaim it.
|
||||
// Still its own fence, but a journal names work under it: the token and the fence both stay.
|
||||
await dropOneFenceWake()
|
||||
host.journal = '{}'
|
||||
expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'serving' })
|
||||
expect(host.fence).toBe(true)
|
||||
expect(await wakeExpectedOrcadUnderOneFence(slot, OLD)).toEqual({ outcome: 'fenced' })
|
||||
})
|
||||
|
||||
it('releases its own journal-free wake fence once the dropped launch is proven serving', async () => {
|
||||
host = stoppedHost()
|
||||
const lost = Object.assign(new Error('connection lost'), { sshChannelCloseConfirmed: false })
|
||||
vi.mocked(execCommand).mockImplementation(async (_conn, command) => {
|
||||
if (command.includes('nohup')) {
|
||||
// The launch reached the host and started the slot; only the reply was lost.
|
||||
host.exec(command)
|
||||
host.alive.add(OLD)
|
||||
throw lost
|
||||
}
|
||||
return host.exec(command)
|
||||
})
|
||||
await expect(wakeStoppedManagedOrcad(slot)).rejects.toBe(lost)
|
||||
vi.mocked(execCommand).mockImplementation(async (_conn, command) => host.exec(command))
|
||||
expect(host.fence).toBe(true)
|
||||
|
||||
expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'serving' })
|
||||
// Released through the owner-guarded command, so deploys no longer see a busy fence.
|
||||
expect(host.fence).toBe(false)
|
||||
expect(host.commands.some((command) => command.includes('echo RELEASED'))).toBe(true)
|
||||
expect(await wakeExpectedOrcadUnderOneFence(slot, OLD)).toEqual({ outcome: 'serving' })
|
||||
})
|
||||
|
||||
it('never releases a fence another run holds, even with the slot serving', async () => {
|
||||
host = stoppedHost()
|
||||
const lost = Object.assign(new Error('connection lost'), { sshChannelCloseConfirmed: false })
|
||||
vi.mocked(execCommand).mockImplementation(async (_conn, command) => {
|
||||
if (command.includes('nohup')) {
|
||||
throw lost
|
||||
}
|
||||
return host.exec(command)
|
||||
})
|
||||
await expect(wakeStoppedManagedOrcad(slot)).rejects.toBe(lost)
|
||||
vi.mocked(execCommand).mockImplementation(async (_conn, command) => host.exec(command))
|
||||
host.alive.add(OLD)
|
||||
host.wakeOwner = 'another-run'
|
||||
|
||||
expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'serving' })
|
||||
expect(host.fence).toBe(true)
|
||||
expect(host.wakeOwner).toBe('another-run')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -113,7 +113,7 @@ async function wakeAfterPrior(
|
||||
}
|
||||
const liveness = await slotLiveness(options, orcadSlotDir(options, before.active))
|
||||
if (!orcadLivenessProvenExited(liveness)) {
|
||||
await forgetSettledInterruptedWake(options, host)
|
||||
await forgetSettledInterruptedWake(options, host, liveness === 'LIVE')
|
||||
return { outcome: liveness === 'LIVE' ? 'serving' : 'unverifiable' }
|
||||
}
|
||||
if (!(await orcadActivationFenceExists(options))) {
|
||||
@@ -192,11 +192,13 @@ function wakeHostKey(options: OrcadSlotOptions): string {
|
||||
/**
|
||||
* A serving slot skips the fence branch that clears an interrupted wake's token, which would
|
||||
* keep the one-fence wake off until restart. Forget it once its fence is gone or another owner's;
|
||||
* an unanswered probe proves neither, so the token stays.
|
||||
* an unanswered probe proves neither, so the token stays. A fence still ours whose dropped launch
|
||||
* did start the slot (proven LIVE, no journal) is released, or deploys would wait for it to age out.
|
||||
*/
|
||||
async function forgetSettledInterruptedWake(
|
||||
options: OrcadSlotOptions,
|
||||
host: string
|
||||
host: string,
|
||||
slotLive: boolean
|
||||
): Promise<void> {
|
||||
const token = interruptedWakes.get(host)
|
||||
if (!token) {
|
||||
@@ -206,10 +208,15 @@ async function forgetSettledInterruptedWake(
|
||||
const owner = (await orcadActivationFenceExists(options))
|
||||
? await readBoundedOrcadRemoteRecord(options, wakeOwnerPath(options), 64)
|
||||
: null
|
||||
if (!owner || (owner.state === 'present' && owner.raw.trim() !== token)) {
|
||||
if (interruptedWakes.get(host) === token) {
|
||||
interruptedWakes.delete(host)
|
||||
}
|
||||
const ours = owner?.state === 'present' && owner.raw.trim() === token
|
||||
if (ours && slotLive && !(await readOrcadActivationTransaction(options))) {
|
||||
// The release command re-proves the owner token on the host before removing anything.
|
||||
await releaseOrcadActivationFence(options, token)
|
||||
} else if (ours || owner?.state === 'absent') {
|
||||
return
|
||||
}
|
||||
if (interruptedWakes.get(host) === token) {
|
||||
interruptedWakes.delete(host)
|
||||
}
|
||||
} catch {
|
||||
// Unverifiable: keep the token for the next wake.
|
||||
|
||||
Reference in New Issue
Block a user