relay: an edge fence refused as a same-epoch conflict goes once more one epoch above

At 1% the edge answered fences 'conflict': it holds the fence's own epoch for another placement
(a move onto the edge whose push landed though the director counted it failed, then a placement
elsewhere minted the same epoch). Retries go to the host's current row at that same epoch, so they
kept failing until the host moved again; meanwhile the edge object claims the host and its phones
get HOST_OFFLINE instead of re-resolving. No later move can produce a same-epoch conflict (it would
answer stale), so the fence now goes once more one epoch above, which lands; a later move onto the
edge re-mints above it. Stale answers are never bumped.
This commit is contained in:
Neil
2026-11-10 16:48:15 -08:00
parent 8ac9fb6eee
commit 0c0cd4591a
2 changed files with 60 additions and 2 deletions
@@ -116,6 +116,43 @@ describe('relay edge fence after a host leaves', () => {
})
})
describe('relay edge fence over a same-epoch conflict', () => {
it('fences once more one epoch above a conflict at its epoch, and never bumps over a stale answer', async () => {
vi.spyOn(console, 'error').mockImplementation(() => {})
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const h = await harness()
h.edge.answers = ['conflict', 'applied']
expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)).toBe(true)
expect(h.edge.calls.map((call) => call.body.assignmentEpoch)).toEqual([9, 10])
expect(h.edge.calls.every((call) => call.body.cellId === 'us-c1')).toBe(true)
expect(logged(warn, 'orca_relay_edge_fence_above_conflict')).toEqual([
{ event: 'orca_relay_edge_fence_above_conflict', outcome: 'applied', assignmentEpoch: 10 }
])
expect(await h.pending()).toEqual([])
// Stale: the edge holds a newer epoch (the host may be back there); no bump, recorded for retry.
h.edge.calls = []
h.edge.answers = ['stale']
expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 11)).toBe(false)
expect(h.edge.calls.map((call) => call.body.assignmentEpoch)).toEqual([11])
})
it('a pending fence retried into a same-epoch conflict lands one epoch above the row', async () => {
vi.spyOn(console, 'error').mockImplementation(() => {})
vi.spyOn(console, 'warn').mockImplementation(() => {})
const h = await harness()
await h.placeRow('us-c1', 7)
h.edge.answers = ['down', 'down', 'down', 'down', 'down', 'down', 'down', 'down', 'down', 'down']
expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 7)).toBe(false)
expect(await h.pending()).toHaveLength(1)
h.edge.calls = []
h.edge.answers = ['conflict', 'applied']
h.clock.now += 60_000
await h.retry()
expect(h.edge.calls.map((call) => call.body.assignmentEpoch)).toEqual([7, 8])
expect(await h.pending()).toEqual([])
})
})
describe('relay edge pending fence retries', () => {
it('re-fences to the current row, not the recorded one, and clears the record once it lands', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
+23 -2
View File
@@ -41,6 +41,27 @@ export async function recordPendingFence(
)
}
// One fence call. `conflict` means the edge holds this very epoch for another placement (a move onto
// the edge whose push landed though the director counted it as failed, then a placement elsewhere
// minted the same epoch). No later move can produce that (a later move has a higher epoch, which
// answers `stale`), so the fence goes once more one epoch above: it lands, and a later move onto
// the edge re-mints above it as for any edge epoch (moveToEdge).
async function assignFence(
admin: EdgeAdmin,
fence: Identity & { cellId: string; assignmentEpoch: number }
): Promise<{ outcome: string; assignmentEpoch: number }> {
const first = await admin.assign(fence)
if (first.outcome !== 'conflict' || first.assignmentEpoch !== fence.assignmentEpoch) {
return { outcome: first.outcome, assignmentEpoch: fence.assignmentEpoch }
}
const above = fence.assignmentEpoch + 1
const second = await admin.assign({ ...fence, assignmentEpoch: above })
console.warn(
JSON.stringify({ event: 'orca_relay_edge_fence_above_conflict', outcome: second.outcome, assignmentEpoch: above })
)
return { outcome: second.outcome, assignmentEpoch: above }
}
// The fence right after a host left (EdgeAssignments.fenceAway). Anything but applied/unchanged
// fails: unavailable, `conflict`, or `stale` (the edge holds a newer epoch, so it still claims the
// host); the failure is logged and recorded for retry. Returns whether the fence landed.
@@ -55,7 +76,7 @@ export async function fenceEdge(
let outcome = 'unavailable'
for (let attempt = 0; attempt < attempts; attempt += 1) {
try {
outcome = (await admin.assign(fence)).outcome
outcome = (await assignFence(admin, fence)).outcome
} catch (error) {
if (!(error instanceof EdgeAdminUnavailableError)) { throw error }
outcome = error.message
@@ -138,7 +159,7 @@ async function retryOne(
const assignmentEpoch = integer(row, 'assignment_epoch')
let outcome: string
try {
outcome = (await admin.assign({ ...identity, cellId, assignmentEpoch })).outcome
outcome = (await assignFence(admin, { ...identity, cellId, assignmentEpoch })).outcome
} catch (error) {
if (!(error instanceof EdgeAdminUnavailableError)) { throw error }
outcome = error.message