From 0c0cd4591ac4e9a55800dc5201a4e719cb3a5dbe Mon Sep 17 00:00:00 2001 From: Neil Date: Tue, 10 Nov 2026 16:48:15 -0800 Subject: [PATCH] relay: an edge fence refused as a same-epoch conflict goes once more one epoch above At 1% the edge answered fences 'conflict': it holds the fence's own epoch for another placement (a move onto the edge whose push landed though the director counted it failed, then a placement elsewhere minted the same epoch). Retries go to the host's current row at that same epoch, so they kept failing until the host moved again; meanwhile the edge object claims the host and its phones get HOST_OFFLINE instead of re-resolving. No later move can produce a same-epoch conflict (it would answer stale), so the fence now goes once more one epoch above, which lands; a later move onto the edge re-mints above it. Stale answers are never bumped. --- cloud/apps/relay/src/edge-relay-fence.test.ts | 37 +++++++++++++++++++ cloud/apps/relay/src/edge-relay-fence.ts | 25 ++++++++++++- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/cloud/apps/relay/src/edge-relay-fence.test.ts b/cloud/apps/relay/src/edge-relay-fence.test.ts index abdef6a01ba..bfee89d38c3 100644 --- a/cloud/apps/relay/src/edge-relay-fence.test.ts +++ b/cloud/apps/relay/src/edge-relay-fence.test.ts @@ -116,6 +116,43 @@ describe('relay edge fence after a host leaves', () => { }) }) +describe('relay edge fence over a same-epoch conflict', () => { + it('fences once more one epoch above a conflict at its epoch, and never bumps over a stale answer', async () => { + vi.spyOn(console, 'error').mockImplementation(() => {}) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const h = await harness() + h.edge.answers = ['conflict', 'applied'] + expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)).toBe(true) + expect(h.edge.calls.map((call) => call.body.assignmentEpoch)).toEqual([9, 10]) + expect(h.edge.calls.every((call) => call.body.cellId === 'us-c1')).toBe(true) + expect(logged(warn, 'orca_relay_edge_fence_above_conflict')).toEqual([ + { event: 'orca_relay_edge_fence_above_conflict', outcome: 'applied', assignmentEpoch: 10 } + ]) + expect(await h.pending()).toEqual([]) + // Stale: the edge holds a newer epoch (the host may be back there); no bump, recorded for retry. + h.edge.calls = [] + h.edge.answers = ['stale'] + expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 11)).toBe(false) + expect(h.edge.calls.map((call) => call.body.assignmentEpoch)).toEqual([11]) + }) + + it('a pending fence retried into a same-epoch conflict lands one epoch above the row', async () => { + vi.spyOn(console, 'error').mockImplementation(() => {}) + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const h = await harness() + await h.placeRow('us-c1', 7) + h.edge.answers = ['down', 'down', 'down', 'down', 'down', 'down', 'down', 'down', 'down', 'down'] + expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 7)).toBe(false) + expect(await h.pending()).toHaveLength(1) + h.edge.calls = [] + h.edge.answers = ['conflict', 'applied'] + h.clock.now += 60_000 + await h.retry() + expect(h.edge.calls.map((call) => call.body.assignmentEpoch)).toEqual([7, 8]) + expect(await h.pending()).toEqual([]) + }) +}) + describe('relay edge pending fence retries', () => { it('re-fences to the current row, not the recorded one, and clears the record once it lands', async () => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) diff --git a/cloud/apps/relay/src/edge-relay-fence.ts b/cloud/apps/relay/src/edge-relay-fence.ts index 3d01795ddaa..e18056b5eeb 100644 --- a/cloud/apps/relay/src/edge-relay-fence.ts +++ b/cloud/apps/relay/src/edge-relay-fence.ts @@ -41,6 +41,27 @@ export async function recordPendingFence( ) } +// One fence call. `conflict` means the edge holds this very epoch for another placement (a move onto +// the edge whose push landed though the director counted it as failed, then a placement elsewhere +// minted the same epoch). No later move can produce that (a later move has a higher epoch, which +// answers `stale`), so the fence goes once more one epoch above: it lands, and a later move onto +// the edge re-mints above it as for any edge epoch (moveToEdge). +async function assignFence( + admin: EdgeAdmin, + fence: Identity & { cellId: string; assignmentEpoch: number } +): Promise<{ outcome: string; assignmentEpoch: number }> { + const first = await admin.assign(fence) + if (first.outcome !== 'conflict' || first.assignmentEpoch !== fence.assignmentEpoch) { + return { outcome: first.outcome, assignmentEpoch: fence.assignmentEpoch } + } + const above = fence.assignmentEpoch + 1 + const second = await admin.assign({ ...fence, assignmentEpoch: above }) + console.warn( + JSON.stringify({ event: 'orca_relay_edge_fence_above_conflict', outcome: second.outcome, assignmentEpoch: above }) + ) + return { outcome: second.outcome, assignmentEpoch: above } +} + // The fence right after a host left (EdgeAssignments.fenceAway). Anything but applied/unchanged // fails: unavailable, `conflict`, or `stale` (the edge holds a newer epoch, so it still claims the // host); the failure is logged and recorded for retry. Returns whether the fence landed. @@ -55,7 +76,7 @@ export async function fenceEdge( let outcome = 'unavailable' for (let attempt = 0; attempt < attempts; attempt += 1) { try { - outcome = (await admin.assign(fence)).outcome + outcome = (await assignFence(admin, fence)).outcome } catch (error) { if (!(error instanceof EdgeAdminUnavailableError)) { throw error } outcome = error.message @@ -138,7 +159,7 @@ async function retryOne( const assignmentEpoch = integer(row, 'assignment_epoch') let outcome: string try { - outcome = (await admin.assign({ ...identity, cellId, assignmentEpoch })).outcome + outcome = (await assignFence(admin, { ...identity, cellId, assignmentEpoch })).outcome } catch (error) { if (!(error instanceof EdgeAdminUnavailableError)) { throw error } outcome = error.message