mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
relay: an edge fence refused as a same-epoch conflict goes once more one epoch above
At 1% the edge answered fences 'conflict': it holds the fence's own epoch for another placement (a move onto the edge whose push landed though the director counted it failed, then a placement elsewhere minted the same epoch). Retries go to the host's current row at that same epoch, so they kept failing until the host moved again; meanwhile the edge object claims the host and its phones get HOST_OFFLINE instead of re-resolving. No later move can produce a same-epoch conflict (it would answer stale), so the fence now goes once more one epoch above, which lands; a later move onto the edge re-mints above it. Stale answers are never bumped.
This commit is contained in:
@@ -116,6 +116,43 @@ describe('relay edge fence after a host leaves', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('relay edge fence over a same-epoch conflict', () => {
|
||||
it('fences once more one epoch above a conflict at its epoch, and never bumps over a stale answer', async () => {
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const h = await harness()
|
||||
h.edge.answers = ['conflict', 'applied']
|
||||
expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 9)).toBe(true)
|
||||
expect(h.edge.calls.map((call) => call.body.assignmentEpoch)).toEqual([9, 10])
|
||||
expect(h.edge.calls.every((call) => call.body.cellId === 'us-c1')).toBe(true)
|
||||
expect(logged(warn, 'orca_relay_edge_fence_above_conflict')).toEqual([
|
||||
{ event: 'orca_relay_edge_fence_above_conflict', outcome: 'applied', assignmentEpoch: 10 }
|
||||
])
|
||||
expect(await h.pending()).toEqual([])
|
||||
// Stale: the edge holds a newer epoch (the host may be back there); no bump, recorded for retry.
|
||||
h.edge.calls = []
|
||||
h.edge.answers = ['stale']
|
||||
expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 11)).toBe(false)
|
||||
expect(h.edge.calls.map((call) => call.body.assignmentEpoch)).toEqual([11])
|
||||
})
|
||||
|
||||
it('a pending fence retried into a same-epoch conflict lands one epoch above the row', async () => {
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const h = await harness()
|
||||
await h.placeRow('us-c1', 7)
|
||||
h.edge.answers = ['down', 'down', 'down', 'down', 'down', 'down', 'down', 'down', 'down', 'down']
|
||||
expect(await h.assignments.fenceAway({ userId: USER, relayHostId: HOST }, 'us-c1', 7)).toBe(false)
|
||||
expect(await h.pending()).toHaveLength(1)
|
||||
h.edge.calls = []
|
||||
h.edge.answers = ['conflict', 'applied']
|
||||
h.clock.now += 60_000
|
||||
await h.retry()
|
||||
expect(h.edge.calls.map((call) => call.body.assignmentEpoch)).toEqual([7, 8])
|
||||
expect(await h.pending()).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('relay edge pending fence retries', () => {
|
||||
it('re-fences to the current row, not the recorded one, and clears the record once it lands', async () => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
@@ -41,6 +41,27 @@ export async function recordPendingFence(
|
||||
)
|
||||
}
|
||||
|
||||
// One fence call. `conflict` means the edge holds this very epoch for another placement (a move onto
|
||||
// the edge whose push landed though the director counted it as failed, then a placement elsewhere
|
||||
// minted the same epoch). No later move can produce that (a later move has a higher epoch, which
|
||||
// answers `stale`), so the fence goes once more one epoch above: it lands, and a later move onto
|
||||
// the edge re-mints above it as for any edge epoch (moveToEdge).
|
||||
async function assignFence(
|
||||
admin: EdgeAdmin,
|
||||
fence: Identity & { cellId: string; assignmentEpoch: number }
|
||||
): Promise<{ outcome: string; assignmentEpoch: number }> {
|
||||
const first = await admin.assign(fence)
|
||||
if (first.outcome !== 'conflict' || first.assignmentEpoch !== fence.assignmentEpoch) {
|
||||
return { outcome: first.outcome, assignmentEpoch: fence.assignmentEpoch }
|
||||
}
|
||||
const above = fence.assignmentEpoch + 1
|
||||
const second = await admin.assign({ ...fence, assignmentEpoch: above })
|
||||
console.warn(
|
||||
JSON.stringify({ event: 'orca_relay_edge_fence_above_conflict', outcome: second.outcome, assignmentEpoch: above })
|
||||
)
|
||||
return { outcome: second.outcome, assignmentEpoch: above }
|
||||
}
|
||||
|
||||
// The fence right after a host left (EdgeAssignments.fenceAway). Anything but applied/unchanged
|
||||
// fails: unavailable, `conflict`, or `stale` (the edge holds a newer epoch, so it still claims the
|
||||
// host); the failure is logged and recorded for retry. Returns whether the fence landed.
|
||||
@@ -55,7 +76,7 @@ export async function fenceEdge(
|
||||
let outcome = 'unavailable'
|
||||
for (let attempt = 0; attempt < attempts; attempt += 1) {
|
||||
try {
|
||||
outcome = (await admin.assign(fence)).outcome
|
||||
outcome = (await assignFence(admin, fence)).outcome
|
||||
} catch (error) {
|
||||
if (!(error instanceof EdgeAdminUnavailableError)) { throw error }
|
||||
outcome = error.message
|
||||
@@ -138,7 +159,7 @@ async function retryOne(
|
||||
const assignmentEpoch = integer(row, 'assignment_epoch')
|
||||
let outcome: string
|
||||
try {
|
||||
outcome = (await admin.assign({ ...identity, cellId, assignmentEpoch })).outcome
|
||||
outcome = (await assignFence(admin, { ...identity, cellId, assignmentEpoch })).outcome
|
||||
} catch (error) {
|
||||
if (!(error instanceof EdgeAdminUnavailableError)) { throw error }
|
||||
outcome = error.message
|
||||
|
||||
Reference in New Issue
Block a user