diff --git a/src/cli/specs/orchestration.ts b/src/cli/specs/orchestration.ts index 20008e45749..1ac088a30ed 100644 --- a/src/cli/specs/orchestration.ts +++ b/src/cli/specs/orchestration.ts @@ -89,8 +89,8 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [ summary: "Check this agent's messages", usage: 'orca orchestration check [--terminal ] [--run ] [--ack ] [--unread | --peek | --all] [--types ] [--format] [--wait] [--timeout-ms ] [--retry-request ] [--json]\n' + - " default: return the bound Run's oldest unacknowledged FIFO batch.\n" + - ' --ack: acknowledge the prior whole batch before checking/waiting.\n' + + " default: return the bound Run's oldest unread FIFO batch and mark it handled.\n" + + ' --ack: optional; acknowledges a batch from an older Orca that does not mark it.\n' + ' --peek: return only unread messages without marking them read.\n' + ' --all: return every message for the handle; does not mark read.\n' + ' --wait: block until a matching message arrives or --timeout-ms expires.\n' + @@ -117,7 +117,7 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [ 'On Windows PowerShell, quote comma-separated type filters, e.g. --types "worker_done,escalation".', '--types is the wake condition for --wait; a returned Delivery is always the whole FIFO batch, so it is never filtered by type. Without --wait it has no effect on consuming checks. Only --peek and --all filter their rows.', '--format renders the returned rows as local text only; it never writes to another terminal.', - 'A bound Run replays the same Delivery until --ack or all its messages are marked read, even with --types; process every message before acknowledging. Several --wait calls may wait on one Run inbox; waiting reserves nothing, listeners woken together get the same Delivery, and a repeated --ack is a no-op.' + 'A consuming check marks the batch it returns as handled, so the next check returns newer mail; no --ack is needed. If the response was lost, rerun with the --retry-request it printed to see the same batch, or use --all to see every message. Several --wait calls may wait on one inbox; waiting reserves nothing, listeners woken together get the same Delivery, and --ack of a handled Delivery is a no-op.' ] }, { diff --git a/src/main/runtime/orchestration-cli-restart-handoff.test.ts b/src/main/runtime/orchestration-cli-restart-handoff.test.ts index 2a5244dc8c2..152eaa033f2 100644 --- a/src/main/runtime/orchestration-cli-restart-handoff.test.ts +++ b/src/main/runtime/orchestration-cli-restart-handoff.test.ts @@ -172,14 +172,16 @@ it("retains CLI work and mail across restart, refuses a worker's own Run, and co expect(await cli(coord, ['task-list'])).toMatchObject({ result: { tasks: [{ id: task, status: 'dispatched', dispatch_id: dispatch }] } }) - expect(check.parse(await cli(coord, ['check'])).result).toEqual(retainedBatch) + expect(await cli(coord, ['check'])).toMatchObject({ result: { count: 0, deliveryId: null } }) + expect(await cli(coord, ['check', '--all'])).toMatchObject({ + result: { runId: root, messages: [{ id: retainedMessage }] } + }) const bindingBefore = db.getRunRaw(root) expect(await cli(worker, ['run-use', '--id', root], 1)).toMatchObject({ error: { code: 'worker_cannot_coordinate_own_run', data: { effectsApplied: false } } }) expect(db.getRunRaw(root)).toEqual(bindingBefore) - expect(check.parse(await cli(coord, ['check'])).result).toEqual(retainedBatch) expect(await cli(coord, ['check', '--ack', retainedBatch.deliveryId])).toMatchObject({ result: { acknowledged: retainedBatch.deliveryId, count: 0 } }) @@ -221,11 +223,10 @@ it("retains CLI work and mail across restart, refuses a worker's own Run, and co } ] }) - expect(db.getMessageById(reply)?.read).toBe(0) + expect(db.getMessageById(reply)?.read).toBe(1) expect(await cli(worker, ['check', '--ack', replyBatch.deliveryId])).toMatchObject({ result: { runId: separate, acknowledged: replyBatch.deliveryId, count: 0 } }) - expect(db.getMessageById(reply)?.read).toBe(1) await cli(coord, ['check', '--ack', questionBatch.deliveryId]) const continuedDispatch = idOf( diff --git a/src/main/runtime/orchestration-dispatch-mailbox-delivery.spec.ts b/src/main/runtime/orchestration-dispatch-mailbox-delivery.spec.ts index dec0dcdf828..1bc46a5879a 100644 --- a/src/main/runtime/orchestration-dispatch-mailbox-delivery.spec.ts +++ b/src/main/runtime/orchestration-dispatch-mailbox-delivery.spec.ts @@ -27,7 +27,7 @@ describe('Dispatch mailbox Delivery', () => { } }) - it('wakes once, replays after restart, and leaves concurrent guidance for the next ack', async () => { + it('wakes once and hands newer guidance after restart to the next check without an ack', async () => { vi.useFakeTimers() const directory = mkdtempSync(join(tmpdir(), 'orca-dispatch-delivery-')) temporaryDirectories.push(directory) @@ -68,7 +68,8 @@ describe('Dispatch mailbox Delivery', () => { const issued = await checkBoundMailbox(first.runtime) expect(issued).toMatchObject({ dispatchId: dispatch.id, count: 1, replayed: false }) expect(issued.messages).toEqual([expect.objectContaining({ id: firstMessage.id })]) - expect(firstDb.getWorkerAttentionFacts(dispatch.id, Date.now()).pendingGuidance).toBe(true) + expect(firstDb.getMessageById(firstMessage.id)?.read).toBe(1) + expect(firstDb.getWorkerAttentionFacts(dispatch.id, Date.now()).pendingGuidance).toBe(false) firstDb.insertMessage({ from: run.coordinator_handle!, to: address, @@ -81,28 +82,20 @@ describe('Dispatch mailbox Delivery', () => { const restarted = createRuntime(restartedDb) await driveToLiveIdle(restarted.runtime) await vi.advanceTimersByTimeAsync(2_500) - // The held batch defers the concurrent mail's nudge, and that nudge names the batch to ack. - expect( - restarted.write.mock.calls.filter(([, payload]) => - String(payload).includes(`--ack ${issued.deliveryId}`) - ) - ).toHaveLength(1) expect(pointerCount(restarted.write)).toBe(1) + expect( + restarted.write.mock.calls.filter(([, payload]) => String(payload).includes('--ack')) + ).toHaveLength(0) - const replayed = await checkBoundMailbox(restarted.runtime) - expect(replayed).toMatchObject({ - dispatchId: dispatch.id, - deliveryId: issued.deliveryId, - count: 1, - replayed: true - }) - const next = await checkBoundMailbox(restarted.runtime, { ack: replayed.deliveryId! }) + const next = await checkBoundMailbox(restarted.runtime) + expect(next).toMatchObject({ dispatchId: dispatch.id, count: 1, replayed: false }) expect(next.messages).toEqual([expect.objectContaining({ subject: 'Concurrent follow-up' })]) - expect(restartedDb.getMessageById(firstMessage.id)?.read).toBe(1) - expect(restartedDb.getWorkerAttentionFacts(dispatch.id, Date.now()).pendingGuidance).toBe(true) - - await checkBoundMailbox(restarted.runtime, { ack: next.deliveryId! }) expect(restartedDb.getWorkerAttentionFacts(dispatch.id, Date.now()).pendingGuidance).toBe(false) + // An older CLI's explicit ack of an already handled batch stays a harmless no-op. + expect(await checkBoundMailbox(restarted.runtime, { ack: issued.deliveryId! })).toMatchObject({ + acknowledged: issued.deliveryId, + count: 0 + }) restartedDb.close() }) @@ -159,10 +152,6 @@ describe('Dispatch mailbox Delivery', () => { ) const delivery = await checkBoundMailbox(restarted.runtime) expect(delivery.messages).toEqual([expect.objectContaining({ id: message.id })]) - expect(restartedDb.getWorkerAttentionFacts(dispatch.id, Date.now()).pendingGuidance).toBe( - true - ) - await checkBoundMailbox(restarted.runtime, { ack: delivery.deliveryId! }) expect(restartedDb.getWorkerAttentionFacts(dispatch.id, Date.now()).pendingGuidance).toBe( false ) diff --git a/src/main/runtime/orchestration-mailbox-detached-routing.spec.ts b/src/main/runtime/orchestration-mailbox-detached-routing.spec.ts index 2dfd1447934..062a728b533 100644 --- a/src/main/runtime/orchestration-mailbox-detached-routing.spec.ts +++ b/src/main/runtime/orchestration-mailbox-detached-routing.spec.ts @@ -55,11 +55,9 @@ describe('orchestration detached mailbox routing', () => { expect(checked.messages).toEqual([expect.objectContaining({ id: message.id })]) expect(db.getMessageById(message.id)).toMatchObject({ to_handle: `dispatch:${dispatch.id}`, - read: 0, + read: 1, delivered_at: expect.any(String) }) - await checkBoundMailbox(harness.runtime, { ack: checked.deliveryId! }) - expect(db.getMessageById(message.id)?.read).toBe(1) db.close() }) diff --git a/src/main/runtime/orchestration-mailbox-notification-consistency.test.ts b/src/main/runtime/orchestration-mailbox-notification-consistency.test.ts index 6556bdaa4bb..cb2a621c741 100644 --- a/src/main/runtime/orchestration-mailbox-notification-consistency.test.ts +++ b/src/main/runtime/orchestration-mailbox-notification-consistency.test.ts @@ -32,7 +32,6 @@ import { WORKTREE_ID } from './orchestration-mailbox-notification-test-harness' import { RpcDispatcher } from './rpc/dispatcher' -import { localOrchestrationCliCommand } from './orchestration/cli-command' import { ORCHESTRATION_METHODS } from './rpc/methods/orchestration' import { MAILBOX_POINTER_WRITE_ATTEMPTED } from './orchestration/db/messages/mailbox-pointer-enter-state' @@ -762,7 +761,7 @@ describe('orchestration notification mailbox consistency', () => { ) expect(db.getMessageById(messages[0]!.id)).toMatchObject({ to_handle: `run:${run.id}`, - read: 0, + read: 1, delivered_at: expect.any(String) }) expect(db.getMessageById(messages[50]!.id)?.delivered_at).toBeNull() @@ -837,47 +836,25 @@ describe('orchestration notification mailbox consistency', () => { db.close() }) - it('points newer Run mail behind an unacked Delivery once, naming the Delivery', async () => { + it('points newer Run mail with the normal text when the coordinator never acked', async () => { vi.useFakeTimers() const db = new OrchestrationDb(':memory:') const harness = createRuntime(db) - const run = createBoundRun(db, 'Outstanding Delivery Run') + const run = createBoundRun(db, 'Never-acked Run') const insert = (subject: string) => db.insertMessage({ from: 'term_worker', to: `run:${run.id}`, subject, runId: run.id }) const firstMessage = insert('First status') const firstDelivery = await checkBoundMailbox(harness.runtime) + expect(firstDelivery.messages).toEqual([expect.objectContaining({ id: firstMessage.id })]) const newerMessage = insert('Newer status') - const enters = () => harness.write.mock.calls.filter(([, payload]) => payload === '\r').length await driveToLiveIdle(harness.runtime) const pointers = harness.write.mock.calls.filter(([, payload]) => isMailboxPointer(payload)) expect(pointers).toHaveLength(1) - expect(String(pointers[0]![1])).toContain( - `after delivery ${firstDelivery.deliveryId}, which you have not acknowledged. If you no longer have it, run \`${localOrchestrationCliCommand()} orchestration check --run ${run.id}\` to see it again; once it is handled, run \`${localOrchestrationCliCommand()} orchestration check --run ${run.id} --ack ${firstDelivery.deliveryId}\`` - ) + expect(String(pointers[0]![1])).toContain('You have 1 orchestration message.') + expect(String(pointers[0]![1])).not.toContain('acknowledged') - // A turn that starts inside the Enter delay keeps the reminder until it ends. - harness.runtime.onPtyData(PTY_ID, '\x1b]0;Codex working\x07', 3) - await harness.runtime.serializeMainTerminalBuffer(PTY_ID) - await vi.advanceTimersByTimeAsync(500) - expect(enters()).toBe(0) - harness.runtime.onPtyData(PTY_ID, '\x1b]0;Codex done\x07', 4) - await harness.runtime.serializeMainTerminalBuffer(PTY_ID) - await vi.advanceTimersByTimeAsync(500) - expect(enters()).toBe(1) - expect(db.getMessageById(newerMessage.id)?.delivered_at).toEqual(expect.any(String)) - - harness.runtime.onPtyData(PTY_ID, '\x1b]0;Codex working\x07', 5) - harness.runtime.onPtyData(PTY_ID, '\x1b]0;Codex done\x07', 6) - await harness.runtime.serializeMainTerminalBuffer(PTY_ID) - await vi.advanceTimersByTimeAsync(500) - expect(pointerCount(harness.write)).toBe(1) - expect(enters()).toBe(1) - - const replayed = await checkBoundMailbox(harness.runtime) - expect(replayed.messages).toEqual([expect.objectContaining({ id: firstMessage.id })]) - expect(replayed.deliveryId).toBe(firstDelivery.deliveryId) - const next = await checkBoundMailbox(harness.runtime, { ack: firstDelivery.deliveryId! }) + const next = await checkBoundMailbox(harness.runtime) expect(next.messages).toEqual([expect.objectContaining({ id: newerMessage.id })]) expect(next.deliveryId).not.toBe(firstDelivery.deliveryId) db.close() diff --git a/src/main/runtime/orchestration-message-delivery-identity.test.ts b/src/main/runtime/orchestration-message-delivery-identity.test.ts index d456c5541ed..3f31aa4f759 100644 --- a/src/main/runtime/orchestration-message-delivery-identity.test.ts +++ b/src/main/runtime/orchestration-message-delivery-identity.test.ts @@ -218,38 +218,33 @@ describe('STA-4325 message and delivery identity', () => { }) await driveToLiveIdle(runtime) - const [first, concurrent] = await Promise.all([check(runtime), check(runtime)]) const expectedIds = [status.id, dispatch.id, done.id] - - expect(pointerPayloads(write)).toEqual([expect.stringContaining('3 orchestration messages')]) - expect(first).toMatchObject({ runId: run.id, count: 3, replayed: false }) - expect(concurrent).toMatchObject({ runId: run.id, count: 3, replayed: true }) - expect(first.deliveryId).toBeTruthy() - expect(concurrent.deliveryId).toBe(first.deliveryId) - expect(first.messages.map((message) => message.id)).toEqual(expectedIds) - expect(concurrent.messages.map((message) => message.id)).toEqual(expectedIds) - const peeked = await check(runtime, { peek: true, unread: false }) const filtered = await check(runtime, { peek: true, unread: false, types: 'worker_done' }) - const all = await check(runtime, { all: true }) expect(peeked.messages.map((message) => message.id)).toEqual( expect.arrayContaining(expectedIds) ) expect(peeked.count).toBe(3) expect(filtered.messages.map((message) => message.id)).toEqual([done.id]) - expect(all.messages.map((message) => message.id)).toEqual(expect.arrayContaining(expectedIds)) - expect(all.count).toBe(3) - - const deliveryRowsBeforeAck = sqliteFor(db) - .prepare('SELECT id, status, message_ids FROM deliveries ORDER BY rowid') - .all() as { id: string; status: string; message_ids: string }[] - expect(deliveryRowsBeforeAck).toEqual([ - { id: first.deliveryId, status: 'outstanding', message_ids: JSON.stringify(expectedIds) } - ]) for (const id of expectedIds) { expect(db.getMessageById(id)).toMatchObject({ to_handle: `run:${run.id}`, read: 0 }) } + const [first, concurrent] = await Promise.all([check(runtime), check(runtime)]) + expect(pointerPayloads(write)).toEqual([expect.stringContaining('3 orchestration messages')]) + expect(first).toMatchObject({ runId: run.id, count: 3, replayed: false }) + expect(concurrent).toMatchObject({ runId: run.id, count: 0, deliveryId: null }) + expect(first.deliveryId).toBeTruthy() + expect(first.messages.map((message) => message.id)).toEqual(expectedIds) + const all = await check(runtime, { all: true }) + expect(all.messages.map((message) => message.id)).toEqual(expect.arrayContaining(expectedIds)) + expect(all.count).toBe(3) + expect( + sqliteFor(db).prepare('SELECT id, status, message_ids FROM deliveries ORDER BY rowid').all() + ).toEqual([ + { id: first.deliveryId, status: 'acknowledged', message_ids: JSON.stringify(expectedIds) } + ]) + const acknowledged = await check(runtime, { ack: first.deliveryId }) expect(acknowledged).toMatchObject({ count: 0, @@ -271,7 +266,7 @@ describe('STA-4325 message and delivery identity', () => { db.close() }) - it('replays one outstanding Delivery across restart and wakes a filtered waiter once', async () => { + it('keeps a handled Delivery handled across restart and wakes a filtered waiter once', async () => { vi.useFakeTimers() const fixture = createDatabase('orca-sta-4325-restart-') const firstRuntime = createRuntime(fixture.db) @@ -297,16 +292,11 @@ describe('STA-4325 message and delivery identity', () => { const restarted = createRuntime(reopened, REMINTED_TERMINAL_HANDLE) await driveToLiveIdle(restarted.runtime) const afterRestart = await check(restarted.runtime, { terminal: REMINTED_TERMINAL_HANDLE }) - expect(afterRestart).toMatchObject({ deliveryId: beforeRestart.deliveryId, replayed: true }) - expect(afterRestart.messages.map((message) => message.id)).toEqual([status.id]) + expect(afterRestart).toMatchObject({ deliveryId: null, count: 0 }) expect(sqliteFor(reopened).prepare('SELECT id FROM deliveries').all()).toEqual([ { id: beforeRestart.deliveryId } ]) - await check(restarted.runtime, { - terminal: REMINTED_TERMINAL_HANDLE, - ack: afterRestart.deliveryId - }) const waiting = check(restarted.runtime, { terminal: REMINTED_TERMINAL_HANDLE, wait: true, diff --git a/src/main/runtime/orchestration/__snapshots__/preamble.test.ts.snap b/src/main/runtime/orchestration/__snapshots__/preamble.test.ts.snap index 0d31c90b926..d5f22ce2542 100644 --- a/src/main/runtime/orchestration/__snapshots__/preamble.test.ts.snap +++ b/src/main/runtime/orchestration/__snapshots__/preamble.test.ts.snap @@ -64,9 +64,9 @@ Don't post to Slack, GitHub, or other channels during the run; report through th # you send worker_done, so a redirect lands before the task settles. orca orchestration check --terminal orca_session_id:WORKER --json - # After processing every message, ack the returned deliveryId; this also reads - # the next batch. Repeat until count is 0, then resume. Unacked batches replay - # and hide newer mail. + # Each check marks the batch it returns as handled; repeat until count is 0, + # then resume. Only if a batch lacks "deliveryAcknowledged": true (an older + # Orca), ack its deliveryId so it does not replay and hide newer mail: orca orchestration check --terminal orca_session_id:WORKER --ack --json \`\`\` @@ -154,9 +154,9 @@ Don't post to Slack, GitHub, or other channels during the run; report through th # you send worker_done, so a redirect lands before the task settles. orca orchestration check --terminal term_WORKER --json - # After processing every message, ack the returned deliveryId; this also reads - # the next batch. Repeat until count is 0, then resume. Unacked batches replay - # and hide newer mail. + # Each check marks the batch it returns as handled; repeat until count is 0, + # then resume. Only if a batch lacks "deliveryAcknowledged": true (an older + # Orca), ack its deliveryId so it does not replay and hide newer mail: orca orchestration check --terminal term_WORKER --ack --json \`\`\` diff --git a/src/main/runtime/orchestration/db/messages/role-mailbox-delivery.ts b/src/main/runtime/orchestration/db/messages/role-mailbox-delivery.ts index 3d10eec23cc..910feddec25 100644 --- a/src/main/runtime/orchestration/db/messages/role-mailbox-delivery.ts +++ b/src/main/runtime/orchestration/db/messages/role-mailbox-delivery.ts @@ -173,6 +173,54 @@ export function acknowledgeMailboxDelivery( } } +// A consuming read hands over the batch it returns; a lost response replays from its request receipt. +export function takeMailboxDelivery( + this: OrchestrationDb, + params: Parameters[0] +): { delivery: DeliveryRow; messages: MessageRow[]; replayed: boolean } | undefined { + const current = this.getOrCreateMailboxDelivery(params) + if (!current) { + return undefined + } + const { delivery } = this.acknowledgeMailboxDelivery({ + ...params, + deliveryId: current.delivery.id + }) + return { ...current, delivery } +} + +// Global rather than per-mailbox so reading it is a rowid lookup, not a table scan. +export function getDeliveryCursor(this: OrchestrationDb): number { + const row = this.db.prepare('SELECT MAX(rowid) AS cursor FROM deliveries').get() as { + cursor: number | null + } + return row.cursor ?? 0 +} + +// Waiters woken by the same arrival share its batch, so a forgotten waiter cannot swallow it. +export function getMailboxDeliveryTakenAfter( + this: OrchestrationDb, + mailboxHandle: string, + consumerGeneration: number, + cursor: number +): { delivery: DeliveryRow; messages: MessageRow[]; replayed: boolean } | undefined { + const delivery = this.db + .prepare( + `SELECT * FROM deliveries + WHERE rowid > ? AND mailbox_handle = ? AND consumer_generation = ? + AND status = 'acknowledged' + ORDER BY rowid ASC LIMIT 1` + ) + .get(cursor, mailboxHandle, consumerGeneration) as DeliveryRow | undefined + return delivery + ? { + delivery: exposeDeliveryTimestamps(delivery), + messages: this.getDeliveryMessages(delivery), + replayed: true + } + : undefined +} + export function hasOutstandingMailboxDelivery( this: OrchestrationDb, mailboxHandle: string @@ -216,6 +264,9 @@ export type RoleMailboxDeliveryMethods = { getDeliveryMessages: typeof getDeliveryMessages getOrCreateMailboxDelivery: typeof getOrCreateMailboxDelivery acknowledgeMailboxDelivery: typeof acknowledgeMailboxDelivery + takeMailboxDelivery: typeof takeMailboxDelivery + getDeliveryCursor: typeof getDeliveryCursor + getMailboxDeliveryTakenAfter: typeof getMailboxDeliveryTakenAfter hasOutstandingMailboxDelivery: typeof hasOutstandingMailboxDelivery getOutstandingMailboxDelivery: typeof getOutstandingMailboxDelivery fenceUnacknowledgedMailboxDeliveries: typeof fenceUnacknowledgedMailboxDeliveries @@ -227,6 +278,9 @@ export function attachRoleMailboxDelivery(ctor: { prototype: object }): void { getDeliveryMessages, getOrCreateMailboxDelivery, acknowledgeMailboxDelivery, + takeMailboxDelivery, + getDeliveryCursor, + getMailboxDeliveryTakenAfter, hasOutstandingMailboxDelivery, getOutstandingMailboxDelivery, fenceUnacknowledgedMailboxDeliveries diff --git a/src/main/runtime/orchestration/db/runs/run-delivery.ts b/src/main/runtime/orchestration/db/runs/run-delivery.ts index 2afac2e4285..36ca087b18c 100644 --- a/src/main/runtime/orchestration/db/runs/run-delivery.ts +++ b/src/main/runtime/orchestration/db/runs/run-delivery.ts @@ -36,6 +36,22 @@ export function getOrCreateRunDelivery( }) } +export function takeRunDelivery( + this: OrchestrationDb, + params: { + runId: string + consumerGeneration: number + wakeTypes?: MessageType[] + } +): { delivery: DeliveryRow; messages: MessageRow[]; replayed: boolean } | undefined { + return this.takeMailboxDelivery({ + runId: params.runId, + mailboxHandle: `run:${params.runId}`, + consumerGeneration: params.consumerGeneration, + wakeTypes: params.wakeTypes + }) +} + export function acknowledgeRunDelivery( this: OrchestrationDb, params: { @@ -119,6 +135,7 @@ export function hasOutstandingRunDelivery(this: OrchestrationDb, runId: string): export type RunDeliveryMethods = { requireCurrentConsumer: typeof requireCurrentConsumer getOrCreateRunDelivery: typeof getOrCreateRunDelivery + takeRunDelivery: typeof takeRunDelivery acknowledgeRunDelivery: typeof acknowledgeRunDelivery getRunMailboxHistory: typeof getRunMailboxHistory getUnreadRunMailbox: typeof getUnreadRunMailbox @@ -129,6 +146,7 @@ export function attachRunDelivery(ctor: { prototype: object }): void { Object.assign(ctor.prototype, { requireCurrentConsumer, getOrCreateRunDelivery, + takeRunDelivery, acknowledgeRunDelivery, getRunMailboxHistory, getUnreadRunMailbox, diff --git a/src/main/runtime/orchestration/preamble.ts b/src/main/runtime/orchestration/preamble.ts index d861191924f..0ea0db5266d 100644 --- a/src/main/runtime/orchestration/preamble.ts +++ b/src/main/runtime/orchestration/preamble.ts @@ -127,9 +127,9 @@ ${heartbeat} # Ask the coordinator a question and block until it answers. # you send worker_done, so a redirect lands before the task settles. ${cli} orchestration check --terminal ${params.workerHandle} --json - # After processing every message, ack the returned deliveryId; this also reads - # the next batch. Repeat until count is 0, then resume. Unacked batches replay - # and hide newer mail. + # Each check marks the batch it returns as handled; repeat until count is 0, + # then resume. Only if a batch lacks "deliveryAcknowledged": true (an older + # Orca), ack its deliveryId so it does not replay and hide newer mail: ${cli} orchestration check --terminal ${params.workerHandle} --ack --json \`\`\` diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/check-run.ts b/src/main/runtime/rpc/methods/orchestration/messaging/check-run.ts index d0c633db196..26655096885 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/check-run.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/check-run.ts @@ -133,7 +133,7 @@ export async function checkRunMailbox(args: { }) const readPeek = () => db.getUnreadRunMailbox(run.id, 100, typeFilter) const readDelivery = (wakeTypes?: MessageType[]) => - db.getOrCreateRunDelivery({ runId: run.id, consumerGeneration: generation, wakeTypes }) + db.takeRunDelivery({ runId: run.id, consumerGeneration: generation, wakeTypes }) let peeked = params.peek ? readPeek() : [] if (params.peek && peeked.length > 0) { return peekResult(peeked) @@ -146,6 +146,7 @@ export async function checkRunMailbox(args: { messages: exposeMessages(current.messages), count: current.messages.length, replayed: current.replayed, + deliveryAcknowledged: true, acknowledged: acknowledged ?? null, timedOut: false, cancelled: false, @@ -171,6 +172,7 @@ export async function checkRunMailbox(args: { } } + const deliveryCursor = db.getDeliveryCursor() const waitResult = await runtime.waitForMessage(address, { typeFilter: typeFilter as string[] | undefined, timeoutMs: params.timeoutMs ?? undefined, @@ -233,13 +235,15 @@ export async function checkRunMailbox(args: { peeked = readPeek() return { ...peekResult(peeked), timedOut: false, cancelled: false, connectionLost: false } } - current = readDelivery(typeFilter) + current = + readDelivery(typeFilter) ?? db.getMailboxDeliveryTakenAfter(address, generation, deliveryCursor) return { runId: run.id, deliveryId: current?.delivery.id ?? null, messages: exposeMessages(current?.messages ?? []), count: current?.messages.length ?? 0, replayed: current?.replayed ?? false, + ...(current ? { deliveryAcknowledged: true } : {}), acknowledged: acknowledged ?? null, timedOut: false, cancelled: false, diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/check-worker-consumer-fencing.test.ts b/src/main/runtime/rpc/methods/orchestration/messaging/check-worker-consumer-fencing.test.ts index 5f39a9637b8..0bdea7f566e 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/check-worker-consumer-fencing.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/check-worker-consumer-fencing.test.ts @@ -84,30 +84,23 @@ describe('orchestration.check on a re-attached Dispatch', () => { code: 'consumer_fenced', message: expect.stringContaining('no longer owns its Dispatch') }) - expect(db.getUnreadMessages(`dispatch:${dispatchId}`)).toHaveLength(1) }) - it('hands the live worker a fresh Delivery with the same unread mail', async () => { + it('leaves mail the replaced process already took in the live worker history', async () => { const dispatchId = attachedDispatchWithMail() - const staleDelivery = (await check(PANE_A)).deliveryId + await check(PANE_A) reattach(dispatchId) - const live = await check(PANE_B) - expect(live.deliveryId).not.toBe(staleDelivery) - expect(live.replayed).toBe(false) - expect(live.messages.map((message) => message.subject)).toEqual(['do the work']) - - await check(PANE_B, { ack: live.deliveryId }) - expect(db.getUnreadMessages(`dispatch:${dispatchId}`)).toEqual([]) + expect(await check(PANE_B)).toMatchObject({ deliveryId: null, count: 0 }) + const history = await check(PANE_B, { all: true }) + expect(history.messages.map((message) => message.subject)).toEqual(['do the work']) }) it('keeps serving a worker whose process restarted without a re-attach', async () => { attachedDispatchWithMail() const first = await check(PANE_A) - const replay = await check(PANE_A) - expect(replay.deliveryId).toBe(first.deliveryId) - expect(replay.replayed).toBe(true) + expect(await check(PANE_A)).toMatchObject({ deliveryId: null, count: 0 }) await expect(check(PANE_A, { ack: first.deliveryId })).resolves.toMatchObject({ acknowledged: first.deliveryId }) @@ -117,6 +110,12 @@ describe('orchestration.check on a re-attached Dispatch', () => { const dispatchId = attachedDispatchWithMail() await check(PANE_A) reattach(dispatchId) + db.insertMessage({ + from: 'term_coord', + to: `dispatch:${dispatchId}`, + subject: 'next step', + runId: db.getDispatchContextById(dispatchId)!.run_id + }) await expect(check(PANE_A)).rejects.toMatchObject({ code: 'consumer_fenced', @@ -125,8 +124,7 @@ describe('orchestration.check on a re-attached Dispatch', () => { expect(db.getUnreadMessages(`dispatch:${dispatchId}`)).toHaveLength(1) const live = await check(PANE_B) - expect(live.messages.map((message) => message.subject)).toEqual(['do the work']) - await check(PANE_B, { ack: live.deliveryId }) + expect(live.messages.map((message) => message.subject)).toEqual(['next step']) expect(db.getUnreadMessages(`dispatch:${dispatchId}`)).toEqual([]) }) diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/check-worker-federated-attachment.test.ts b/src/main/runtime/rpc/methods/orchestration/messaging/check-worker-federated-attachment.test.ts index f0d5e4933a6..40854dc7c96 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/check-worker-federated-attachment.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/check-worker-federated-attachment.test.ts @@ -95,7 +95,7 @@ describe('orchestration.check on a federated attachment across a restart', () => store.markRemoteAttachmentReady(dispatchId) } - it('replays the coordinator instruction and takes its ack after the app restarts', async () => { + it('keeps a handled coordinator instruction handled and acks idempotently after the app restarts', async () => { directory = mkdtempSync(join(tmpdir(), 'orca-federated-check-')) const path = join(directory, 'orchestration.db') @@ -123,12 +123,11 @@ describe('orchestration.check on a federated attachment across a restart', () => expect(deliveryId).not.toBeNull() ;(db as OrchestrationDb).close() - // The worker's process outlives the app; its instruction is still unacknowledged. + // The worker's process outlives the app; an older CLI may still ack the handled batch. const second = launch(path) - const replayed = await check(second) - expect(replayed.deliveryId).toBe(deliveryId) - expect(replayed.replayed).toBe(true) - expect(replayed.messages.map((message) => message.id)).toEqual(['msg_federated_1']) + expect(await check(second)).toMatchObject({ deliveryId: null, count: 0 }) + const history = await check(second, { all: true }) + expect(history.messages.map((message) => message.id)).toEqual(['msg_federated_1']) const acknowledged = await check(second, { ack: deliveryId }) expect(acknowledged.acknowledged).toBe(deliveryId) diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/check-worker.ts b/src/main/runtime/rpc/methods/orchestration/messaging/check-worker.ts index 73fc93a77c4..688c9271c20 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/check-worker.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/check-worker.ts @@ -1,4 +1,4 @@ -import type { MessageType, OrchestrationDb } from '../../../../orchestration/db' +import type { MessageRow, MessageType, OrchestrationDb } from '../../../../orchestration/db' import type { OrcaRuntimeService } from '../../../../orca-runtime' import { OrchestrationError } from '../../../../orchestration/orchestration-error' import { formatMessageBanner } from '../../../../orchestration/formatter' @@ -191,6 +191,10 @@ export async function checkWorkerMailbox(args: { interruptedAcknowledgedCheck(deliveryRunId, acknowledged.delivery.id, 'outcome_unknown') ) } + const formatted = (messages: MessageRow[]) => + params.format || params.inject + ? { formatted: messages.map(formatMessageBanner).join('\n\n') } + : {} const showAll = params.all === true || (params.unread === false && params.peek !== true) const readPeek = () => db.getUnreadMessages(address, typeFilter) const readDelivery = (wakeTypes?: MessageType[]) => { @@ -198,7 +202,7 @@ export async function checkWorkerMailbox(args: { return undefined } try { - return db.getOrCreateMailboxDelivery({ + return db.takeMailboxDelivery({ runId: deliveryRunId, mailboxHandle: address, consumerGeneration: workerMailbox.generation, @@ -216,9 +220,7 @@ export async function checkWorkerMailbox(args: { messages: exposeMessages(messages), count: messages.length, acknowledged: acknowledged?.delivery.id ?? null, - ...(params.format || params.inject - ? { formatted: messages.map(formatMessageBanner).join('\n\n') } - : {}) + ...formatted(messages) } } if (params.peek) { @@ -229,9 +231,7 @@ export async function checkWorkerMailbox(args: { messages: exposeMessages(messages), count: messages.length, acknowledged: acknowledged?.delivery.id ?? null, - ...(params.format || params.inject - ? { formatted: messages.map(formatMessageBanner).join('\n\n') } - : {}) + ...formatted(messages) } } } else { @@ -243,16 +243,16 @@ export async function checkWorkerMailbox(args: { messages: exposeMessages(current?.messages ?? []), count: current?.messages.length ?? 0, replayed: current?.replayed ?? false, + ...(current ? { deliveryAcknowledged: true } : {}), acknowledged: acknowledged?.delivery.id ?? null, timedOut: false, cancelled: false, connectionLost: false, - ...(params.format || params.inject - ? { formatted: current?.messages.map(formatMessageBanner).join('\n\n') ?? '' } - : {}) + ...formatted(current?.messages ?? []) } } } + const deliveryCursor = db.getDeliveryCursor() // Binding can happen during recovery, before run-create/run-use can cancel this wait. const waitResult = activeDispatch && currentDispatchAssigneeRun(runtime, db, activeDispatch) @@ -284,21 +284,20 @@ export async function checkWorkerMailbox(args: { messages: exposeMessages(arrived), count: arrived.length, acknowledged: acknowledged?.delivery.id ?? null, - ...(params.format || params.inject - ? { formatted: arrived.map(formatMessageBanner).join('\n\n') } - : {}) + ...formatted(arrived) } } - const arrived = readDelivery(typeFilter) + const arrived = + readDelivery(typeFilter) ?? + db.getMailboxDeliveryTakenAfter(address, workerMailbox.generation, deliveryCursor) return { ...mailboxIdentity, deliveryId: arrived?.delivery.id ?? null, messages: exposeMessages(arrived?.messages ?? []), count: arrived?.messages.length ?? 0, replayed: arrived?.replayed ?? false, + ...(arrived ? { deliveryAcknowledged: true } : {}), acknowledged: acknowledged?.delivery.id ?? null, - ...(params.format || params.inject - ? { formatted: arrived?.messages.map(formatMessageBanner).join('\n\n') ?? '' } - : {}) + ...formatted(arrived?.messages ?? []) } } diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/check.test.ts b/src/main/runtime/rpc/methods/orchestration/messaging/check.test.ts index 4c743f7c6e7..d251324a660 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/check.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/check.test.ts @@ -282,11 +282,6 @@ describe('orchestration RPC methods', () => { code: 'stale_delivery', message: `${queued.id} is a message id, not a delivery id. Acknowledge the batch with the deliveryId field from the check response; process the entire batch before acknowledging.` }) - expect(db.getMessageById(queued.id)).toMatchObject({ id: queued.id, read: 0 }) - expect(await call('orchestration.check', { terminal: 'term_coord' })).toMatchObject({ - deliveryId, - messages: [{ id: queued.id }] - }) expect( await call('orchestration.check', { terminal: 'term_coord', ack: deliveryId }) ).toMatchObject({ acknowledged: deliveryId }) @@ -400,7 +395,7 @@ describe('orchestration RPC methods', () => { expect(result.messages[0].type).toBe('worker_done') expect(db.getTask(task.id)?.status).toBe('completed') expect(db.getDispatchContextById(dispatch.id)?.status).toBe('completed') - expect(db.getUnreadMessages(`run:${activeRunId}`)).toHaveLength(1) + expect(db.getUnreadMessages(`run:${activeRunId}`)).toHaveLength(0) const taskList = (await call('orchestration.taskList', {})) as { tasks: { id: string @@ -423,12 +418,11 @@ describe('orchestration RPC methods', () => { const repeated = (await call('orchestration.check', { terminal: 'term_coord', types: 'worker_done' - })) as { count: number; deliveryId: string } - expect(repeated.count).toBe(1) - expect(repeated.deliveryId).toBe(result.deliveryId) + })) as { count: number } + expect(repeated.count).toBe(0) const acknowledged = (await call('orchestration.check', { terminal: 'term_coord', - ack: repeated.deliveryId, + ack: result.deliveryId, types: 'worker_done' })) as { count: number } expect(acknowledged.count).toBe(0) diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/run-bound-mailbox-interleavings.test.ts b/src/main/runtime/rpc/methods/orchestration/messaging/run-bound-mailbox-interleavings.test.ts index d8ad40d65a0..8c9e6036ad4 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/run-bound-mailbox-interleavings.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/run-bound-mailbox-interleavings.test.ts @@ -116,7 +116,7 @@ describe('Run binding during Dispatch checks', () => { }) it.each(['before', 'during recovery'] as const)( - 'replays Run delivery created %s residual acknowledgment', + 'serves residual mail around a Run delivery created %s residual acknowledgment', async (phase) => { const ids = Array.from( { length: ORCHESTRATION_DELIVERY_BATCH_LIMIT + 1 }, @@ -158,23 +158,24 @@ describe('Run binding during Dispatch checks', () => { } ) } - const result = await check({ ack }) - expect(result).toMatchObject({ - runId: run.id, - deliveryId: runDelivery, - acknowledged: ack, - replayed: true - }) + const tail = { runId: dispatch.run_id, messages: [{ id: last }], count: 1 } + if (phase === 'before') { + expect(await check({ ack })).toMatchObject({ ...tail, acknowledged: ack }) + } else { + // A batch left outstanding by another consumer is replayed once, then handled. + expect(await check({ ack })).toMatchObject({ + runId: run.id, + deliveryId: runDelivery, + acknowledged: ack, + replayed: true + }) + expect(await check()).toMatchObject(tail) + } expect(state.db.hasOutstandingMailboxDelivery(`dispatch:${dispatch.id}`)).toBe(false) - for (const id of ids.slice(0, -1)) { + for (const id of ids) { expect(state.db.getMessageById(id)).toMatchObject({ read: 1, run_id: dispatch.run_id }) } - expect(state.db.getMessageById(last)).toMatchObject({ read: 0, run_id: dispatch.run_id }) - expect(await check({ ack })).toMatchObject({ deliveryId: runDelivery, replayed: true }) - const tail = await check({ ack: runDelivery }) - expect(tail).toMatchObject({ runId: dispatch.run_id, messages: [{ id: last }], count: 1 }) - expect(await check({ ack: deliveryId(tail) })).toMatchObject({ count: 0 }) - expect(state.db.getMessageById(last)?.read).toBe(1) + expect(await check({ ack: runDelivery })).toMatchObject({ count: 0 }) } ) }) diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/run-bound-mailbox-safety.test.ts b/src/main/runtime/rpc/methods/orchestration/messaging/run-bound-mailbox-safety.test.ts index d665b751ad5..b9d565e4ffb 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/run-bound-mailbox-safety.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/run-bound-mailbox-safety.test.ts @@ -447,7 +447,7 @@ describe('Run-bound lead mailbox boundaries', () => { expect(state.db.getMessageById(privateMail.id)).toMatchObject({ read: 0, run_id: foreign.id }) }) - it('treats types as a wake condition and replays the entire residual FIFO batch', async () => { + it('treats types as a wake condition and returns the entire residual FIFO batch', async () => { residual('older status') residual('decision needed', 'question') runMail() @@ -455,10 +455,6 @@ describe('Run-bound lead mailbox boundaries', () => { expect(first).toMatchObject({ messages: [{ subject: 'older status' }, { subject: 'decision needed' }] }) - expect(await check({ wait: true, types: 'worker_done' })).toMatchObject({ - deliveryId: deliveryId(first), - replayed: true - }) expect(await check({ ack: deliveryId(first) })).toMatchObject({ runId: leadRun.id, acknowledged: deliveryId(first), @@ -484,16 +480,12 @@ describe('Run-bound lead mailbox boundaries', () => { } ) - it('replays an outstanding Run batch before exposing older nonmatching residual mail', async () => { + it('hands older nonmatching residual mail to the check after a filtered Run wait', async () => { residual('status') runMail() const first = await check({ wait: true, types: 'question' }) expect(first).toMatchObject({ runId: leadRun.id }) - expect(await check()).toMatchObject({ deliveryId: deliveryId(first), replayed: true }) - expect(await check({ ack: deliveryId(first) })).toMatchObject({ - acknowledged: deliveryId(first), - messages: [{ subject: 'status' }] - }) + expect(await check()).toMatchObject({ messages: [{ subject: 'status' }] }) }) it('keeps explicit Run checks scoped and refuses the parent Run', async () => { diff --git a/src/main/runtime/rpc/orchestration-current-authority-precedence.test.ts b/src/main/runtime/rpc/orchestration-current-authority-precedence.test.ts index 17664f10b8e..a408ebf02d6 100644 --- a/src/main/runtime/rpc/orchestration-current-authority-precedence.test.ts +++ b/src/main/runtime/rpc/orchestration-current-authority-precedence.test.ts @@ -195,23 +195,22 @@ describe('current orchestration authority precedence', () => { count: 1 } }) - expect(harness.db.getMessageById(message.id)?.read).toBe(0) + expect(harness.db.getMessageById(message.id)?.read).toBe(1) const deliveryId = (response as { result: { deliveryId: string } }).result.deliveryId + // A response lost in transit replays from the request receipt when retried with the same id. const replayed = await harness.dispatcher.dispatch( request( 'orchestration.check', { terminal: CURRENT_WORKER_HANDLE }, currentEvidence('worker'), - 'current-worker-check-replay' + 'current-worker-check' ) ) - expect(replayed).toMatchObject({ ok: true, - result: { deliveryId, replayed: true, messages: [{ id: message.id }], count: 1 } + result: { deliveryId, messages: [{ id: message.id }], count: 1 } }) - expect(harness.db.getMessageById(message.id)?.read).toBe(0) const acknowledged = await harness.dispatcher.dispatch( request( diff --git a/src/main/runtime/rpc/orchestration-session-coordinator.test.ts b/src/main/runtime/rpc/orchestration-session-coordinator.test.ts index 1f5b77205c3..d01f7efc010 100644 --- a/src/main/runtime/rpc/orchestration-session-coordinator.test.ts +++ b/src/main/runtime/rpc/orchestration-session-coordinator.test.ts @@ -311,11 +311,9 @@ describe('a structured chat coordinates through the same verbs as a terminal', ( ) ).toMatchObject({ ok: false, error: { code: 'consumer_fenced' } }) expect(acknowledge).not.toHaveBeenCalled() - expect(h.db.getDeliveryRaw(ack)?.acknowledged_at).toBeNull() expect(await as(SESSION_Y, 'orchestration.check', {})).toMatchObject({ - deliveryId: ack, - replayed: true, - messages: [{ id: idOf(message) }] + deliveryId: null, + count: 0 }) expect(await as(SESSION_Y, 'orchestration.check', { ack })).toMatchObject({ acknowledged: ack, diff --git a/src/main/runtime/runtime-rpc-long-poll-transport.test.ts b/src/main/runtime/runtime-rpc-long-poll-transport.test.ts index 8086b4f2562..6d978f2d54e 100644 --- a/src/main/runtime/runtime-rpc-long-poll-transport.test.ts +++ b/src/main/runtime/runtime-rpc-long-poll-transport.test.ts @@ -551,11 +551,10 @@ describe('OrcaRuntimeRpcServer', () => { db.insertMessage({ from: 'term_worker', to: `run:${run.id}`, subject: 'go', runId: run.id }) runtime.notifyMessageArrived(`run:${run.id}`) await Promise.all([b.done, c.done]) - const consumerGeneration = run.consumer_generation - const deliveryId = db.getOrCreateRunDelivery({ runId: run.id, consumerGeneration }) - ?.delivery.id const results = [b, c].map((s) => s.frames.find((f) => f.ok !== undefined)?.result) - expect(results).toMatchObject([{ deliveryId }, { deliveryId }]) + const batch = { deliveryId: expect.stringMatching(/^delivery_/), count: 1 } + expect(results).toMatchObject([batch, batch]) + expect(db.db.prepare('SELECT COUNT(*) AS n FROM deliveries').get()).toEqual({ n: 1 }) await waitFor(() => server['activeLongPolls'] === 0) } finally { db.close() diff --git a/src/shared/orchestration-check-output.test.ts b/src/shared/orchestration-check-output.test.ts index df3248bd2fd..a88980b9be2 100644 --- a/src/shared/orchestration-check-output.test.ts +++ b/src/shared/orchestration-check-output.test.ts @@ -52,10 +52,24 @@ describe('formatted delivery acknowledgment', () => { messages: [{ id: 'msg_one', from_handle: 'worker' }], count: 1, deliveryId: 'delivery_one', + deliveryAcknowledged: true, formatted: 'Message body' }, 'term_coord' ) ).toBe('Delivery delivery_one\nMessage body') }) + + it('tells the caller to ack when an older runtime did not', () => { + expect( + formatOrchestrationCheckText( + { + messages: [{ id: 'msg_one', from_handle: 'worker', subject: 'done' }], + count: 1, + deliveryId: 'delivery_one' + }, + 'term_coord' + ) + ).toContain('rerun this check with --ack delivery_one.') + }) }) diff --git a/src/shared/orchestration-check-output.ts b/src/shared/orchestration-check-output.ts index 4c9012e5117..cbb587a7d64 100644 --- a/src/shared/orchestration-check-output.ts +++ b/src/shared/orchestration-check-output.ts @@ -37,6 +37,8 @@ export type OrchestrationCheckOutput = { count: number formatted?: string deliveryId?: string | null + // Older runtimes omit it and replay the batch until an explicit --ack. + deliveryAcknowledged?: boolean timedOut?: boolean cancelled?: boolean connectionLost?: boolean @@ -80,7 +82,11 @@ export function formatOrchestrationCheckText( ? '[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]\n' : '[LEGACY COMPATIBILITY]\n' : '' - const deliveryNotice = formatCurrentDeliveryNotice(prepared.legacyCompatibility?.currentDelivery) + const deliveryNotice = + formatCurrentDeliveryNotice(prepared.legacyCompatibility?.currentDelivery) + + (prepared.deliveryId && prepared.deliveryAcknowledged !== true + ? `\nThis Orca version does not acknowledge batches automatically. After handling every message, rerun this check with --ack ${prepared.deliveryId}.` + : '') if (prepared.formatted) { return `${legacyHeader}${prepared.deliveryId ? `Delivery ${prepared.deliveryId}\n` : ''}${prepared.formatted}${deliveryNotice}` }