From bfe476f9221f4401bd4f5841bd06be34e3685113 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 27 Sep 2026 20:38:00 -0700 Subject: [PATCH] fix(native-chat): a message is accepted, then delivered (#22821) * refactor(native-chat): remove the unused terminal handoff No client ever called agentSession.requestHandoff or mounted the handoff chrome. Delete the handoff coordinator, the terminal-owner runtime, the proof write path and the unmounted UI. Keep agentSession.handoffStatus, which released desktop clients read for worktree activation, and let records an older build left mid handoff reconcile through the ordinary restart and recovery paths. * fix(native-chat): never let the pre-stop snapshot hold a chat's stop Eviction now drains delivered events before quit's resume-offer snapshot. An unbounded wait there sits ahead of the provider stop, so a sink whose journal write stalls kept the child running until the step deadline aborted the eviction. The offer is advisory: bound the drain and stop the child regardless. Co-Authored-By: Claude * refactor(native-chat): drop helpers only the terminal handoff called `claudeAuthEnvCarriedForward`, `isPathWithinDirectory` and `queryWindowsProcessRowsFresh` lost their last caller with the handoff. The fresh-scan tests now go through `queryWindowsProcessDescendants({ fresh: true })`, the teardown path that still depends on that contract. Co-Authored-By: Claude * docs(native-chat): stop citing the removed handoff in lifecycle comments Six comments still named the handoff coordinator, a handoff suspend, or a terminal-owned session as live participants in the flows they describe. Co-Authored-By: Claude * test(native-chat): type the stalled snapshot drain without a cast Co-Authored-By: Claude * test(native-chat): pin that a start dead before proving owes no settlement The removed restart handoff test pinned this branch; nothing else did. Co-Authored-By: Claude * fix(native-chat): keep the owner-status read behind an in-flight attach The handoff removal dropped the per-session queue from `handoffStatus`, so a read landing mid-start reported the reservation (no owner) instead of the settled chat owner, and shipped desktop clients blocked worktree activation on it. The read is queued again, as it was before the removal. Co-Authored-By: Claude * refactor(terminal): remove the agent-session PTY write gate The gate only refused a write when a PTY had been bound to a chat session, and the only code that ever bound one was the terminal handoff this branch removes. With it gone, every admit/readmit returned "admitted" unconditionally, so the checks on the renderer write path, the runtime controller backstop, terminal.send, agent prompts, preview input and orchestration pointers, the refusal fields on terminal.send and worker-start receipts, the plugin and CLI refusal copy, and the adopted-pane orchestration routing could no longer run. Ordinary writes take the same path in the same order as before. Co-Authored-By: Claude * refactor(native-chat): drop the transcript helpers only the handoff called appendLegacyTranscriptMessages fed the terminal transcript catch-up and proveClaudeTranscriptBranch backed the terminal owner's exit proof. Both lost their last caller with the handoff. Their tests now go through the live entry points instead: the roster bounds through the legacy import, the pinned-read and growth tests through the ancestry replay the history window uses, and the marker rules through the string proof in their own file rather than the session-file resolver's. Co-Authored-By: Claude * fix(native-chat): stop calling a starting chat "mid-handoff" A send refused because the chat's owner is not settled showed "The session is mid-handoff ()." in the composer. With the handoff gone, the stages that reach it are a chat that is still starting, or one whose previous agent process has not yet been confirmed stopped. The message now says which of the two it is. The refusal code is unchanged. Co-Authored-By: Claude * test(native-chat): type the stand-in roster decoder without a cast Co-Authored-By: Claude * refactor(codex): name the pinned rollout lookup for what it does With the terminal handoff gone, the module named codex-tui-rollout-proof holds only the pinned rollout lookup that structured Codex launches use to resume a thread, so the name described code that no longer exists. Rename the module and its options type. Also drop a mobile allowlist assertion that pinned the removed agentSession.requestHandoff method, which no longer exists to allow. * refactor(native-chat): type the owner-status reply as the host sends it The handoffStatus reply type still listed the terminal handoff's fields and states (terminal placement, host label, proof retry, queued and waiting phases, the to-terminal direction). No host writes them any more and the only client reader parses the reply as unknown, so they described nothing. The reply on the wire is unchanged. * refactor(native-chat): normalize terminal-handoff lease values once at decode Nothing in this build writes a terminal owner (`runtimeKind: 'tui'`) or the handoff's `preparing` / `old-owner-stopped` stages, but the in-memory types still admitted them, so readers across the host kept branches for values no path produces and the compiler could not point at them. The store now validates the on-disk shape, which still accepts those values so an older record is not quarantined, and maps them once while parsing: - `preparing` and `old-owner-stopped` become `recovering` - a `tui` lease becomes `native`; when it records a process it also becomes `conflicted`, the claim every build probes but never stops. A plain native owner would be stopped by restart recovery, here and in older builds. Revisions are taken over the normalized state on both sides of every compare, and the mapped record reaches disk with the store's first transaction, the same way the tab-id backfill does. The in-memory types narrow to what this build writes, and the branches that existed only for the removed values go. Structured-worker identity keeps its verdict for a former terminal owner by refusing a conflicted claim rather than a non-native kind. * refactor(native-chat): stop threading the owner kind through a reservation A reservation only ever names a native owner now, so the request no longer carries a kind and the reserved lease records `native` directly. The attach params keep `runtimeKind`: agentSession.ensure and create accept it, and the operation fingerprint stored in the ledger covers it. * test(native-chat): pin the legacy-lease rewrite with a transaction that changes nothing else Hiding a tab also committed the visibility index, so the no-op transaction wrote the file even when its open-time revision was wrong. Committing the index first leaves the pending rewrite as the only reason to write. * fix(native-chat): name a chat write by its target, not the owner generation A write carried the fence of the last frame the pane read, and the host refused it unless that fence was still current. An idle release and the restart after it each move the fence, and the release publishes nothing, so a send after a release was refused "Expected runtime fence 1; the session is at 3", and a Stop queued behind a cold start was refused as stale. Every write already names what it acts on: a send its conversation, a cancel its turn, a prompt answer its item revision, a rewind its epoch; an option is last-writer-wins. So admission stops comparing the client's fence, and the rebase that papered over one restart (admitAtResumedFence, resumedFromFence) goes with it. The writer-lease check stays, and so does the attach's compare-and-swap. Frames now stamp the fence read when each frame is sent instead of a copy each subscriber kept, which went stale on the same release. * fix(native-chat): every journal append reaches the chats that are open A journal write and its delivery to open readers were two calls, and some writers made only the first. A failed start whose lease could not be handed back, a provider revision with no frame behind it, and eviction's settlement were all journaled without reaching an open chat. A journal handle now reports every durable change, and the host's session map binds that report to the session's readers when the handle is set. Writers no longer publish what they append; the per-writer publish calls are deleted. * test(native-chat): an epoch replacement reaches the open chat * test(native-chat): each row reaches an open chat once, and a live handle enters only through the map * test(native-chat): give the legacy-lease store test a tab id so the backfill cannot supply its rewrite The seeded record had no surface tab id, so the next open backfilled one and that rewrite alone made the no-op transaction write. The test passed with the legacy-lease rewrite signal removed. * test(worktree-activation): restore the OMP surfaced-agent resume test The handoff removal deleted it alongside the terminal-owner tests, but it covers the surfaced-PTY block that still guards resume, including an agent whose ownership is unknown. * perf(native-chat): a publish behind a delivered commit reads nothing Each commit now delivers itself, so the publish a provider frame still sends afterwards found every reader caught up but still read rows and rebuilt the timeline for each one. A caught-up reader now skips the read. * test(native-chat): state why the teardown test's fake journal is safe to cast * docs(native-chat): say mutation admission checks only the writer lease * docs(native-chat): drop the send rebase from comments that still described it * fix(native-chat): a message is accepted, then delivered A send to a chat with no running agent restarted the agent inside the send call, before the message was recorded, so the client waited for the whole start and a failed restart refused the message. Claude held prompts sent during startup, and those could settle as "unconfirmed". A send is now accepted inside the session's serialized queue: one ledger row and one submission row marked handoverRecorded, published, answered pending. A per-session delivery loop exists while a message is queued. It starts the agent through the same serialized attach a hold uses, waits outside the queue for a Claude child to prove its start, and hands the oldest queued message over as its own serialized step, writing dispatch{pending} before the adapter call. A start it needed and did not get writes one error-tone row and rejects every queued message with the same words; a start Stop cancelled writes none. Settlement follows from the rows. A queued message is provably unwritten, so a close, an eviction or an exit rejects it. A handed-over message stays in doubt. A queued row at or below the sequence a handle found when it opened was left by an earlier process and is rejected at open, with no latch. Stop withdraws queued messages with no writer lease and no fence. An attach failure keeps the conversation open, and the attach adopts its journal. Owed work counts the loop and queued rows. A compaction or rewind found prepared when a conversation opens was started under a child this process no longer has, so the open settles it rather than leaving it to refuse every send until a view attaches. The open cursor is scoped to its epoch, because sequences restart when an epoch is replaced. Deleted: restart-before-admission, recordFailedRestart, the fence rebase, Claude's startup gate, the attach's forget on failure and its own crash boundary. Clients without agent-session.accepted-send.v1 get their reply held until the handover; the desktop and paired desktop lists advertise it. * fix(native-chat): settle queued messages only for the child that ended A child that proved its start and then exited before its message was handed over left the message queued: the exit settlement returned early when nothing else was in flight. Delivery then started another child for it, and a child that died the same way started another, without end and without a row. A retried settlement for an earlier generation, run by the attach that delivery started, did the opposite: with that generation's turn unfinished it rejected the message queued for the child being attached. The settlement now takes the rejection for queued messages from its caller. The unexpected exit and the eviction pass one, and it applies even with no other work in flight; the retry for an earlier generation passes none. * fix(native-chat): an adoption that fails to import keeps the conversation open The attach now writes into the conversation's own open journal, but a failed transcript import still closed it as if it were the attach's provisional one. The conversation stayed indexed with a closed journal, so every later send answered "could not be recorded" and every attach failed again until the app restarted. The import now closes only a journal the attach opened for itself. * perf(native-chat): the recovering open reads the journal once Every conversation open now goes through the recovering open, including the read restore of every chat at startup, which used to replay its journal once. The recovering open replayed it twice: once to probe it and again inside the open. The probe is now handed to the open as its load. * fix(native-chat): an attach that fails after indexing its child leaves no child behind A failed attach now keeps the conversation open, but a failure after `onAttached` indexed the child (the rewind or compaction recovery, or the attach's own success record) left that entry claiming a child the failure path had already released. The next send found the phantom, skipped the start, and wrote at a fence the journal had moved past, so the message stayed queued for good. The entry now drops the released child and its event sink, and follows the record's fence, as a failure before indexing already did. * fix(native-chat): a withdrawn message shows no error, and a rejection outlasts the send's answer The error strip for a message the host accepted and then did not deliver matched the entry before the outbox reconciled, so a Stop's withdrawal, which the reconcile drops, showed "Orca could not send your message" with nothing to retry. It now reads the reconciled entry. A rejection the journal records before the send's own pending answer lands is final as well: that answer no longer puts the entry back to dispatching with no Retry. * fix(orchestration): a structured worker whose agent outlasts the preamble wait is left unknown, not torn down The preamble waits for its submission to be delivered while the worker's agent starts. When that wait ran out it threw operation_unknown, and the failed-start teardown then closed the session, which rejected the very preamble the host was about to deliver. It now reports a turn start nobody observed yet: the worker is start-unknown with its session kept, the host delivers the preamble when the agent starts, and the worker's report settles the dispatch as for any unobserved start. The receipt no longer suggests reading a screen a structured worker lacks. * fix(native-chat): a message rejected while its chat was closed reads as not sent A remount reads an entry it left dispatching as unconfirmed. When the journal had rejected it meanwhile, as a failed start or a quit now does, the reconcile left it unconfirmed: it blocked every later message behind a Retry and no reason, and the delivery probe, seeing the journal already answered, never ran. The reconcile now settles it as rejected like a dispatching one. * test(orchestration): name why the readiness settlement fakes are cast * fix(native-chat): keep each pane's own fence on frames so a failed restart is not resent * docs(native-chat): drop the fence from the admission the send effects run behind * docs(native-chat): give the fence move on release the reason that still holds * docs(native-chat): stop citing a write fence check in launch and mailbox comments Three places still gave the removed fence check as a reason: the launch replay said admission puts the ledger ahead of the fence, the launch surface said a send must name the lease it was admitted against, and the direct-mailbox path said the lease fence decides whether delivery is safe. Admission now checks only the writer lease. * refactor(native-chat): the provider child is its own record A conversation now outlives any number of provider children, so the child is one record on the conversation's entry instead of five loose fields beside its journal. It is written in one place: indexed only once an attach has fully succeeded, and ended through one function that an exit, a failed re-attach, a Stop and an eviction all share, matched on the child's generation and fence. - A failed attach writes no child, so there is nothing to unwind: the field unwind and the fence patch after it are gone. - Conversation writes read the record's fence, the way mutation admission already does; a child's own writes use its fence. The four stored-fence patches, and the settlement retry's overwrite of the conversation's fence, are gone. - The owed wind-down is its own tombstone, carrying the child it is owed for, and is no longer dropped when an attach replaced the whole entry. - Stop on a child still proving its start stops only the child: its lease goes back and the chat is told it is idle, but the journal, the holders and the readers stay. Close is that stop plus the conversation's close. - The settlement retry uses the conversation's own journal, opened through the host's one open. * fix(native-chat): the delivery loop alone settles a message its start or child failed A queued message was settled by whichever path happened to end the child first: the loop, the unexpected exit, eviction's work settlement, the open's leftover rule, and the startup branch that rejected every pending row. That gave two failure rows with different tones for one start, a loop that could hand over to a different child than the one it waited on, and a Claude start that died while starting reading unlike every other failed start. - The loop remembers the child it waited on. At handover, if that child is gone or replaced, it reads how it ended: a Stop continues; anything else writes one failure row and rejects every queued message with the same words, then stops. A child still starting whose start the adapter says did not land fails the same way. The exit, eviction and the settlement retry only settle the handed-over and legacy rows of the child that ended. - One failure row, always an error, keyed by the start. A start a view began that dies with nothing queued writes the same row through the same builder, so a second report revises it. - The open no longer rejects leftovers; the loop's first step does, and the open wakes it. - `awaitStarted` answers why a start did not land, so the row says it even when the loop sees the failure before the exit is processed. - Quit closes every conversation the way closing a chat does: what is still queued is rejected as closed, with or without a child, and a start the loop already has in flight is waited for so the child it produces is stopped rather than left behind. * refactor(native-chat): a stopped child ends on the one reading of its stop The eviction step reads a stop's result through `stopAgentSessionProviderRoot` and hands that verdict to the child's ending, so the host never forms a second view of whether the root is gone. Every ending carries it: a stop's comes from that reading, an exit's root is gone by definition, and a failed re-attach passes what its release saw. The end-of-child record can therefore also carry a stop whose root was not seen to go, which nothing ends on yet. * feat(native-chat): the host says it accepts a send before any agent has it The host now lists agent-session.accepted-send.v1 among its own runtime capabilities, the same string capable clients already send. A client can then tell a host that answers a send at acceptance, and admits a Stop with no writer before a turn starts, from an older one that still restarts the agent inside the send. Additive: an older client ignores a capability it does not know. * refactor(native-chat): an attach never opens a journal of its own The attach adopts the conversation's open journal, which outlives it, so it no longer opens one for a direct caller either. That leaves nothing for a failed adopted import to close, and the flag that told the two cases apart is gone. Tests that attach without a host open the conversation the way a host does. * fix(native-chat): a moved fence resends nothing on a host that accepts first The outbox treated any fence change as a new owner: it dropped the answer of a send in flight, queued that send to go out again under the same id, and unblocked a refused head. On an older host that is how a send the restart refused, unrecorded, gets another try. On a host that records every send before it starts an agent, a fence moves because that start ran, so the same rule resent into every failed start. With a fence stamped on every frame, that became a loop. The outbox now reacts to a fence change only when the host has not advertised that it accepts a send before any agent has it. On such a host, only a Retry or a new send goes out, and a failed start reaches the client as a rejected message it keeps with its Retry. Against an older host, or before one has answered, the outbox behaves as it did. Desktop and paired web share this hook. * refactor(native-chat): a child's end says whether the user or the host stopped it The end-of-child record's cause now tells a user's Stop from the host stopping the child for a cause of its own: `user-stop` and `host-stop` replace `stop`. The delivery loop goes on after a user's Stop, as before, and fails the start it was waiting on after a host stop, with the one error row and every queued message rejected, in the stop's reason when it gave one. The reason stays description only. Stop passes `user-stop`; nothing passes `host-stop` yet. * fix(native-chat): a chat whose only work is a queued message is not offered for resume A message accepted while the agent was starting counts as working in the chat, and quit rejects it as never sent. The teardown snapshot read the same working rule, so a relaunch offered to resume a chat whose agent never had the message. The snapshot now reads only what was handed over. * test(native-chat): type the queued-message fixtures in the resume-offer tests * fix(native-chat): a start that dies while a message waits on it is that message's failed start Opening a chat's tab starts an agent for the view, and a send accepted meanwhile waits on it. When that start died, its exit wrote the start's error row and left the message queued, so the delivery loop started a second agent into the same failure and wrote a second row. A child's end now records where the conversation's journal stood, and the loop settles a message accepted before a failed start ended with that start: one row, under its key, and no second start. A message sent after the failure still gets a fresh start. * docs(native-chat): say what an attach's open conversation and unconfirmed ids are now * test(native-chat): pin what a failed start settles, and what a resume offer names A view's child that dies while a sent message waits settles that message only when it died starting and no child has taken its place: a proven child's crash, or a second start since, gets the message delivered. The resume offer names the handed-over message, never a newer one still queued. * test(native-chat): the failed-start pins fail on what the message became, not on a timeout * test(orchestration): the preamble's host stub is typed, not cast The preamble send now takes only what it reads of the host, the send, the settlement wait and the record's fence, so its test builds that host with real types instead of `as never`. * fix(native-chat): a view never restarts a chat whose last start failed A Claude chat whose CLI exits during startup left one red row per start, and every time a view bound to it (the chat opening right after its create died, or the user switching back to it) the hold started the CLI again, so the same launch-failure row repeated. Only a send retries a failed start now, the same rule provider-exit recovery already applied; the rule lives in one predicate the hold, exit recovery and the delivery loop share. * test(native-chat): start the child the loop waits on with an attach, not a second view A view no longer starts a child whose last start failed, so the R2 case that waits on a child started since the failure now gets that child from a client attach, the one non-send starter left. * fix(native-chat): settle a gone generation's turn wherever a conversation opens A send that opens a chat this process had not read yet (after a crash, from a phone or the CLI) went through the delivery open, which never settled what the dead generation left running; only the read restore and a successful acquire did. When the send's start then failed, the turn stayed running for every reader. The settlement now runs in the one journal open, at the crash boundary, for every opener except an acquisition, which settles from the evidence it read before its reserve; the read restore's separate step is gone. * test(native-chat): prove the next child's start settles the turn an earlier child left The R1 case lost its only settlement assertion when the latch it checked was deleted. It now seeds the running turn the earlier child left and asserts it ends at the exit's receipt, with the exit's row, before the message is handed to the new child. * test(native-chat): count a failed start's rows by row, not by text Comparing the set of texts passed when two different rows carried the same words, which is the duplicate the test exists to catch. * test(native-chat): give the failed-start and stale-turn waits a loaded runner's budget * test(native-chat): pin the open's and the send's start and row counts, however the view binds Opening a fresh chat whose starts fail makes one start and one row, with two views bound before or after the create's child died; one send makes one more of each. * fix(native-chat): settle a gone generation's turn at every open but an acquisition's The journal open skipped the settlement whenever the lease read reserved or live, to leave an acquisition's own open to the acquisition. But a lease a crashed process left in recovery also reads live, until the next acquire resolves it. A send that opened such a chat, from a phone or the CLI after a crash on a host that could not prove the old owner gone, skipped the settlement; when its start then failed, the dead turn stayed running for every reader. The acquisition now says it is the opener, and every other open settles, whatever the lease still claims. * test(native-chat): hold the create's start open until the views bind The "view binds while the create is still starting" case gave the create a 300 ms head start and asserted the views bound before it died. On a loaded runner the holds took longer, the create's exit landed first, and the case failed its own precondition. The create's initialize now waits on a gate the test releases once the views are bound. * refactor(native-chat): drop the composer's second error formatter After the merge with main, every chat write in the composer path reports its failure as a typed outcome worded by the refusal-notice table, so the send's catch sees only a local throw. The {code, message} formatter this branch added for it has no payload left to format, and its claim to be the one way a chat words a failure is no longer true. The composer send is main's again. * test(native-chat): pin the reason on a message rejected while its chat was closed The reopen test checked only that the message reads as not sent; it now also checks the Retry row carries the host's reason. --------- Co-authored-by: Claude --- ...claude-structured-dispatch-test-support.ts | 4 +- src/main/claude/claude-structured-dispatch.ts | 19 +- .../claude/claude-structured-options.test.ts | 4 +- .../claude-structured-prompt-ownership.ts | 18 +- .../claude/claude-structured-real-cli.test.ts | 2 +- .../claude-structured-session-acquisition.ts | 50 +- .../claude-structured-session-adapter.ts | 22 +- ...laude-structured-session-exit-lifecycle.ts | 4 +- .../claude-structured-session-publication.ts | 4 +- ...claude-structured-session-recovery.test.ts | 2 +- .../claude-structured-session-startup-gate.ts | 172 ----- ...claude-structured-session-startup-state.ts | 58 ++ .../claude-structured-session-startup.test.ts | 136 +--- .../claude-structured-session-startup.ts | 17 +- .../claude/claude-structured-session-state.ts | 4 +- .../claude-structured-session-test-support.ts | 2 +- .../desktop-renderer-runtime-capabilities.ts | 2 + .../journal-pending-submission-recovery.ts | 40 +- .../agent-session-journal/journal-reducer.ts | 6 +- .../journal-restart-reconciliation.ts | 7 +- .../journal-row-builders.ts | 5 +- .../journal-row-schema.ts | 3 + .../journal-store-contracts.ts | 2 + .../journal-store.test.ts | 28 + .../agent-session-journal/journal-store.ts | 24 +- .../agent-session-journal-recovery.test.ts | 25 +- .../agent-session-journal-recovery.ts | 4 +- ...-agent-session-accept-then-deliver.test.ts | 671 +++++++++++++++++ ...-agent-session-acquisition-options.test.ts | 11 + ...structured-agent-session-adapter-router.ts | 2 + .../structured-agent-session-adapter.ts | 4 + ...tured-agent-session-adopted-import.test.ts | 51 +- ...structured-agent-session-adopted-import.ts | 10 +- ...ured-agent-session-append-delivery.test.ts | 20 +- ...structured-agent-session-attach-context.ts | 9 +- ...structured-agent-session-attach-failure.ts | 3 +- .../structured-agent-session-attach-flow.ts | 9 +- ...ured-agent-session-attach-orchestration.ts | 169 +++-- ...gent-session-attach-reconciliation.test.ts | 35 + ...-agent-session-attach-test-conversation.ts | 18 + .../structured-agent-session-attach.ts | 71 +- ...d-agent-session-background-task-channel.ts | 9 +- ...red-agent-session-claude-root-exit.test.ts | 9 +- ...ructured-agent-session-close-retry.test.ts | 73 +- ...uctured-agent-session-conversation-open.ts | 159 ++++ ...ctured-agent-session-conversations.test.ts | 5 +- ...session-dead-generation-settlement.test.ts | 6 +- ...gent-session-dead-generation-settlement.ts | 43 +- .../structured-agent-session-delivery-loop.ts | 248 +++++++ ...structured-agent-session-event-recovery.ts | 7 +- .../structured-agent-session-eviction.test.ts | 34 +- .../structured-agent-session-eviction.ts | 33 +- ...ctured-agent-session-forget-status.test.ts | 22 +- ...red-agent-session-hold-resume-race.test.ts | 3 + .../structured-agent-session-holds.test.ts | 11 +- .../structured-agent-session-holds.ts | 10 +- .../structured-agent-session-host-delivery.ts | 101 +++ .../structured-agent-session-host-lifetime.ts | 174 +++-- ...structured-agent-session-host-mutations.ts | 83 ++- .../structured-agent-session-host-teardown.ts | 14 +- .../structured-agent-session-host-types.ts | 64 +- .../structured-agent-session-host.test.ts | 16 +- .../structured-agent-session-host.ts | 68 +- ...ured-agent-session-journal-handles.test.ts | 71 +- ...ured-agent-session-late-settlement.test.ts | 33 +- ...t-session-launch-send-after-create.test.ts | 18 +- ...gent-session-legacy-handoff-record.test.ts | 57 +- ...ctured-agent-session-mutation-admission.ts | 6 +- ...structured-agent-session-mutation-plans.ts | 16 +- ...agent-session-operation-settlement.test.ts | 40 +- ...ured-agent-session-operation-settlement.ts | 7 +- ...ed-agent-session-option-settlement.test.ts | 5 + ...ed-agent-session-owed-work-release.test.ts | 40 +- ...nt-session-processless-reservation.test.ts | 6 + ...gent-session-provider-child-record.test.ts | 691 ++++++++++++++++++ ...structured-agent-session-provider-child.ts | 78 ++ ...red-agent-session-provider-started.test.ts | 8 +- ...ructured-agent-session-provider-started.ts | 10 +- ...uctured-agent-session-read-restore.test.ts | 21 +- .../structured-agent-session-read-restore.ts | 108 +-- ...ed-agent-session-readable-restorer.test.ts | 10 +- ...uctured-agent-session-readable-restorer.ts | 33 +- ...ctured-agent-session-refusal-retry.test.ts | 34 +- ...-session-restart-continuation-wait.test.ts | 63 ++ ...ured-agent-session-restart-continuation.ts | 13 +- ...ent-session-restart-failure-filing.test.ts | 23 +- ...ssion-restart-interruption-test-harness.ts | 15 +- ...ed-agent-session-restart-ownership.test.ts | 13 +- ...ured-agent-session-restart-restore.test.ts | 68 +- ...tructured-agent-session-restart-restore.ts | 43 +- ...tured-agent-session-restart-resume-host.ts | 7 +- ...ent-session-restart-resume-test-harness.ts | 2 +- ...red-agent-session-restart-resume-wiring.ts | 10 +- ...tured-agent-session-restart-resume.test.ts | 93 ++- ...ctured-agent-session-resume-eligibility.ts | 2 +- .../structured-agent-session-reveal.test.ts | 24 +- .../structured-agent-session-reveal.ts | 24 +- .../structured-agent-session-rewind.test.ts | 8 + ...red-agent-session-send-idempotency.test.ts | 9 +- ...agent-session-send-open-stale-turn.test.ts | 137 ++++ ...red-agent-session-send-preparation.test.ts | 304 ++++---- ...ructured-agent-session-send-preparation.ts | 227 ++---- ...session-send-restarts-failed-start.test.ts | 29 +- ...ured-agent-session-send-settlement.test.ts | 2 +- ...tructured-agent-session-send-settlement.ts | 42 +- .../structured-agent-session-send.test.ts | 127 ++-- ...agent-session-settled-attach-retry.test.ts | 4 +- ...uctured-agent-session-start-failure-row.ts | 59 ++ ...agent-session-startup-failure-exit.test.ts | 18 +- ...-agent-session-status-feed-test-session.ts | 10 +- ...ructured-agent-session-status-feed.test.ts | 26 +- .../structured-agent-session-status-feed.ts | 18 +- ...red-agent-session-subagent-recency.test.ts | 4 +- ...red-agent-session-surface-lifetime.test.ts | 76 +- .../structured-agent-session-turns.ts | 151 ++-- ...ession-unanswered-dispatch-release.test.ts | 10 +- ...ured-agent-session-unexpected-exit.test.ts | 49 +- ...tructured-agent-session-unexpected-exit.ts | 93 ++- ...sion-view-start-after-failed-start.test.ts | 207 ++++++ ...t-session-wedged-profile-migration.test.ts | 8 +- ...tured-agent-session-working-at-teardown.ts | 18 +- .../structured-compaction-recovery.ts | 21 +- ...ructured-conversation-command-admission.ts | 9 +- .../agent-session-operation-admission.ts | 4 +- ...e-structured-send-held-for-startup.test.ts | 27 +- ...-send-restart-dies-before-dispatch.test.ts | 143 ++-- .../structured-mailbox-pointer-host.test.ts | 24 +- .../structured-mailbox-pointer-host.ts | 16 +- ...stration-structured-worker-session.test.ts | 78 +- ...orchestration-structured-worker-session.ts | 32 +- .../orchestration-worker-mode-opacity.test.ts | 1 + ...ration-worker-start-mode-selection.test.ts | 2 +- .../deliver-worker-dispatch-preamble.ts | 40 +- .../worker-start-readiness-settlement.test.ts | 90 +++ .../worker-start-readiness-settlement.ts | 17 +- .../worker/worker-start-turn-observation.ts | 2 + ...d-agent-session-send-compatibility.test.ts | 84 +++ ...ctured-agent-session-send-compatibility.ts | 23 +- .../methods/structured-agent-session.test.ts | 44 +- ...ree-structured-agent-rows-liveness.test.ts | 2 +- ...d-agent-session-integration-replay.test.ts | 5 +- ...ructured-agent-session-integration.test.ts | 23 +- ...uctured-agent-session-runtime-exit.test.ts | 3 +- .../native-chat/NativeChatDeliveryRetry.tsx | 8 +- ...tructured-agent-session-outbox-dispatch.ts | 12 + ...ctured-agent-session-outbox-fence.test.tsx | 139 ++++ ...nt-session-outbox-rejection-cause.test.tsx | 267 ++++++- ...e-structured-agent-session-outbox.test.tsx | 10 +- .../use-structured-agent-session-outbox.ts | 35 +- ...-session-accepted-send-capability.test.tsx | 51 ++ ...-agent-session-accepted-send-capability.ts | 87 +++ src/shared/agent-session-journal-schemas.ts | 4 +- src/shared/agent-session-journal-types.ts | 7 + src/shared/agent-session-mutation-envelope.ts | 6 + src/shared/agent-session-queued-submission.ts | 13 + src/shared/agent-session-wire.ts | 2 +- src/shared/protocol-version.ts | 9 + ...ctured-agent-session-dispatch-rejection.ts | 10 +- src/shared/structured-agent-session-outbox.ts | 33 +- ...ructured-agent-session-send-disposition.ts | 24 +- ...tured-agent-session-unanswered-dispatch.ts | 5 + ...x-child-approval-activity-row.unit.test.ts | 4 +- ...ss-version-agent-session-wire.unit.test.ts | 103 ++- 163 files changed, 5792 insertions(+), 1994 deletions(-) delete mode 100644 src/main/claude/claude-structured-session-startup-gate.ts create mode 100644 src/main/claude/claude-structured-session-startup-state.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-attach-test-conversation.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation-wait.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-view-start-after-failed-start.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.test.ts create mode 100644 src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts create mode 100644 src/renderer/src/components/native-chat/use-structured-agent-session-outbox-fence.test.tsx create mode 100644 src/renderer/src/runtime/structured-agent-session-accepted-send-capability.test.tsx create mode 100644 src/renderer/src/runtime/structured-agent-session-accepted-send-capability.ts create mode 100644 src/shared/agent-session-queued-submission.ts diff --git a/src/main/claude/claude-structured-dispatch-test-support.ts b/src/main/claude/claude-structured-dispatch-test-support.ts index a309bd9d0b9..5af0f17243b 100644 --- a/src/main/claude/claude-structured-dispatch-test-support.ts +++ b/src/main/claude/claude-structured-dispatch-test-support.ts @@ -5,7 +5,7 @@ import type { ClaudeSession } from './claude-structured-session-state' import { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker' import { ClaudeChildWorkDecoder } from './claude-child-work-decoder' import { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog' -import { createClaudeSessionStartupGate } from './claude-structured-session-startup-gate' +import { createClaudeSessionStartup } from './claude-structured-session-startup-state' export function sessionFor(send: Mock = vi.fn().mockResolvedValue(undefined)): ClaudeSession { return { @@ -32,7 +32,7 @@ export function sessionFor(send: Mock = vi.fn().mockResolvedValue(undefined)): C capabilities: [], events: undefined, translator: null, - startup: { ...createClaudeSessionStartupGate(), state: 'proven' } + startup: { ...createClaudeSessionStartup(), state: 'proven' } } } diff --git a/src/main/claude/claude-structured-dispatch.ts b/src/main/claude/claude-structured-dispatch.ts index d93a8ecab0a..5a5b2796595 100644 --- a/src/main/claude/claude-structured-dispatch.ts +++ b/src/main/claude/claude-structured-dispatch.ts @@ -35,10 +35,8 @@ import { import { AgentSessionPreDispatchError } from '../native-chat/agent-session-wire/structured-agent-session-operation-settlement' import { claudeStartupFailureReason, - claudeStartupHoldsWrites, - failClaudeStartupGate, - holdClaudeStartupWrite -} from './claude-structured-session-startup-gate' + failClaudeStartup +} from './claude-structured-session-startup-state' const MAX_ACTIVE_DISPATCH_WAITERS = 64 @@ -226,7 +224,7 @@ export function settleCancelledClaudeDispatchWaiters( * Retired rather than dropped: their identities stay joinable, bounded by * `MAX_RETIRED_DISPATCH_WAITERS`. */ export function retireClaudeDispatchWaiters(session: ClaudeSession): void { - failClaudeStartupGate(session, new Error('claude stream-json ended before startup completed')) + failClaudeStartup(session, new Error('claude stream-json ended before startup completed')) for (const waiter of session.dispatchWaiters.splice(0)) { retireWaiter(session, waiter) waiter.resolve(null) @@ -236,8 +234,7 @@ export function retireClaudeDispatchWaiters(session: ClaudeSession): void { export async function dispatchClaudeTurn( session: ClaudeSession, input: { clientMessageId?: string; body: AgentJournalMessageItem; requestedAt?: number }, - beforeDispatch?: () => Promise, - onSettledLate?: ClaudeLateDispatchSettlement + beforeDispatch?: () => Promise ): Promise { let content: unknown[] try { @@ -276,14 +273,6 @@ export async function dispatchClaudeTurn( parent_tool_use_id: null, session_id: session.providerSessionId } - if (claudeStartupHoldsWrites(session)) { - return holdClaudeStartupWrite(session, { - message, - arm, - ...(beforeDispatch ? { beforeDispatch } : {}), - ...(onSettledLate ? { settleLate: onSettledLate } : {}) - }) - } const pending = { replay: beforeDispatch ? undefined : arm() } const authorize = beforeDispatch ? async () => { diff --git a/src/main/claude/claude-structured-options.test.ts b/src/main/claude/claude-structured-options.test.ts index e02647f3625..a92149a2cdd 100644 --- a/src/main/claude/claude-structured-options.test.ts +++ b/src/main/claude/claude-structured-options.test.ts @@ -11,7 +11,7 @@ import { import { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker' import { ClaudeChildWorkDecoder } from './claude-child-work-decoder' import { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog' -import { createClaudeSessionStartupGate } from './claude-structured-session-startup-gate' +import { createClaudeSessionStartup } from './claude-structured-session-startup-state' import { claudeStructuredSessionOptionsFrom, observeClaudeFastModeFacts, @@ -48,7 +48,7 @@ function sessionFor(setModel: ClaudeSession['connection']['setModel']): ClaudeSe capabilities: [], events: undefined, translator: null, - startup: { ...createClaudeSessionStartupGate(), state: 'proven' } + startup: { ...createClaudeSessionStartup(), state: 'proven' } } } diff --git a/src/main/claude/claude-structured-prompt-ownership.ts b/src/main/claude/claude-structured-prompt-ownership.ts index 885ea2dfa74..864a99a0b77 100644 --- a/src/main/claude/claude-structured-prompt-ownership.ts +++ b/src/main/claude/claude-structured-prompt-ownership.ts @@ -15,11 +15,6 @@ import { buildClaudePromptReply } from './claude-structured-prompt-replies' import type { ClaudeSession } from './claude-structured-session-state' import type { ClaudePendingPrompt } from './claude-prompt-registry' import type { PermissionResult } from '@anthropic-ai/claude-agent-sdk' -import { - claudeStartupHoldsWrites, - rejectClaudeStartupWrites -} from './claude-structured-session-startup-gate' -import { DISPATCH_REJECTED_CANCELLED } from '../../shared/structured-agent-session-dispatch-rejection' /** Conservative user-facing window: below the 30s control deadline, trading * residual slow-pump risk for ensuring delivery bookkeeping cannot block Stop indefinitely. */ @@ -107,14 +102,9 @@ export async function cancelClaudeStructuredTurn(input: { const session = requireSession(sessions, request.sessionId) const acquisitionGeneration = session.acquisitionGeneration const prompt = request.prompt - // A held prompt was never written, so Stop withdraws it; the drain only writes what it still - // holds. Before startup lands nothing was written, so there is nothing to interrupt either. - let withdrewHeld = false - if (!prompt && claudeStartupHoldsWrites(session) && session.fence === request.fence) { - withdrewHeld = rejectClaudeStartupWrites(session, DISPATCH_REJECTED_CANCELLED) - if (session.startup.state === 'pending') { - return { cancelled: withdrewHeld } - } + // Before startup lands nothing was written, so there is nothing to interrupt. + if (!prompt && session.startup.state === 'pending') { + return { cancelled: false } } if (prompt && session.fence !== request.fence) { return { cancelled: false } @@ -197,7 +187,7 @@ export async function cancelClaudeStructuredTurn(input: { } else if (claim) { session.prompts.releaseClaim(claim) } - return withdrewHeld ? { ...result, cancelled: true } : result + return result } catch (error) { if (claim && !interruptConfirmed) { session.prompts.releaseClaim(claim) diff --git a/src/main/claude/claude-structured-real-cli.test.ts b/src/main/claude/claude-structured-real-cli.test.ts index 4181dab43c7..c117e3e863f 100644 --- a/src/main/claude/claude-structured-real-cli.test.ts +++ b/src/main/claude/claude-structured-real-cli.test.ts @@ -70,7 +70,7 @@ function realAdapter( const acquire = adapter.acquire adapter.acquire = async (input) => { const acquisition = await acquire(input) - await adapter.drainStartup(input.identity.sessionId) + await adapter.awaitStarted(input.identity.sessionId) return acquisition } return adapter diff --git a/src/main/claude/claude-structured-session-acquisition.ts b/src/main/claude/claude-structured-session-acquisition.ts index b2efed691ee..6728f960ef0 100644 --- a/src/main/claude/claude-structured-session-acquisition.ts +++ b/src/main/claude/claude-structured-session-acquisition.ts @@ -248,30 +248,34 @@ export async function acquireClaudeSession({ event() } }) - session.startup.settled = settleClaudeSessionStartup({ - session, - facts: readClaudeStartupFacts({ - connection, - initProof, - sessionId, - providerSessionId: launch.providerSessionId, - resumesTranscript: launch.resumesTranscript, - inputOptions: input.options, - requestTimeoutMs: deps.requestTimeoutMs, - emit - }), - isCurrent: () => sessions.get(sessionId) === session, - requestTimeoutMs: deps.requestTimeoutMs, - fault: (error) => callbacks.handleExit(sessionId, attempt, error), - onStarted: (options) => - emit({ - type: 'started', + // Whichever comes first: the start landing or faulting, or the child being ended. + session.startup.settled = Promise.race([ + session.startup.settled, + settleClaudeSessionStartup({ + session, + facts: readClaudeStartupFacts({ + connection, + initProof, sessionId, - fence: input.fence, - acquisitionGeneration: session.acquisitionGeneration, - ...options - }) - }) + providerSessionId: launch.providerSessionId, + resumesTranscript: launch.resumesTranscript, + inputOptions: input.options, + requestTimeoutMs: deps.requestTimeoutMs, + emit + }), + isCurrent: () => sessions.get(sessionId) === session, + requestTimeoutMs: deps.requestTimeoutMs, + fault: (error) => callbacks.handleExit(sessionId, attempt, error), + onStarted: (options) => + emit({ + type: 'started', + sessionId, + fence: input.fence, + acquisitionGeneration: session.acquisitionGeneration, + ...options + }) + }) + ]) // A child whose exit already reached `handleExit` is not handed over as live: the create // fails with the CLI's own diagnostic, as one that died before publish does. if (sessions.get(sessionId) !== session) { diff --git a/src/main/claude/claude-structured-session-adapter.ts b/src/main/claude/claude-structured-session-adapter.ts index b8cc5c1dec5..5234606a1cc 100644 --- a/src/main/claude/claude-structured-session-adapter.ts +++ b/src/main/claude/claude-structured-session-adapter.ts @@ -12,7 +12,10 @@ import { acquireClaudeSession } from './claude-structured-session-acquisition' import { supportsClaudeStructuredLocation } from './claude-structured-location-support' import { setClaudeStructuredSessionOption } from './claude-structured-options' import { readClaudeStructuredSessionOptions } from './claude-structured-session-options' -import { claudeStartupSettledWithin } from './claude-structured-session-startup-gate' +import { + claudeStartupFailureReason, + claudeStartupSettledWithin +} from './claude-structured-session-startup-state' import { CLAUDE_DEFAULT_REQUEST_TIMEOUT_MS } from './claude-agent-sdk-control-requests' import { ClaudeAcquisitionRegistry, @@ -115,9 +118,16 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda * apart without guessing at wall-clock. */ drainObservedExits = (): Promise => drainClaudeObservedExits(this.exits) - /** Resolves once a published session's startup has landed or faulted it. */ - drainStartup = (sessionId: string): Promise => - this.sessions.get(sessionId)?.startup.settled ?? Promise.resolve() + /** Resolves once a published session's startup has landed, faulted, or been ended by a close; + * with the reason when it did not land. */ + awaitStarted = async (sessionId: string): Promise => { + const session = this.sessions.get(sessionId) + if (!session) { + return + } + await session.startup.settled + return claudeStartupFailureReason(session) ?? undefined + } /** Restart reconciliation reads the transcript a resume replays; these maps track liveness. */ providerHistoryWindow: NonNullable = ( @@ -179,9 +189,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda } dispatch: StructuredAgentSessionAdapter['dispatch'] = (input) => - dispatchClaudeTurn(this.session(input.sessionId), input, input.beforeDispatch, (settlement) => - this.deps.onDispatchSettledLate?.({ sessionId: input.sessionId, ...settlement }) - ) + dispatchClaudeTurn(this.session(input.sessionId), input, input.beforeDispatch) compact: NonNullable = (input) => compactClaudeSession(this.session(input.sessionId), this.compactions, input) diff --git a/src/main/claude/claude-structured-session-exit-lifecycle.ts b/src/main/claude/claude-structured-session-exit-lifecycle.ts index 4f39a8dc467..d0bddaf0a11 100644 --- a/src/main/claude/claude-structured-session-exit-lifecycle.ts +++ b/src/main/claude/claude-structured-session-exit-lifecycle.ts @@ -3,7 +3,7 @@ import { claudeRootExitObserved, settleClaudeExitedSession } from './claude-structured-session-close' -import { failClaudeStartupGate } from './claude-structured-session-startup-gate' +import { failClaudeStartup } from './claude-structured-session-startup-state' import type { ClaudeAcquisitionAttempt, ClaudeSession, @@ -32,7 +32,7 @@ export function observeClaudeSessionExit( return } lifecycle.sessions.delete(sessionId) - failClaudeStartupGate(session, error) + failClaudeStartup(session, error) // Re-enter the provider's close ladder before publishing lifecycle recovery. // An exit callback is root evidence only; the retained tree proof must run // before the host releases and reacquires this exact child. diff --git a/src/main/claude/claude-structured-session-publication.ts b/src/main/claude/claude-structured-session-publication.ts index d64273f9b8b..f20bc0c7907 100644 --- a/src/main/claude/claude-structured-session-publication.ts +++ b/src/main/claude/claude-structured-session-publication.ts @@ -6,7 +6,7 @@ import type { ClaudeSession } from './claude-structured-session-state' import { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker' import { ClaudeChildWorkDecoder } from './claude-child-work-decoder' import { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog' -import { createClaudeSessionStartupGate } from './claude-structured-session-startup-gate' +import { createClaudeSessionStartup } from './claude-structured-session-startup-state' /** The session as published at spawn: nothing the CLI reports at init is assumed yet. */ export function createClaudeSessionPublication(input: { @@ -67,7 +67,7 @@ export function createClaudeSessionPublication(input: { translator: input.translator, events: input.events, ...(input.unbindReadingControl ? { unbindReadingControl: input.unbindReadingControl } : {}), - startup: createClaudeSessionStartupGate() + startup: createClaudeSessionStartup() } } } diff --git a/src/main/claude/claude-structured-session-recovery.test.ts b/src/main/claude/claude-structured-session-recovery.test.ts index ef8e0c65469..8ddc29a2e67 100644 --- a/src/main/claude/claude-structured-session-recovery.test.ts +++ b/src/main/claude/claude-structured-session-recovery.test.ts @@ -338,7 +338,7 @@ describe('ClaudeStructuredSessionAdapter close and exit recovery', () => { spawnToken: 'spawn-9', events: journalSink }) - await adapter.drainStartup('session-1') + await adapter.awaitStarted('session-1') const first = claude.connections[0] const oldPrompt = invokeCanUseTool(first, 'Bash', 'permission-retained', 'tool-retained') const oldSession = ( diff --git a/src/main/claude/claude-structured-session-startup-gate.ts b/src/main/claude/claude-structured-session-startup-gate.ts deleted file mode 100644 index 0df78936191..00000000000 --- a/src/main/claude/claude-structured-session-startup-gate.ts +++ /dev/null @@ -1,172 +0,0 @@ -// A Claude session is published once its child is spawned, before the CLI has answered -// initialize. Prompts sent in that window are held here and written, in order, once startup -// lands (init facts read and saved options restored), so a first turn never runs under -// defaults the restore was about to replace. A held prompt was never written, so a startup -// that fails rejects it rather than leaving its delivery in doubt. - -import type { AgentSessionDispatchOutcome } from '../native-chat/agent-session-wire/structured-agent-session-adapter' -import { AgentSessionPreDispatchError } from '../native-chat/agent-session-wire/structured-agent-session-operation-settlement' -import { dispatchWriteFailureReason } from '../../shared/structured-agent-session-dispatch-rejection' -import { providerStartupFailureRejection } from '../native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement' -import { claudeUserMessageWasProvablyUnwritten } from './claude-agent-sdk-user-message-queue' -import { - forgetRetiredWaiter, - forgetWaiter, - retireWaiter -} from './claude-structured-dispatch-waiters' -import type { - ClaudeDispatchWaiter, - ClaudeLateDispatchOutcome, - ClaudeSession -} from './claude-structured-session-state' - -type ClaudeStartupHeldWrite = { - waiter: ClaudeDispatchWaiter - message: Record - settleLate?: (outcome: ClaudeLateDispatchOutcome) => void -} - -export type ClaudeSessionStartupGate = { - state: 'pending' | 'proven' | 'failed' - held: ClaudeStartupHeldWrite[] - /** Held prompts are still being written; later prompts must queue behind them. */ - draining: boolean - failure: Error | null - /** Resolves once startup has landed or faulted the session; never rejects. */ - settled: Promise -} - -export function createClaudeSessionStartupGate(): ClaudeSessionStartupGate { - return { state: 'pending', held: [], draining: false, failure: null, settled: Promise.resolve() } -} - -export function claudeStartupFailureReason(session: ClaudeSession): string | null { - return session.startup.state === 'failed' - ? providerStartupFailureRejection(session.startup.failure ?? undefined) - : null -} - -/** Resolves when startup lands or `timeoutMs` passes; a stuck start then refuses the write as before. */ -export function claudeStartupSettledWithin( - session: ClaudeSession | undefined, - timeoutMs: number -): Promise { - if (session?.startup.state !== 'pending') { - return Promise.resolve() - } - let timer: ReturnType | undefined - return Promise.race([ - session.startup.settled, - new Promise((resolve) => { - timer = setTimeout(resolve, timeoutMs) - }) - ]).finally(() => clearTimeout(timer)) -} - -export function claudeStartupHoldsWrites(session: ClaudeSession): boolean { - return session.startup.state === 'pending' || session.startup.draining -} - -/** Admits a prompt while startup is pending; it is written when `openClaudeStartupGate` runs. */ -export async function holdClaudeStartupWrite( - session: ClaudeSession, - input: { - message: Record - arm: () => { waiter: ClaudeDispatchWaiter } - beforeDispatch?: () => Promise - settleLate?: (outcome: ClaudeLateDispatchOutcome) => void - } -): Promise { - if (input.beforeDispatch) { - try { - await input.beforeDispatch() - } catch (error) { - if (error instanceof AgentSessionPreDispatchError) { - throw error - } - return { state: 'rejected', reason: dispatchWriteFailureReason(error) } - } - } - // Startup may have failed while the admission barrier ran. - const failed = claudeStartupFailureReason(session) - if (failed) { - return { state: 'rejected', reason: failed } - } - const { waiter } = input.arm() - session.startup.held.push({ - waiter, - message: input.message, - ...(input.settleLate ? { settleLate: input.settleLate } : {}) - }) - // Startup finished writing what it held while the barrier ran; nothing else would drain this. - if (!claudeStartupHoldsWrites(session)) { - void drainClaudeStartupWrites(session) - } - return { state: 'admitted' } -} - -export async function openClaudeStartupGate(session: ClaudeSession): Promise { - const gate = session.startup - if (gate.state !== 'pending') { - return - } - gate.state = 'proven' - await drainClaudeStartupWrites(session) -} - -async function drainClaudeStartupWrites(session: ClaudeSession): Promise { - const gate = session.startup - gate.draining = true - try { - for (let held = gate.held.shift(); held; held = gate.held.shift()) { - await writeHeld(session, held) - } - } finally { - gate.draining = false - } -} - -async function writeHeld(session: ClaudeSession, held: ClaudeStartupHeldWrite): Promise { - try { - await session.connection.send(held.message) - } catch (error) { - if (held.waiter.settledUuid) { - return - } - if (claudeUserMessageWasProvablyUnwritten(error)) { - rejectHeld(session, held, dispatchWriteFailureReason(error)) - return - } - // Possibly written: only a replay or the child's exit can settle it now. - retireWaiter(session, held.waiter) - held.waiter.resolve(null) - } -} - -function rejectHeld(session: ClaudeSession, held: ClaudeStartupHeldWrite, reason: string): void { - forgetWaiter(session, held.waiter) - forgetRetiredWaiter(session, held.waiter) - held.waiter.resolve(null) - if (held.waiter.clientMessageId) { - held.settleLate?.({ clientMessageId: held.waiter.clientMessageId, state: 'rejected', reason }) - } -} - -/** Rejects every held prompt with `reason`; true when any was held. */ -export function rejectClaudeStartupWrites(session: ClaudeSession, reason: string): boolean { - const held = session.startup.held.splice(0) - for (const entry of held) { - rejectHeld(session, entry, reason) - } - return held.length > 0 -} - -/** Startup cannot land any more; nothing held was written, so all of it is rejected. */ -export function failClaudeStartupGate(session: ClaudeSession, error: Error): void { - const gate = session.startup - if (gate.state === 'pending') { - gate.state = 'failed' - gate.failure = error - } - rejectClaudeStartupWrites(session, providerStartupFailureRejection(error)) -} diff --git a/src/main/claude/claude-structured-session-startup-state.ts b/src/main/claude/claude-structured-session-startup-state.ts new file mode 100644 index 00000000000..1bda560377a --- /dev/null +++ b/src/main/claude/claude-structured-session-startup-state.ts @@ -0,0 +1,58 @@ +// Where a Claude start stands. A session is published once its child is spawned, before the CLI +// has answered initialize. Nothing is written to it until startup lands (init facts read and saved +// options restored): the host's delivery loop waits on `settled` before it hands a message over, +// so a first turn never runs under defaults the restore was about to replace. + +import { providerStartupFailureRejection } from '../native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement' +import type { ClaudeSession } from './claude-structured-session-state' + +export type ClaudeSessionStartup = { + state: 'pending' | 'proven' | 'failed' + failure: Error | null + /** Resolves once startup has landed or faulted, or the child exited or was closed; never + * rejects. A close must end it: the delivery loop waits here, and a start Stop cut short + * would otherwise hold that loop forever. */ + settled: Promise + end: () => void +} + +export function createClaudeSessionStartup(): ClaudeSessionStartup { + let end: () => void = () => undefined + const ended = new Promise((resolve) => { + end = resolve + }) + return { state: 'pending', failure: null, settled: ended, end } +} + +export function claudeStartupFailureReason(session: ClaudeSession): string | null { + return session.startup.state === 'failed' + ? providerStartupFailureRejection(session.startup.failure ?? undefined) + : null +} + +/** Resolves when startup lands or `timeoutMs` passes; a stuck start then refuses the write as before. */ +export function claudeStartupSettledWithin( + session: ClaudeSession | undefined, + timeoutMs: number +): Promise { + if (session?.startup.state !== 'pending') { + return Promise.resolve() + } + let timer: ReturnType | undefined + return Promise.race([ + session.startup.settled, + new Promise((resolve) => { + timer = setTimeout(resolve, timeoutMs) + }) + ]).finally(() => clearTimeout(timer)) +} + +/** Startup cannot land any more: the child exited, was closed, or its start faulted. */ +export function failClaudeStartup(session: ClaudeSession, error: Error): void { + const startup = session.startup + if (startup.state === 'pending') { + startup.state = 'failed' + startup.failure = error + } + startup.end() +} diff --git a/src/main/claude/claude-structured-session-startup.test.ts b/src/main/claude/claude-structured-session-startup.test.ts index 362f4d691d0..ceb9e9cdf6d 100644 --- a/src/main/claude/claude-structured-session-startup.test.ts +++ b/src/main/claude/claude-structured-session-startup.test.ts @@ -55,7 +55,7 @@ describe('Claude structured session publishes before the CLI answers initialize' expect(adapter.readCommands('session-1')).toBeUndefined() await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') + await adapter.awaitStarted('session-1') expect(events.find((event) => event.type === 'options')).toMatchObject({ models: [{ value: 'claude-sonnet' }] @@ -65,7 +65,7 @@ describe('Claude structured session publishes before the CLI answers initialize' await adapter.closeAll() }) - it('reports `started` once saved options are restored, before any held prompt is written', async () => { + it('reports `started` once saved options are restored, having written no prompt of its own', async () => { const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS, initModel: 'claude-opus-9' }) const { adapter, events } = startingAdapter(claude) const order: string[] = [] @@ -74,11 +74,10 @@ describe('Claude structured session publishes before the CLI answers initialize' return undefined } await adapter.acquire({ ...ACQUIRE, options: { model: 'opus' } }) - await adapter.dispatch(PROMPT) expect(events.some((event) => event.type === 'started')).toBe(false) await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') + await adapter.awaitStarted('session-1') const startedAt = events.findIndex((event) => event.type === 'started') expect(events[startedAt]).toEqual({ @@ -90,9 +89,9 @@ describe('Claude structured session publishes before the CLI answers initialize' reportedOptions: expect.objectContaining({ model: 'opus' }), restoreSkippedOptions: [] }) - // The restore wrote the saved model before `started`, and the held prompt only after it. + // The restore wrote the saved model before `started`; no message waits inside the adapter. expect(order).toEqual(['set_model']) - expect(claude.connections[0].sent).toHaveLength(1) + expect(claude.connections[0].sent).toEqual([]) expect(events.slice(0, startedAt).some((event) => event.type === 'options')).toBe(true) await adapter.closeAll() }) @@ -133,52 +132,37 @@ describe('Claude structured session publishes before the CLI answers initialize' await adapter.closeAll() }) - it('holds a prompt sent before init and writes it once startup lands', async () => { + // The host's delivery loop waits here before it hands a message over, so the adapter no longer + // holds prompts of its own: nothing is written until startup lands because nothing is sent. + it('resolves awaitStarted only once startup lands', async () => { const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) const { adapter } = startingAdapter(claude) await adapter.acquire(ACQUIRE) - - await expect(adapter.dispatch(PROMPT)).resolves.toEqual({ state: 'admitted' }) - expect(claude.connections[0].sent).toEqual([]) - - await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') - - expect(claude.connections[0].sent).toHaveLength(1) - expect(claude.connections[0].sent[0]).toMatchObject({ type: 'user' }) - await adapter.closeAll() - }) - - it('writes a prompt whose admission barrier was still running when startup landed', async () => { - const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) - const { adapter } = startingAdapter(claude) - await adapter.acquire(ACQUIRE) - let passBarrier = (): void => {} - const barrier = new Promise((resolve) => { - passBarrier = resolve + let started = false + const waited = adapter.awaitStarted('session-1').then(() => { + started = true }) - const dispatched = adapter.dispatch({ ...PROMPT, beforeDispatch: () => barrier }) - await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') - passBarrier() + await vi.advanceTimersByTimeAsync(SLOW_INIT_MS - 1) + expect(started).toBe(false) + await vi.advanceTimersByTimeAsync(1) + await waited - await expect(dispatched).resolves.toEqual({ state: 'admitted' }) - expect(claude.connections[0].sent.filter((message) => message.type === 'user')).toHaveLength(1) + await expect(adapter.dispatch(PROMPT)).resolves.toEqual({ state: 'admitted' }) + expect(claude.connections[0].sent).toEqual([expect.objectContaining({ type: 'user' })]) await adapter.closeAll() }) - it('ends the session with the exit reason when the CLI dies before init, and rejects held prompts', async () => { + it('ends the session with the exit reason when the CLI dies before init', async () => { const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS, exitBeforeInit: 'claude stream-json exited (code 1): stderr says no' }) - const { adapter, events, late } = startingAdapter(claude) + const { adapter, events } = startingAdapter(claude) await adapter.acquire(ACQUIRE) - await adapter.dispatch(PROMPT) await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') + await adapter.awaitStarted('session-1') await adapter.drainObservedExits() expect(events.find((event) => event.type === 'ended')).toMatchObject({ @@ -186,9 +170,6 @@ describe('Claude structured session publishes before the CLI answers initialize' cause: 'unexpected-exit', startupUnproven: true }) - expect(late).toEqual([ - expect.objectContaining({ clientMessageId: 'client-1', state: 'rejected' }) - ]) expect(claude.connections[0].sent).toEqual([]) expect(claude.connections[0].closeCount).toBe(1) }) @@ -203,7 +184,7 @@ describe('Claude structured session publishes before the CLI answers initialize' await adapter.acquire(ACQUIRE) await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') + await adapter.awaitStarted('session-1') await adapter.drainObservedExits() // A failed start is released on the same evidence a failed create is. @@ -217,7 +198,7 @@ describe('Claude structured session publishes before the CLI answers initialize' const claude = fakeClaude({ initAccount: { apiProvider: 'firstParty', tokenSource: 'none' } }) const { adapter, events } = startingAdapter(claude) await adapter.acquire(ACQUIRE) - await adapter.drainStartup('session-1') + await adapter.awaitStarted('session-1') await adapter.drainObservedExits() expect(events.find((event) => event.type === 'ended')).toMatchObject({ @@ -226,82 +207,41 @@ describe('Claude structured session publishes before the CLI answers initialize' }) }) - it('closes a session stopped before init without faulting it or writing held prompts', async () => { - const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) - const { adapter, events, late } = startingAdapter(claude) + // A Stop that closes a child still starting must end the wait the host's delivery loop is in, + // though initialize never answers; otherwise every later send joins a loop that never moves. + it('ends the wait on a start closed before init, without faulting it', async () => { + const claude = fakeClaude({ initDelayMs: 10 * SLOW_INIT_MS }) + const { adapter, events } = startingAdapter(claude) await adapter.acquire(ACQUIRE) - await adapter.dispatch(PROMPT) + let ended = false + const waited = adapter.awaitStarted('session-1').then(() => { + ended = true + }) await expect(adapter.closeSession('session-1')).resolves.toBe(true) - await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') + await vi.advanceTimersByTimeAsync(0) + await waited + expect(ended).toBe(true) const connection = claude.connections[0] expect(connection.closeCount).toBe(1) expect(connection.sent).toEqual([]) expect(connection.calls.map(({ subtype }) => subtype)).not.toContain('get_settings') - expect(late).toEqual([ - expect.objectContaining({ clientMessageId: 'client-1', state: 'rejected' }) - ]) expect(events.some((event) => event.type === 'ended' && event.startupUnproven)).toBe(false) expect(events.some((event) => event.type === 'started')).toBe(false) }) - it('withdraws a held prompt when the turn is cancelled before init', async () => { + it('interrupts nothing when Stop lands before init: nothing was written', async () => { const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) - const { adapter, late } = startingAdapter(claude) + const { adapter } = startingAdapter(claude) await adapter.acquire(ACQUIRE) - await adapter.dispatch(PROMPT) await expect( adapter.cancelTurn({ sessionId: 'session-1', turnId: 'turn-1', fence: 7 }) - ).resolves.toEqual({ cancelled: true }) - await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - await adapter.drainStartup('session-1') + ).resolves.toEqual({ cancelled: false }) + expect(claude.connections[0].calls.map(({ subtype }) => subtype)).not.toContain('interrupt') expect(claude.connections[0].sent).toEqual([]) - expect(late).toEqual([ - expect.objectContaining({ clientMessageId: 'client-1', state: 'rejected' }) - ]) - await adapter.closeAll() - }) - - it('withdraws the prompts still held when Stop lands while startup is writing them', async () => { - const claude = fakeClaude({ initDelayMs: SLOW_INIT_MS }) - const { adapter, late } = startingAdapter(claude) - await adapter.acquire(ACQUIRE) - const connection = claude.connections[0] - const send = connection.send - let landFirstWrite = (): void => {} - const firstWrite = new Promise((resolve) => { - landFirstWrite = resolve - }) - let writes = 0 - connection.send = async (message, beforeDispatch) => { - writes += 1 - if (writes === 1) { - await firstWrite - } - return send(message, beforeDispatch) - } - await adapter.dispatch(PROMPT) - await adapter.dispatch({ ...PROMPT, clientMessageId: 'client-2' }) - - await vi.advanceTimersByTimeAsync(SLOW_INIT_MS) - // Startup has landed and is writing the first held prompt. - expect(writes).toBe(1) - const cancelled = adapter.cancelTurn({ sessionId: 'session-1', turnId: 'turn-1', fence: 7 }) - await vi.advanceTimersByTimeAsync(0) - landFirstWrite() - await vi.advanceTimersByTimeAsync(5_000) - await adapter.drainStartup('session-1') - - expect(connection.sent.filter((message) => message.type === 'user')).toHaveLength(1) - expect(late).toContainEqual( - expect.objectContaining({ clientMessageId: 'client-2', state: 'rejected' }) - ) - // Stop withdrew something, so it answers as a cancel whatever the interrupt made of the turn. - await expect(cancelled).resolves.toEqual({ cancelled: true }) await adapter.closeAll() }) }) diff --git a/src/main/claude/claude-structured-session-startup.ts b/src/main/claude/claude-structured-session-startup.ts index aa1f9a02ecc..d486a670a13 100644 --- a/src/main/claude/claude-structured-session-startup.ts +++ b/src/main/claude/claude-structured-session-startup.ts @@ -26,10 +26,7 @@ import { observeClaudeSettingsApplied, readClaudeSettingsEffort } from './claude-structured-session-options' -import { - failClaudeStartupGate, - openClaudeStartupGate -} from './claude-structured-session-startup-gate' +import { failClaudeStartup } from './claude-structured-session-startup-state' import type { ClaudeSession, ClaudeStructuredSessionEvent } from './claude-structured-session-state' export type ClaudeInitProof = { @@ -163,8 +160,8 @@ function claudeStartedReportedOptions( return persisted } -/** Applies startup facts to the published session, restores saved options, then releases - * held prompts. Any failure faults the session so the user sees why it never started. */ +/** Applies startup facts to the published session and restores saved options; only then does the + * session take input. Any failure faults the session so the user sees why it never started. */ export async function settleClaudeSessionStartup(input: { session: ClaudeSession facts: Promise @@ -179,7 +176,7 @@ export async function settleClaudeSessionStartup(input: { if (input.isCurrent()) { return false } - failClaudeStartupGate(session, new Error('claude session closed before startup completed')) + failClaudeStartup(session, new Error('claude session closed before startup completed')) return true } try { @@ -198,13 +195,15 @@ export async function settleClaudeSessionStartup(input: { ), restoreSkippedOptions: [...session.restoreSkippedOptions] }) - await openClaudeStartupGate(session) + if (session.startup.state === 'pending') { + session.startup.state = 'proven' + } } } catch (caught) { const error = caught instanceof Error ? caught : new Error(String(caught)) // A close or exit that already ended startup owns how the session ends. const endedElsewhere = session.startup.state !== 'pending' - failClaudeStartupGate(session, error) + failClaudeStartup(session, error) if (!endedElsewhere && input.isCurrent()) { input.fault(error) } diff --git a/src/main/claude/claude-structured-session-state.ts b/src/main/claude/claude-structured-session-state.ts index 60055874b37..b671dd8b632 100644 --- a/src/main/claude/claude-structured-session-state.ts +++ b/src/main/claude/claude-structured-session-state.ts @@ -25,7 +25,7 @@ import type { AgentChildWorkEvidence } from '../../shared/agent-status-child-wor import type { ClaudeBackgroundTaskTracker } from './claude-background-task-tracker' import type { ClaudeChildWorkDecoder } from './claude-child-work-decoder' import type { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog' -import type { ClaudeSessionStartupGate } from './claude-structured-session-startup-gate' +import type { ClaudeSessionStartup } from './claude-structured-session-startup-state' export type ClaudeAuthDiagnostic = { apiKeySourceConfigured: boolean @@ -195,7 +195,7 @@ export type ClaudeSession = { events: StructuredAgentSessionEventSink | undefined unbindReadingControl?: () => void /** Published at spawn; init facts, option restore and queued prompts land when startup does. */ - startup: ClaudeSessionStartupGate + startup: ClaudeSessionStartup } export function mintClaudeAcquisitionGeneration(deps: ClaudeStructuredSessionAdapterDeps): string { diff --git a/src/main/claude/claude-structured-session-test-support.ts b/src/main/claude/claude-structured-session-test-support.ts index 023e0ba46b7..c8e3d86b0e8 100644 --- a/src/main/claude/claude-structured-session-test-support.ts +++ b/src/main/claude/claude-structured-session-test-support.ts @@ -220,7 +220,7 @@ export function adapterFor( const acquire = adapter.acquire adapter.acquire = async (input) => { const acquisition = await acquire(input) - await adapter.drainStartup(input.identity.sessionId) + await adapter.awaitStarted(input.identity.sessionId) return acquisition } return adapter diff --git a/src/main/ipc/desktop-renderer-runtime-capabilities.ts b/src/main/ipc/desktop-renderer-runtime-capabilities.ts index 63ba9b1d0b1..0bcfd1bd686 100644 --- a/src/main/ipc/desktop-renderer-runtime-capabilities.ts +++ b/src/main/ipc/desktop-renderer-runtime-capabilities.ts @@ -1,5 +1,6 @@ import { AGENT_LAUNCH_RUNTIME_CAPABILITY, + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, AGENT_SESSION_BACKGROUND_TASK_ROW_STOP_CAPABILITY, AGENT_SESSION_BACKGROUND_TASK_STOP_CAPABILITY, AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY, @@ -25,6 +26,7 @@ import { export const DESKTOP_RENDERER_RUNTIME_CLIENT_CAPABILITIES: readonly RuntimeCapability[] = [ AGENT_SESSION_BACKGROUND_TASK_STOP_CAPABILITY, AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY, + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, AGENT_SESSION_TURN_ITEM_CAPABILITY, AGENT_SESSION_BACKGROUND_TASK_ROW_STOP_CAPABILITY, STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY, diff --git a/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts b/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts index c4138ba2241..c356d7904b2 100644 --- a/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts +++ b/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts @@ -1,8 +1,11 @@ +import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import { DISPATCH_DOUBT_HOST_RESTARTED } from './journal-dispatch-doubt-reasons' import type { AgentSessionJournal } from './journal-store' /** Settles every submission a process fact left unanswerable. Doubt is never - * proof of non-delivery, so nothing here ever becomes re-deliverable. */ + * proof of non-delivery, so nothing here ever becomes re-deliverable. A queued + * submission was never handed over, so it is not in doubt and is left alone. */ export async function markJournalPendingSubmissionsUnknown( journal: AgentSessionJournal, fence: number, @@ -12,8 +15,9 @@ export async function markJournalPendingSubmissionsUnknown( .submissions() .filter( (entry) => - entry.dispatchState === 'pending' || - (entry.dispatchState === 'unknown' && entry.recovered !== true) + !isQueuedAgentJournalSubmission(entry) && + (entry.dispatchState === 'pending' || + (entry.dispatchState === 'unknown' && entry.recovered !== true)) ) for (const entry of unresolved) { // An earlier reason already names a sharper fact than "the host restarted". @@ -31,7 +35,8 @@ export async function markJournalPendingSubmissionsUnknown( } /** Settles every submission a child that never proved its start left unanswered as `rejected`: - * such a child accepted nothing, so each is provably unwritten and safe to send again. */ + * such a child accepted nothing, so each is provably unwritten and safe to send again. A queued + * submission was never handed to that child; the delivery loop settles it. */ export async function rejectJournalPendingSubmissions( journal: AgentSessionJournal, fence: number, @@ -41,8 +46,9 @@ export async function rejectJournalPendingSubmissions( .submissions() .filter( (entry) => - entry.dispatchState === 'pending' || - (entry.dispatchState === 'unknown' && entry.recovered !== true) + !isQueuedAgentJournalSubmission(entry) && + (entry.dispatchState === 'pending' || + (entry.dispatchState === 'unknown' && entry.recovered !== true)) ) for (const entry of unwritten) { await journal.resolveDispatch({ @@ -55,3 +61,25 @@ export async function rejectJournalPendingSubmissions( } return unwritten.map((entry) => entry.clientMessageId) } + +/** Rejects queued submissions — accepted, never handed over, so provably unwritten. */ +export async function rejectJournalQueuedSubmissions( + journal: AgentSessionJournal, + fence: number, + reason: string, + which: (submission: AgentJournalSubmission) => boolean = () => true +): Promise { + const queued = journal + .submissions() + .filter((entry) => isQueuedAgentJournalSubmission(entry) && which(entry)) + for (const entry of queued) { + await journal.resolveDispatch({ + clientMessageId: entry.clientMessageId, + state: 'rejected', + reason, + fence, + recovered: true + }) + } + return queued.map((entry) => entry.clientMessageId) +} diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.ts b/src/main/native-chat/agent-session-journal/journal-reducer.ts index 273502c8cf2..3fbee85a417 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.ts @@ -254,7 +254,8 @@ function applySubmission( providerItemId: null, reason: null, submittedAt: row.ts, - resolvedAt: null + resolvedAt: null, + ...(row.handoverRecorded ? { handoverRecorded: true, acceptedSequence: row.seq } : {}) }) const itemId = agentJournalSubmissionKey(row.clientMessageId) upsertItem(state, itemId, 0, journalRenderItem(itemId, 0, row.body, row)) @@ -277,6 +278,9 @@ function applyDispatch( submission.providerItemId = row.providerItemId submission.reason = row.reason submission.resolvedAt = row.state === 'pending' ? null : row.ts + if (row.state === 'pending') { + submission.handedOverAt = row.ts + } if (row.recovered) { submission.recovered = row.recovered } else { diff --git a/src/main/native-chat/agent-session-journal/journal-restart-reconciliation.ts b/src/main/native-chat/agent-session-journal/journal-restart-reconciliation.ts index 626baa9820a..f37571814bb 100644 --- a/src/main/native-chat/agent-session-journal/journal-restart-reconciliation.ts +++ b/src/main/native-chat/agent-session-journal/journal-restart-reconciliation.ts @@ -18,6 +18,7 @@ import { agentJournalItemKey, agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import type { AgentSessionJournal } from './journal-store' import { reconcileSubmissions, type ProviderHistoryWindow } from './journal-submission-reconciler' @@ -42,7 +43,11 @@ function comparableSubmissions(journal: AgentSessionJournal): AgentJournalSubmis const { items, submissions } = journal.snapshot() const bodies = new Map(items.map((item) => [item.itemId, item.body])) return submissions.filter((submission) => { - if (submission.dispatchState !== 'pending' && submission.dispatchState !== 'unknown') { + if ( + (submission.dispatchState !== 'pending' && submission.dispatchState !== 'unknown') || + // Never handed over, so provider history cannot hold it. + isQueuedAgentJournalSubmission(submission) + ) { return false } const body = bodies.get(agentJournalSubmissionKey(submission.clientMessageId)) diff --git a/src/main/native-chat/agent-session-journal/journal-row-builders.ts b/src/main/native-chat/agent-session-journal/journal-row-builders.ts index e17ccf0cdf8..e56ca86a7fb 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-builders.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-builders.ts @@ -65,6 +65,7 @@ export function journalSubmissionRowBuilder( payloadFingerprint: string body: AgentJournalMessageItem fence: number + handoverRecorded?: true } ): RowBuilder { return (seq, ts) => @@ -267,6 +268,7 @@ export function buildJournalSubmissionRow(input: { seq: number fence: number ts: number + handoverRecorded?: true }): JournalSubmissionRow { return { kind: 'submission', @@ -274,7 +276,8 @@ export function buildJournalSubmissionRow(input: { payloadFingerprint: input.payloadFingerprint, providerHandle: input.providerHandle, body: input.body, - ...journalRowBase(input.state.epoch, input.seq, input.fence, input.ts) + ...journalRowBase(input.state.epoch, input.seq, input.fence, input.ts), + ...(input.handoverRecorded ? { handoverRecorded: true } : {}) } } diff --git a/src/main/native-chat/agent-session-journal/journal-row-schema.ts b/src/main/native-chat/agent-session-journal/journal-row-schema.ts index 99cb47971e9..0f29f8b3879 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-schema.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-schema.ts @@ -77,6 +77,9 @@ export type JournalSubmissionRow = JournalRowBase & { payloadFingerprint: string providerHandle: AgentSessionProviderHandle body: AgentJournalMessageItem + /** Accepted to be handed over by a later `dispatch{pending}` row; absent on rows whose writer + * dispatched in the same step. Older readers keep the key and ignore it. */ + handoverRecorded?: true } export type JournalDispatchRow = JournalRowBase & { diff --git a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts index cea47e381fe..79d86f80dbb 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts @@ -59,6 +59,8 @@ export type JournalSubmissionInput = { payloadFingerprint: string body: AgentJournalMessageItem fence: number + /** The send is accepted now and handed over later, by a `dispatch{pending}` row. */ + handoverRecorded?: true } export type JournalItemAppendInput = { diff --git a/src/main/native-chat/agent-session-journal/journal-store.test.ts b/src/main/native-chat/agent-session-journal/journal-store.test.ts index ffaea5c5d6f..3637f03dab6 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.test.ts @@ -424,3 +424,31 @@ async function withJournalDatabase( opened.db.close() } } + +describe('what a handle found on disk when it opened', () => { + const submission = (clientMessageId: string) => ({ + clientMessageId, + payloadFingerprint: 'fp', + body: { kind: 'message' as const, role: 'user' as const, blocks: [] }, + fence: 1, + handoverRecorded: true as const + }) + + it('names rows an earlier handle wrote, and never a row of a later epoch', async () => { + const earlier = await open() + await earlier.appendItem(item(1), body('one'), { fence: 1 }) + await earlier.appendSubmission(submission('earlier')) + await earlier.close() + + const journal = await open() + const leftover = journal.submissions().find((entry) => entry.clientMessageId === 'earlier') + expect(journal.wroteBeforeOpen(leftover?.acceptedSequence)).toBe(true) + + // Sequences restart with an epoch, so a row accepted after it can sit below the open cursor. + await journal.replaceEpochItems('handle_forked', 1, []) + await journal.appendSubmission(submission('later')) + const later = journal.submissions().find((entry) => entry.clientMessageId === 'later') + expect(later?.acceptedSequence).toBeLessThanOrEqual(2) + expect(journal.wroteBeforeOpen(later?.acceptedSequence)).toBe(false) + }) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 568fbe4d39d..5119f7ef597 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -28,7 +28,8 @@ import { readJournalRowsAfterCursor, type JournalLoad } from './journal-open' import { journalDatabaseFile } from './journal-paths' import { markJournalPendingSubmissionsUnknown, - rejectJournalPendingSubmissions + rejectJournalPendingSubmissions, + rejectJournalQueuedSubmissions } from './journal-pending-submission-recovery' import { applyJournalRow, @@ -75,6 +76,7 @@ export class AgentSessionJournal { private state: JournalReducerState private readOnly = false private malformedRows = 0 + private openedThrough: AgentJournalCursor = { epoch: '', sequence: 0 } private database: OpenJournalDatabase | null = null private onCommitted: (() => void) | null = null private readonly queue: JournalWriteQueue @@ -147,6 +149,16 @@ export class AgentSessionJournal { return this.journalDir } + /** Whether a row at this sequence was on disk when this handle opened, so an earlier handle + * wrote it. Sequences restart with each epoch, so a row of a later epoch never was. */ + wroteBeforeOpen(sequence: number | undefined): boolean { + return ( + sequence !== undefined && + this.state.epoch === this.openedThrough.epoch && + sequence <= this.openedThrough.sequence + ) + } + /** What the last open's repair did. */ get repair(): { malformedRows: number } { return { malformedRows: this.malformedRows } @@ -157,6 +169,7 @@ export class AgentSessionJournal { this.database = openJournalDatabase(this.dbPath) try { await this.restore() + this.openedThrough = this.cursor() } catch (error) { // Nothing else holds a reference to this connection, so a throw here is // the leak site unless the store releases it itself — and a close that @@ -310,6 +323,15 @@ export class AgentSessionJournal { return rejectJournalPendingSubmissions(this, fence, reason) } + /** Reject sends accepted but never handed over, optionally only those `which` names. */ + async rejectQueuedSubmissions( + fence: number, + reason: string, + which?: (submission: AgentJournalSubmission) => boolean + ): Promise { + return rejectJournalQueuedSubmissions(this, fence, reason, which) + } + /** The escape hatch for corruption, an unreconcilable prefix, a forked handle, * and an unreadable schema. It invalidates every cursor; clients reload. */ async rollEpoch(reason: AgentJournalEpochReason, fence: number): Promise { diff --git a/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.test.ts b/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.test.ts index 214c889b5c9..1c60029c7d3 100644 --- a/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.test.ts +++ b/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.test.ts @@ -6,14 +6,14 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { AgentJournalItemIdentity, AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' import { openJournalDatabase } from '../agent-session-journal/journal-database' import { JOURNAL_DB_SCHEMA_VERSION } from '../agent-session-journal/journal-database-schema' -import { loadJournal } from '../agent-session-journal/journal-open' +import { loadJournal, replayJournal } from '../agent-session-journal/journal-open' import { journalDatabaseFile } from '../agent-session-journal/journal-paths' import { readJournalEpochRows } from '../agent-session-journal/journal-row-table' import { createTrackedJournalOpener } from '../agent-session-journal/journal-store-test-open' @@ -24,6 +24,12 @@ import { recoveryJournalDir } from './agent-session-journal-recovery' +// Only the store's own replay goes through the mock; the probe's, inside the same module, does not. +vi.mock('../agent-session-journal/journal-open', async (importOriginal) => { + const actual = await importOriginal<{ replayJournal: typeof replayJournal }>() + return { ...actual, replayJournal: vi.fn(actual.replayJournal) } +}) + const CODEX_SESSION = '019fd532-7c11-7a90-b6de-4e1a2c3d5f60' const IDENTITY: AgentSessionJournalIdentity = { @@ -160,6 +166,21 @@ describe('openAgentSessionJournalWithRecovery', () => { expect(opened.snapshot().items).toHaveLength(2) }) + it('reads the journal once: the probe is the open', async () => { + await seedJournal(2) + vi.mocked(replayJournal).mockClear() + const opened = journals.track( + await openAgentSessionJournalWithRecovery({ + identity: IDENTITY, + journalDir, + fence: 1, + historyFilePath + }).then((result) => result.journal) + ) + expect(opened.snapshot().items).toHaveLength(2) + expect(replayJournal).not.toHaveBeenCalled() + }) + it('rebuilds a holed journal in place on a fresh epoch', async () => { await seedJournal(3) await deleteRow(3) diff --git a/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.ts b/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.ts index 6e771a31809..29d324be971 100644 --- a/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.ts +++ b/src/main/native-chat/agent-session-wire/agent-session-journal-recovery.ts @@ -67,7 +67,9 @@ export async function openAgentSessionJournalWithRecovery(input: { } const journal = await openAgentSessionJournal({ identity: input.identity, - journalDir: input.journalDir + journalDir: input.journalDir, + // The probe is this open's replay; omitted, not `null`, when there was nothing to load. + ...(probe ? { loaded: probe } : {}) }) if (!probe?.corrupt) { return { journal, recovery: null } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts new file mode 100644 index 00000000000..186127847a6 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts @@ -0,0 +1,671 @@ +// A send is accepted, then delivered: the host answers once the message is recorded, and the +// session's delivery loop starts a provider child for it and hands it over. Against the real host, +// store and journal; each assertion reads what an open chat or the journal's next reader sees. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' +import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire' +import { + DISPATCH_REJECTED_CANCELLED, + DISPATCH_REJECTED_HOST_RESTARTED, + DISPATCH_REJECTED_PROVIDER_CLOSED +} from '../../../shared/structured-agent-session-dispatch-rejection' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { journalDirectoryFor } from '../agent-session-journal/journal-paths' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' +import { + AgentSessionPreSpawnError, + type StructuredAgentSessionAdapter +} from './structured-agent-session-adapter' +import { journalIdentityFor } from './structured-agent-session-attach' +import { attachParamsForRecord } from './structured-agent-session-conversation-open' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { persistRewindRecord } from './structured-rewind-recovery' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +const CALLER = { callerKey: 'client-1' } + +function eventually(assertion: () => void | Promise): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost +let acquire: Mock +let dispatch: Mock +let adapterExtras: Partial +let releaseGraceMs: number + +const spawnChild: StructuredAgentSessionAdapter['acquire'] = async ({ fence, spawnToken }) => ({ + process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, + acquisitionGeneration: `generation-${acquire.mock.calls.length}`, + link: { + linkId: `link-${fence}`, + handle: { provider: 'codex' as const, threadId: THREAD }, + origin: store.getRecord(SESSION)?.providerHandleChain.length + ? ('resumed' as const) + : ('created' as const), + mintedAtFence: fence, + observedAt: NOW + } +}) + +async function startHost(): Promise { + host = new StructuredAgentSessionHost({ + store, + adapter: { + acquire, + dispatch, + closeSession: vi.fn(async () => true), + releaseAcquisition: vi.fn(async () => true), + cancelTurn: vi.fn(async () => ({ cancelled: false })), + answerPrompt: vi.fn(async () => undefined), + setOption: vi.fn(async () => undefined), + ...adapterExtras + }, + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => `spawn-${acquire.mock.calls.length}`, + releaseGraceMs, + now: () => NOW + }) +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-accept-deliver-')) + resetHostTestOperationIds() + adapterExtras = {} + releaseGraceMs = 60_000 + acquire = vi.fn(spawnChild) + dispatch = vi.fn(async () => ({ + state: 'accepted' as const, + providerIdentity: { + provider: 'codex' as const, + threadId: THREAD, + turnId: `turn-${dispatch.mock.calls.length}`, + ordinal: dispatch.mock.calls.length + } + })) + store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) + await startHost() + expect(await host.attach(CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true }) +}) + +afterEach(async () => { + await host.flushAllStreamedEvents() + await rm(root, { recursive: true, force: true }) +}) + +function sendParams(text: string) { + const body = hostTestMessage(text) + return { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: 1, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + }, + body + } +} + +/** Accepted at once, whatever the child is doing; answers the message id. */ +async function accept(text: string): Promise { + const params = sendParams(text) + expect(await host.send(CALLER, params)).toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'pending', handoverRecorded: true } } + }) + return params.envelope.clientOperationId +} + +function stop() { + const turnId = 'turn-none' + return host.cancel(CALLER, { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: null, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.cancel', + sessionId: SESSION, + fields: { turnId } + }) + }, + turnId + }) +} + +function submission(id: string): AgentJournalSubmission | undefined { + return host.journalSnapshot(SESSION).submissions.find((entry) => entry.clientMessageId === id) +} + +/** Read back after the conversation was closed, through the same open any reader takes. */ +async function reopened(id: string): Promise { + await host.revealSession(SESSION) + return submission(id) +} + +function errorRows(): string[] { + return host + .journalSnapshot(SESSION) + .items.flatMap((item) => + item.body.kind === 'status' && item.body.tone === 'error' ? [item.body.text] : [] + ) +} + +let subscriptions = 0 + +function subscribe(): AgentSessionSubscribeEvent[] { + const events: AgentSessionSubscribeEvent[] = [] + subscriptions += 1 + // Cloned as received: a frame shares the journal's live objects, which later rows revise. + host.subscribe({ + id: `sub-${subscriptions}`, + sessionId: SESSION, + emit: (event) => events.push(structuredClone(event)) + }) + return events +} + +/** What an open chat was told about one message, in frame order. */ +function framedStates(events: AgentSessionSubscribeEvent[], id: string): string[] { + return events.flatMap((event) => + event.type === 'batch' + ? event.batch.submissions + .filter((entry) => entry.clientMessageId === id) + .map((entry) => + entry.dispatchState === 'pending' && entry.handedOverAt !== undefined + ? 'handed-over' + : entry.dispatchState + ) + : [] + ) +} + +function deferred() { + let resolve!: (value: T) => void + let reject!: (error: unknown) => void + const promise = new Promise((next, fail) => { + resolve = next + reject = fail + }) + return { promise, resolve, reject } +} + +/** Rows written by an earlier host process that ended before handing them over. */ +async function writeAsEarlierProcess( + write: (journal: AgentSessionJournal, fence: number) => Promise +): Promise { + await host.close(SESSION) + const record = store.getRecord(SESSION)! + const params = attachParamsForRecord(record, { + clientOperationId: 'earlier', + expectedRuntimeFence: record.lease.runtimeFence + }) + const journal = await openAgentSessionJournal({ + identity: journalIdentityFor(record, params), + journalDir: journalDirectoryFor(root, { + workspaceId: record.location.workspaceId, + sessionId: SESSION + }) + }) + await write(journal, record.lease.runtimeFence) + await journal.close() +} + +function earlierSubmission(id: string, text: string, handoverRecorded?: true) { + const body = hostTestMessage(text) + return { + clientMessageId: id, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }), + body, + ...(handoverRecorded ? { handoverRecorded } : {}) + } +} + +describe('a send is answered at acceptance', () => { + it('answers before the child starts, then an open chat sees the handover and the reply (W2)', async () => { + await host.close(SESSION) + const starting = deferred() + acquire.mockImplementationOnce(async (input) => { + await starting.promise + return spawnChild(input) + }) + + const answering = deferred() + const answer = dispatch.getMockImplementation()! + dispatch.mockImplementationOnce(async (input) => { + await answering.promise + return answer(input) + }) + + const id = await accept('hello') + const events = subscribe() + await eventually(() => expect(acquire).toHaveBeenCalledTimes(2)) + expect(dispatch).not.toHaveBeenCalled() + + starting.resolve() + await eventually(() => expect(framedStates(events, id)).toEqual(['handed-over'])) + answering.resolve() + await eventually(() => expect(framedStates(events, id)).toEqual(['handed-over', 'accepted'])) + }) + + it('accepts a second send while the first one starts the child, before handing either over (W6)', async () => { + await host.close(SESSION) + const starting = deferred() + acquire.mockImplementationOnce(async (input) => { + await starting.promise + return spawnChild(input) + }) + const first = await accept('first') + await eventually(() => expect(acquire).toHaveBeenCalledTimes(2)) + const second = host.send(CALLER, sendParams('second')) + + starting.resolve() + const secondResult = await second + if (!secondResult.ok) { + throw new Error('the second send was refused') + } + const secondId = secondResult.value.clientMessageId + await eventually(() => expect(submission(secondId)?.dispatchState).toBe('accepted')) + expect(dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([first, secondId]) + // The second was accepted while the start held the queue — before the first was handed over. + expect(submission(secondId)!.submittedAt).toBeLessThanOrEqual(submission(first)!.handedOverAt!) + const rows = host.journalSnapshot(SESSION).submissions + expect(rows.map((row) => row.clientMessageId)).toEqual([first, secondId]) + }) +}) + +describe('a start the chat needed and did not get', () => { + it('writes one error row and rejects every queued message with it; the next send starts (W3)', async () => { + await host.close(SESSION) + acquire.mockRejectedValueOnce(new Error('spawn codex ENOENT')) + const first = await accept('first') + const second = await accept('second') + + await eventually(() => expect(submission(second)?.dispatchState).toBe('rejected')) + const rows = errorRows() + expect(rows).toHaveLength(1) + expect(rows[0]).toContain('spawn codex ENOENT') + expect(submission(first)).toMatchObject({ dispatchState: 'rejected', reason: rows[0] }) + expect(submission(second)).toMatchObject({ dispatchState: 'rejected', reason: rows[0] }) + + const next = await accept('after the fix') + await eventually(() => expect(submission(next)?.dispatchState).toBe('accepted')) + expect(errorRows()).toHaveLength(1) + }) + + it.each([ + [ + 'eligibility', + () => { + adapterExtras = { supportsLocation: () => false } + }, + 'cannot resume' + ], + [ + 'spawn', + () => acquire.mockRejectedValueOnce(new Error('spawn codex ENOENT')), + 'spawn codex ENOENT' + ], + [ + 'auth', + () => + acquire.mockRejectedValueOnce( + new AgentSessionPreSpawnError(new Error('Not logged in. Please run /login.')) + ), + 'Not logged in' + ] + ])('writes one row a live chat sees for a %s refusal (W14)', async (_source, arrange, cause) => { + await host.close(SESSION) + arrange() + await host.flushAllStreamedEvents() + await startHost() + const id = await accept('hello') + const events = subscribe() + + await eventually(() => expect(submission(id)?.dispatchState).toBe('rejected')) + expect(errorRows()).toHaveLength(1) + expect(errorRows()[0]).toContain(cause) + const framedRows = events.flatMap((event) => + event.type === 'batch' || event.type === 'snapshot' + ? (event.type === 'batch' ? event.batch.items : event.page.items).filter( + (item) => item.body.kind === 'status' && item.body.tone === 'error' + ) + : [] + ) + expect(framedRows.length).toBeGreaterThan(0) + }) + + it('names the start failure on queued messages when the attach fails after acquiring (W4′a)', async () => { + await host.close(SESSION) + const id = await accept('hello') + // The attach's own success record is the post-acquisition step that fails. + const record = vi.spyOn(store, 'recordOperationOutcome') + record.mockImplementation(async (input) => { + if (input.operationId !== id && input.outcome.status === 'succeeded') { + record.mockRestore() + throw new Error('record store write failed') + } + return AgentSessionRecordStore.prototype.recordOperationOutcome.call(store, input) + }) + + // Read through the open any reader takes, so a conversation the failure dropped is reopened + // and its message read as that reopen settles it. + let settled: AgentJournalSubmission | undefined + await eventually(async () => { + settled = await reopened(id) + expect(settled?.dispatchState).not.toBe('pending') + }) + // The message names the start that failed, not a close or a restart it never met. + expect(settled).toMatchObject({ + dispatchState: 'rejected', + reason: expect.stringContaining('record store write failed') + }) + expect(errorRows()).toEqual([settled?.reason]) + }) +}) + +describe('an attach that fails after indexing its child', () => { + it('leaves no child behind, so the next send starts one and is delivered', async () => { + await host.close(SESSION) + const owned: boolean[] = [] + host.subscribeStatus({ + id: 'list-1', + emit: (event) => { + if (event.type === 'status') { + owned.push(event.session.hostExecutionOwned === true) + } + } + }) + const first = await accept('hello') + // The attach's own success record is the step after `onAttached` indexed the child. + const record = vi.spyOn(store, 'recordOperationOutcome') + record.mockImplementation(async (input) => { + if (input.operationId !== first && input.outcome.status === 'succeeded') { + record.mockRestore() + throw new Error('record store write failed') + } + return AgentSessionRecordStore.prototype.recordOperationOutcome.call(store, input) + }) + await eventually(() => expect(submission(first)?.dispatchState).toBe('rejected')) + // Nothing was indexed, so nothing had to be taken back: no list ever showed a child. + expect(host['sessions'].get(SESSION)?.child).toBeNull() + expect(owned).not.toContain(true) + const acquiresBefore = acquire.mock.calls.length + + const next = await accept('after the failure') + + await eventually(() => expect(submission(next)?.dispatchState).toBe('accepted')) + expect(acquire).toHaveBeenCalledTimes(acquiresBefore + 1) + expect(dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([next]) + }) +}) + +describe('what an earlier host process left behind', () => { + it('rejects a message it accepted and never handed over, as not sent (W4′b)', async () => { + await writeAsEarlierProcess(async (journal, fence) => { + await journal.appendSubmission({ ...earlierSubmission('queued', 'q', true), fence }) + }) + + await host.revealSession(SESSION) + + expect(submission('queued')).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_HOST_RESTARTED + }) + expect(dispatch).not.toHaveBeenCalled() + }) + + it('leaves a legacy pending message and a handed-over one in doubt, never re-sent (W4′c)', async () => { + const providerHistoryWindow = vi.fn(async () => null) + adapterExtras = { providerHistoryWindow } + await writeAsEarlierProcess(async (journal, fence) => { + await journal.appendSubmission({ ...earlierSubmission('legacy', 'l'), fence }) + await journal.appendSubmission({ ...earlierSubmission('handed', 'h', true), fence }) + await journal.resolveDispatch({ clientMessageId: 'handed', state: 'pending', fence }) + }) + await host.flushAllStreamedEvents() + await startHost() + + await host.revealSession(SESSION) + + expect(submission('legacy')).toMatchObject({ dispatchState: 'unknown', recovered: true }) + expect(submission('handed')).toMatchObject({ dispatchState: 'unknown', recovered: true }) + // Deciding them from provider history waits for a won lease (W4′d). + expect(providerHistoryWindow).not.toHaveBeenCalled() + expect(dispatch).not.toHaveBeenCalled() + }) +}) + +describe('a child that exits before its message is handed over', () => { + it('rejects the message with the exit reason instead of starting another child (W24)', async () => { + // Each child the loop starts dies between its start step and its handover step. + const awaitStarted = vi.fn(async (sessionId: string) => { + await host.handleAdapterEvent({ + type: 'ended', + sessionId, + fence: store.getRecord(sessionId)!.lease.runtimeFence, + acquisitionGeneration: `generation-${acquire.mock.calls.length}`, + reason: 'codex app-server crashed', + cause: 'unexpected-exit' + }) + }) + adapterExtras = { awaitStarted } + await host.close(SESSION) + await startHost() + + const id = await accept('hello') + + await eventually(() => expect(submission(id)?.dispatchState).toBe('rejected')) + expect(submission(id)?.reason).toContain('codex app-server crashed') + expect(acquire).toHaveBeenCalledTimes(2) + expect(dispatch).not.toHaveBeenCalled() + }) +}) + +describe('Stop withdraws what is queued', () => { + it('withdraws a crash leftover ahead of any delivery step (W17a)', async () => { + await writeAsEarlierProcess(async (journal, fence) => { + await journal.appendSubmission({ ...earlierSubmission('leftover', 'l', true), fence }) + }) + + // Stop's own open wakes the delivery loop, whose first step queues behind this Stop. + expect(await stop()).toMatchObject({ ok: true }) + + expect(submission('leftover')).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_CANCELLED + }) + expect(acquire).toHaveBeenCalledTimes(1) + }) + + it('withdraws a message whose start holds the queue: nothing is handed over (W17b)', async () => { + await host.close(SESSION) + const starting = deferred() + acquire.mockImplementationOnce(async (input) => { + await starting.promise + return spawnChild(input) + }) + const id = await accept('hello') + await eventually(() => expect(acquire).toHaveBeenCalledTimes(2)) + const stopped = stop() + + starting.resolve() + expect(await stopped).toMatchObject({ ok: true }) + await eventually(() => expect(submission(id)?.dispatchState).toBe('rejected')) + expect(submission(id)).toMatchObject({ reason: DISPATCH_REJECTED_CANCELLED }) + expect(submission(id)?.handedOverAt).toBeUndefined() + expect(dispatch).not.toHaveBeenCalled() + }) + + it('stops a child still proving its start, and the delivery loop ends with it (W17c)', async () => { + const ended = deferred() + const awaitStarted = vi.fn(() => ended.promise) + const closeSession = vi.fn(async () => { + ended.resolve() + return true + }) + adapterExtras = { awaitStarted, closeSession } + await host.close(SESSION) + await startHost() + acquire.mockImplementationOnce(async (input) => ({ + ...(await spawnChild(input)), + providerChildPhase: 'starting' as const + })) + const id = await accept('hello') + await eventually(() => expect(awaitStarted).toHaveBeenCalled()) + + expect(await stop()).toMatchObject({ ok: true, value: { cancelled: true } }) + + expect(await reopened(id)).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_CANCELLED + }) + expect(closeSession).toHaveBeenCalled() + // A loop still waiting on that start would swallow this send; it starts a new child instead. + awaitStarted.mockImplementation(async () => undefined) + const next = await accept('after stop') + await eventually(() => expect(submission(next)?.dispatchState).toBe('accepted')) + expect(dispatch).toHaveBeenCalledTimes(1) + }) +}) + +describe('an eviction between acceptance and handover', () => { + it('rejects the message as not sent, never leaves it in doubt (W24)', async () => { + const started = deferred() + adapterExtras = { awaitStarted: () => started.promise } + await host.close(SESSION) + await startHost() + await host.hold(SESSION, 'surface-1') + const id = await accept('hello') + await eventually(() => expect(acquire).toHaveBeenCalledTimes(2)) + // Between the delivery loop's start step and its handover step. + await host.close(SESSION) + started.resolve() + + expect(await reopened(id)).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_PROVIDER_CLOSED + }) + expect(dispatch).not.toHaveBeenCalled() + }) + + it('rejects a queued message behind a handed-over one, which alone stays in doubt (W24)', async () => { + const second = deferred() + const awaitStarted = vi.fn(async (): Promise => undefined) + adapterExtras = { awaitStarted } + await host.close(SESSION) + await startHost() + await host.hold(SESSION, 'surface-1') + dispatch.mockResolvedValueOnce({ state: 'admitted' }) + const handed = await accept('handed over') + await eventually(() => expect(submission(handed)?.handedOverAt).toBeDefined()) + awaitStarted.mockImplementation(() => second.promise) + const queued = await accept('still queued') + await eventually(() => expect(awaitStarted).toHaveBeenCalledTimes(2)) + + await host.close(SESSION) + second.resolve() + + expect(await reopened(queued)).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_PROVIDER_CLOSED + }) + expect(submission(handed)).toMatchObject({ dispatchState: 'unknown' }) + expect(dispatch).toHaveBeenCalledTimes(1) + }) + + it('does not evict an idle child while a message is still queued for it (W24)', async () => { + releaseGraceMs = 0 + const started = deferred() + adapterExtras = { awaitStarted: () => started.promise } + await host.close(SESSION) + await startHost() + await host.hold(SESSION, 'surface-1') + const id = await accept('hello') + await eventually(() => expect(acquire).toHaveBeenCalledTimes(2)) + host.release(SESSION, 'surface-1') + await new Promise((resolve) => setTimeout(resolve, 20)) + + started.resolve() + // Handed to the child it was queued for; the eviction may follow once nothing is owed. + await eventually(() => expect(dispatch).toHaveBeenCalledTimes(1)) + expect(dispatch.mock.calls[0]?.[0].clientMessageId).toBe(id) + expect(acquire).toHaveBeenCalledTimes(2) + }) +}) + +describe('a compaction or rewind an earlier child left prepared', () => { + async function leftPrepared(prepare: (fence: number) => Promise): Promise { + await host.close(SESSION) + await prepare(store.getRecord(SESSION)!.lease.runtimeFence) + // A new process: nothing is open and no view attaches. + await host.flushAllStreamedEvents() + await startHost() + } + + it('settles an interrupted compaction at open, so a send is accepted and delivered (R16)', async () => { + await leftPrepared((fence) => + store.setConversationCommand(SESSION, fence, { + command: 'compact', + runtimeFence: fence, + operationId: 'compact-op', + callerKey: 'client-1', + phase: 'prepared', + state: 'unknown' + }) + ) + + const id = await accept('after the compaction') + + await eventually(() => expect(submission(id)?.dispatchState).toBe('accepted')) + expect(store.getRecord(SESSION)?.conversationCommand).toMatchObject({ + phase: 'committed', + state: 'unknown' + }) + }) + + it('completes a rewind the provider already applied at open, so a send is accepted (R16)', async () => { + await leftPrepared((fence) => + persistRewindRecord(store, SESSION, fence, { + operationId: 'rewind-op', + callerKey: 'client-1', + itemId: 'orca:rewound', + providerItemId: `codex:${THREAD}:turn-1:0`, + expectedEpoch: 'epoch-before', + phase: 'provider-succeeded', + hydrationVerified: true, + retained: [] + }) + ) + + const id = await accept('after the rewind') + + await eventually(() => expect(submission(id)?.dispatchState).toBe('accepted')) + expect(store.getRecord(SESSION)?.rewind).toMatchObject({ phase: 'completed' }) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-options.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-options.test.ts index e3da044d43c..8302b403033 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-options.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-options.test.ts @@ -15,6 +15,7 @@ import { attachFingerprintFields, type AgentSessionAttachParams } from './structured-agent-session-attach' +import { openTestAttachConversation } from './structured-agent-session-attach-test-conversation' import { performAttach } from './structured-agent-session-attach-flow' import type { AgentSessionCreatePhaseRecorder } from '../../observability/agent-session-instrumentation' @@ -154,6 +155,7 @@ describe('structured session acquisition options', () => { store: initialStore, adapter: withHistory('created'), journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -194,6 +196,7 @@ describe('structured session acquisition options', () => { store, adapter: withHistory('resumed'), journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-b', claimKeyId: 'key-1', @@ -228,6 +231,7 @@ describe('structured session acquisition options', () => { store, adapter: sessionAdapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -260,6 +264,7 @@ describe('structured session acquisition options', () => { store, adapter: sessionAdapter, journalRoot: root!, + openConversation: openTestAttachConversation(root!), authority: { spawnToken, claimKeyId: 'key-1', @@ -291,6 +296,7 @@ describe('structured session acquisition options', () => { store, adapter: adapter({ origin: 'created' }), journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -330,6 +336,7 @@ describe('structured session acquisition options', () => { } }), journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-b', claimKeyId: 'key-1', @@ -369,6 +376,7 @@ describe('structured session acquisition options', () => { store, adapter: sessionAdapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -408,6 +416,7 @@ describe('structured session acquisition options', () => { store, adapter: failingAdapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -499,6 +508,7 @@ describe('structured session acquisition options', () => { store: target, adapter: failingAdapter, journalRoot: root!, + openConversation: openTestAttachConversation(root!, failingAdapter), authority: { spawnToken: operationId === CREATE_OPERATION ? 'spawn-a' : 'spawn-b', claimKeyId: 'key-1', @@ -596,6 +606,7 @@ describe('the tab a create reserves', () => { store, adapter: adapter({ origin: 'created' }), journalRoot: root!, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts index 7fa1446d6c5..566e41a6635 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter-router.ts @@ -122,6 +122,8 @@ export class StructuredAgentSessionAdapterRouter implements StructuredAgentSessi awaitOptionWritable = (sessionId: string): Promise => this.liveOwnerOrNull(sessionId)?.awaitOptionWritable?.(sessionId) ?? Promise.resolve() + awaitStarted = (sessionId: string): Promise => + this.liveOwnerOrNull(sessionId)?.awaitStarted?.(sessionId) ?? Promise.resolve() readOptions = (input: { sessionId: string; fence: number }) => { const reader = this.owner(input.sessionId).readOptions diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts index 9174afc7fd2..7a293193375 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts @@ -289,6 +289,10 @@ export type StructuredAgentSessionAdapter = { ): Promise>> /** Resolves once a live session can take an option write, or after a bound; never rejects. */ awaitOptionWritable?(sessionId: string): Promise + /** Resolves once a session published before it proved its start has proven it, failed, or been + * closed; at once for any other. A start that did not land resolves with the chat's words for + * why. Never rejects. */ + awaitStarted?(sessionId: string): Promise readOptions?(input: { sessionId: string; fence: number }): Promise /** Option keys skipped after a provider rejected their persisted restore value. */ readOptionRestoreFailures?(sessionId: string): readonly string[] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.test.ts index 0248799980a..4b8e1d87ba3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.test.ts @@ -11,10 +11,12 @@ import { attachFingerprintFields, type AgentSessionAttachParams } from './structured-agent-session-attach' +import { openTestAttachConversation } from './structured-agent-session-attach-test-conversation' import { performAttach, type AttachFlowInput } from './structured-agent-session-attach-flow' import { AgentSessionJournal } from '../agent-session-journal/journal-store' import { agentSessionJournalCloseRetries } from '../agent-session-journal/journal-close-retry' import * as legacyImport from '../agent-session-journal/journal-legacy-import' +import { StructuredAgentSessionHost } from './structured-agent-session-host' const NOW = 1_800_000_000_000 const SESSION = 'codex_adopting_session' @@ -121,6 +123,7 @@ async function attach( store, adapter: sessionAdapter, journalRoot: root!, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -210,7 +213,7 @@ describe('adopting a provider conversation on create', () => { } ) - it('still releases acquisition and closes the provisional journal on an import write failure', async () => { + it('still releases acquisition on an import write failure, and leaves the conversation open', async () => { root = await mkdtemp(join(tmpdir(), 'orca-adopt-write-failure-')) const transcriptPath = join(root, 'rollout.jsonl') await writeCodexRollout(transcriptPath, 'valid source') @@ -222,7 +225,51 @@ describe('adopting a provider conversation on create', () => { await expect(attach(transcriptPath, sessionAdapter)).rejects.toThrow('disk write failed') expect(sessionAdapter.acquire).toHaveBeenCalledTimes(1) expect(sessionAdapter.releaseAcquisition).toHaveBeenCalledTimes(1) - expect(close).toHaveBeenCalledTimes(1) + // The journal is the conversation's, not the attach's: a failed import closes nothing. + expect(close).not.toHaveBeenCalled() + }) + + it('leaves the conversation writable when the import fails after acquiring', async () => { + root = await mkdtemp(join(tmpdir(), 'orca-adopt-host-failure-')) + const transcriptPath = join(root, 'rollout.jsonl') + await writeCodexRollout(transcriptPath, 'valid source') + store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) + const host = new StructuredAgentSessionHost({ + store, + adapter: adapter(), + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-a', + now: () => NOW + }) + vi.spyOn(AgentSessionJournal.prototype, 'replaceEpochItems').mockRejectedValueOnce( + new Error('disk write failed') + ) + const attached = await host + .attach({ callerKey: 'client-1' }, attachParams(transcriptPath)) + .catch(() => null) + expect(attached?.ok).not.toBe(true) + + const body = { kind: 'message' as const, role: 'user' as const, blocks: [] } + const sent = await host.send( + { callerKey: 'client-1' }, + { + envelope: { + sessionId: SESSION, + clientOperationId: `${NOW}-${'2'.padStart(32, '0')}`, + expectedRuntimeFence: null, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + }, + body + } + ) + // The failed attach kept the conversation's own journal open, so the send is recorded. + expect(sent).toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) + await host.flushAllStreamedEvents() }) it('prepares a valid source once before acquisition and imports those exact items', async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.ts index 459d21b1f3b..7a85de6a473 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.ts @@ -1,7 +1,6 @@ import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionAttachParams, AttachedJournal } from './structured-agent-session-attach' -import { agentSessionJournalCloseRetries } from '../agent-session-journal/journal-close-retry' import type { JournalReplacementItem } from '../agent-session-journal/journal-epoch-replacement' import { importLegacyTranscriptIntoJournal, @@ -62,13 +61,8 @@ export async function importAdoptedTranscript( record: AgentSessionRecord, prepared: JournalReplacementItem[] | null ): Promise { - try { - await applyAdoptedTranscript(params, attached, record, prepared) - } catch (error) { - // Publication has not taken ownership of this provisional journal yet. - await agentSessionJournalCloseRetries.closeOrRetain(attached.journal) - throw error - } + // The journal is the conversation's, which outlives a failed import; nothing here closes it. + await applyAdoptedTranscript(params, attached, record, prepared) } async function applyAdoptedTranscript( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts index c8c908aae70..eef9297e82a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts @@ -50,8 +50,16 @@ function liveReader() { submissions.push(...event.page.submissions) } } + const rows = new Map() + for (const item of items) { + if (item.body.kind === 'status') { + rows.set(item.itemId, item.body.text) + } + } return { statuses: items.flatMap((item) => (item.body.kind === 'status' ? [item.body.text] : [])), + /** Each status row as the chat renders it: its latest revision, once. */ + statusRows: [...rows.values()], submissions, batches: events.slice(opened).filter((event) => event.type === 'batch').length } @@ -150,12 +158,16 @@ describe('an open chat receives every row its journal commits', () => { await exitBeforeProof() - expect(pane.received().statuses).toEqual([ + // The exit ends the child; the delivery loop, which reads why, rejects what it had queued. + await vi.waitFor(() => + expect(pane.received().submissions).toContainEqual( + expect.objectContaining({ clientMessageId: held, dispatchState: 'rejected' }) + ) + ) + // One row, however many of its writers reported the start. + expect(pane.received().statusRows).toEqual([ expect.stringMatching(/stopped before it finished starting: .*not signed in/) ]) - expect(pane.received().submissions).toContainEqual( - expect.objectContaining({ clientMessageId: held, dispatchState: 'rejected' }) - ) }) it('shows a revision the provider queued with no publish behind it', async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts index 4b9b46716cb..d6e5e9097a2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts @@ -15,6 +15,7 @@ import type { } from './structured-agent-session-host-types' import type { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' import type { StructuredAgentSessionTaskQueue } from './structured-agent-session-task-queue' +import type { StructuredAgentSessionConversationOpenOptions } from './structured-agent-session-conversation-open' export type StructuredAgentSessionAttachContext = { deps: StructuredAgentSessionHostDeps @@ -38,8 +39,12 @@ export type StructuredAgentSessionAttachContext = { reconcileLeases: (sessionId: string) => Promise serialize: (sessionId: string, task: () => Promise) => Promise now: () => number - /** Paired with `sessions.delete` by `forgetStructuredAgentSession`; a failed attach that only - * deleted would leave the store's row behind. */ + /** Paired with `sessions.delete` by `forgetStructuredAgentSession`, which a close runs. */ forgetStatus: (sessionId: string) => void publishStatus?: (sessionId: string) => void + /** The conversation's one open journal, opened when closed; see `conversation-open`. */ + openConversation: ( + sessionId: string, + options?: StructuredAgentSessionConversationOpenOptions + ) => Promise } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts index 3a77aa2d6cc..6af8eb369d5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts @@ -24,8 +24,7 @@ export async function settlePostAcquisitionAttachFailure( ? 'root-exit-observed' : 'exit-proven' } - // A failed close must not prevent durable failure settlement. - await Promise.resolve(input.onAttachFailed?.()).catch(() => undefined) + input.onAcquisitionReleased?.(cause, { rootGone: exitProof !== 'unproven' }) try { await input.store.settleFailedPostAcquisitionAttachment({ sessionId: record.sessionId, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts index cf37a9a6fe2..2969cb62f09 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts @@ -41,6 +41,7 @@ import { type AgentSessionCreatePhaseRecorder } from '../../observability/agent-session-instrumentation' import type { ProviderHistoryWindow } from '../agent-session-journal/journal-submission-reconciler' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' export type AttachFlowInput = { store: AgentSessionRecordStore @@ -66,8 +67,11 @@ export type AttachFlowInput = { onAcquiring?: () => Promise | void /** Settles writes already captured by the superseded journal before opening another. */ beforeJournalOpen?: () => Promise | void - /** Closes and removes partial publication after journal attachment fails. */ - onAttachFailed?: () => Promise + /** The conversation's own open journal, which the attach adopts: it never opens one itself. */ + openConversation: (record: AgentSessionRecord) => Promise + /** A failure after acquisition released the session's acquisition; `cause` is that failure and + * `rootGone` whether the release saw the provider root go. */ + onAcquisitionReleased?: (cause: unknown, verdict: { rootGone: boolean }) => void } export async function performAttach( @@ -208,6 +212,7 @@ export async function performAttach( params, journalRoot: input.journalRoot, adapter: input.adapter, + openConversation: input.openConversation, providerHistoryWindow }) await importAdoptedTranscript(params, attached, record, preparedTranscript.items) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts index bf62c3fffce..591c09d8e32 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts @@ -23,9 +23,16 @@ import { import { refuseAgentSessionMutation } from './structured-agent-session-mutation-admission' import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' -import { forgetStructuredAgentSession } from './structured-agent-session-host-lifetime' +import type { + StructuredAgentSessionProviderChild, + StructuredAgentSessionStopVerdict +} from './structured-agent-session-host-types' +import { + endProviderChild, + indexProviderChild, + structuredAgentSessionConversationFence +} from './structured-agent-session-provider-child' import type { DeferredStructuredAgentSessionEventSink } from './structured-agent-session-event-sink' -import { agentSessionJournalCloseRetries } from '../agent-session-journal/journal-close-retry' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { addAgentSessionCreatePhaseAttributes, @@ -94,6 +101,10 @@ async function runAttach( ): Promise> { const sessionId = params.envelope.sessionId const recordPhase = options.recordPhase + // Readers of a conversation already open are re-baselined when this attach moves its fence. + const fenceBefore = context.sessions.has(sessionId) + ? structuredAgentSessionConversationFence(context.deps.store, sessionId) + : null if (options.admitRecoveryTicket && !options.admitRecoveryTicket()) { return refuseAgentSessionMutation({ code: 'agent_session_checkpoint_stale', @@ -113,17 +124,18 @@ async function runAttach( context.runtimeState.probeOwner(sessionId) ) // A child this attach spawns writes through a sink this attempt owns. Only a successful - // attach makes it the session's; any other exit closes it with whatever the child queued. + // attach makes the child and its sink the session's; any other exit closes the sink with + // whatever the child queued, and leaves the conversation's child as it was. const attemptSink = context.runtimeState.mintEventSink(sessionId) - let attemptSinkAdopted = false // Read before the reserve clears it: how the previous generation ended decides how whatever it // left running is settled. const priorDeathEvidence = context.deps.store.getRecord(sessionId)?.lease.deathEvidence ?? null - const attached = stampFailedCreateOwnerVerdict( - context.deps.store, - callerKey, - params.envelope, - await performAttach({ + const attempt: { candidate: AttachCandidate | null; committed: boolean } = { + candidate: null, + committed: false + } + try { + const attached = await performAttach({ store: context.deps.store, adapter: context.deps.adapter, journalRoot: context.deps.journalRoot, @@ -147,66 +159,47 @@ async function runAttach( params, now: () => context.now(), recordPhase, - // Site 9: this closes the PRIOR map entry it drops, never the provisional - // journal — it has no reference to that one. `onAttached` owns that. - onAttachFailed: async () => { - await forgetStructuredAgentSession(context, sessionId) - context.runtimeState.currentEventSink(sessionId)?.close() - context.runtimeState.discardEventSink(sessionId) + openConversation: async (record) => { + const conversation = await context.openConversation(record.sessionId, { + acquisition: true + }) + if (!conversation) { + throw new Error('agent_session_identity_required') + } + return conversation.journal }, + // The cleanup released the acquisition, which for a re-attach is the live child itself. + onAcquisitionReleased: (cause, verdict) => + endReleasedChild(context, sessionId, cause, verdict), onAttached: async (attached, acquisitionGeneration, acquiredOwner, providerChildPhase) => { - const fence = context.deps.store.getRecord(sessionId)?.lease.runtimeFence ?? 0 - const previous = context.sessions.get(sessionId) - const previousFence = previous?.fence + const fence = structuredAgentSessionConversationFence(context.deps.store, sessionId) + const current = context.sessions.get(sessionId)?.child ?? null // A re-attach to a live child keeps the sink that child already writes through. const eventSink = acquiredOwner ? attemptSink : (context.runtimeState.currentEventSink(sessionId) ?? attemptSink) - // Site 8: the provisional journal has no owner until the map takes it, - // and the barrier below throws by design. - try { - if (acquiredOwner) { - // Before the drain: the buffered events are the new child's, never a stale row's. - await settleStaleStructuredAgentSessionState({ - journal: attached.journal, - sessionId, - fence, - acquisitionGeneration, - deathEvidence: priorDeathEvidence - }) - } - await bindAndDrain(eventSink, attached.journal, fence, (activity) => - context.subscribers.publish(sessionId, attached.journal, activity) - ) - } catch (error) { - await agentSessionJournalCloseRetries.closeOrRetain(attached.journal) - throw error + if (acquiredOwner) { + // Before the drain: the buffered events are the new child's, never a stale row's. + await settleStaleStructuredAgentSessionState({ + journal: attached.journal, + sessionId, + fence, + acquisitionGeneration, + deathEvidence: priorDeathEvidence + }) } - // Site 10: a `set` over a live entry would orphan its handle — and a - // close that REJECTED did not release it. The replacement is therefore - // ABORTED rather than completed over a handle nothing can reach again: - // `previous` stays indexed, so teardown still owns it and can retry. - if (previous && previous.journal !== attached.journal) { - try { - await previous.journal.close() - } catch (error) { - await agentSessionJournalCloseRetries.closeOrRetain(attached.journal) - throw error + await bindAndDrain(eventSink, attached.journal, fence, (activity) => + context.subscribers.publish(sessionId, attached.journal, activity) + ) + attempt.candidate = { + sink: eventSink, + child: { + generation: acquisitionGeneration ?? current?.generation ?? null, + fence, + // A re-attach to a live child keeps what that child already proved. + phase: acquiredOwner ? providerChildPhase : (current?.phase ?? 'ready') } } - context.runtimeState.adoptEventSink(sessionId, eventSink) - attemptSinkAdopted = eventSink === attemptSink - context.sessions.set(sessionId, { - journal: attached.journal, - params, - fence, - hasProviderChild: true, - // A re-attach to a live child keeps what that child already proved. - providerChildPhase: acquiredOwner - ? providerChildPhase - : (previous?.providerChildPhase ?? 'ready'), - acquisitionGeneration: acquisitionGeneration ?? previous?.acquisitionGeneration ?? null - }) await recoverStructuredRewind( context.deps.store, sessionId, @@ -216,21 +209,59 @@ async function runAttach( context.now ) await recoverInterruptedCompaction(context.deps.store, sessionId, attached.journal, fence) - if (attached.recovery) { - context.subscribers.reset(sessionId, attached.journal, attached.recovery.reset, fence) - } else if (previousFence !== undefined && previousFence !== fence) { + if (fenceBefore !== null && fence !== fenceBefore) { context.subscribers.snapshot(sessionId, attached.journal, fence) } else { context.subscribers.publish(sessionId, attached.journal) } } - }).finally(() => { - if (!attemptSinkAdopted) { - attemptSink.close() - } }) - ) - return attached + const { candidate } = attempt + const conversation = context.sessions.get(sessionId) + if (attached.ok && candidate && conversation) { + context.runtimeState.adoptEventSink(sessionId, candidate.sink) + attempt.committed = candidate.sink === attemptSink + indexProviderChild(conversation, candidate.child) + context.publishStatus?.(sessionId) + } + return stampFailedCreateOwnerVerdict(context.deps.store, callerKey, params.envelope, attached) + } finally { + if (!attempt.committed) { + attemptSink.close() + } + } +} + +type AttachCandidate = { + child: StructuredAgentSessionProviderChild + sink: DeferredStructuredAgentSessionEventSink +} + +function endReleasedChild( + context: StructuredAgentSessionAttachContext, + sessionId: string, + cause: unknown, + verdict: StructuredAgentSessionStopVerdict +): void { + const session = context.sessions.get(sessionId) + const child = session?.child + if ( + !session || + !child || + !endProviderChild(session, { + generation: child.generation, + fence: child.fence, + cause: 'attach-failed', + reason: cause instanceof Error ? cause.message : String(cause), + duringStartup: child.phase === 'starting', + ...verdict + }) + ) { + return + } + context.runtimeState.currentEventSink(sessionId)?.close() + context.runtimeState.discardEventSink(sessionId) + context.publishStatus?.(sessionId) } /** Binds the sink to the journal and waits for the barrier the host publishes diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts index 97f429933d8..4e15b23d4d3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts @@ -13,6 +13,7 @@ import { journalDirectoryFor } from '../agent-session-journal/journal-paths' import type { ProviderHistoryWindow } from '../agent-session-journal/journal-submission-reconciler' import { createTrackedJournalOpener } from '../agent-session-journal/journal-store-test-open' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { openTestAttachConversation } from './structured-agent-session-attach-test-conversation' import { attachJournal, journalIdentityFor, @@ -85,6 +86,7 @@ async function attach(adapter: StructuredAgentSessionAdapter) { record: RECORD, params: PARAMS, journalRoot: root, + openConversation: openTestAttachConversation(root), adapter }) journals.track(attached.journal) @@ -147,4 +149,37 @@ describe('attachJournal restart reconciliation', () => { expect(attached.unconfirmedClientMessageIds).toEqual(['cm_1']) expect(attached.journal.submissions()[0]?.dispatchState).toBe('unknown') }) + + it('leaves a message the open conversation still has queued alone (W4′e)', async () => { + const journal = await journals.open({ + identity: IDENTITY, + journalDir: journalDirectoryFor(root, { + workspaceId: IDENTITY.workspaceId, + sessionId: IDENTITY.sessionId + }) + }) + await journal.appendSubmission({ + clientMessageId: 'queued', + payloadFingerprint: digestPayload('still queued'), + body: userMessage('still queued'), + fence: RECORD.lease.runtimeFence, + handoverRecorded: true + }) + // History that holds nothing: absence would prove a handed-over message undelivered. + const { adapter, dispatch } = adapterWith(async () => window()) + + const attached = await attachJournal({ + record: RECORD, + params: PARAMS, + journalRoot: root, + + adapter, + openConversation: async () => journal + }) + + expect(attached.journal).toBe(journal) + expect(journal.submissions()[0]).toMatchObject({ dispatchState: 'pending' }) + expect(journal.submissions()[0]?.handedOverAt).toBeUndefined() + expect(dispatch).not.toHaveBeenCalled() + }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-test-conversation.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-test-conversation.ts new file mode 100644 index 00000000000..1e2322cbc6d --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-test-conversation.ts @@ -0,0 +1,18 @@ +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { openStructuredAgentSessionConversationJournal } from './structured-agent-session-conversation-open' + +/** For a test that attaches without a host: the conversation's journal, through the one open a + * host would take, so the attach under test adopts it the way it adopts the host's. */ +export function openTestAttachConversation( + journalRoot: string, + adapter: Pick = {} +): (record: AgentSessionRecord) => Promise { + return async (record) => + ( + await openStructuredAgentSessionConversationJournal({ journalRoot, adapter }, record, { + acquisition: true + }) + ).session.journal +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach.ts index de5fa297daf..4cf6505d276 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach.ts @@ -35,15 +35,9 @@ import { } from '../../../shared/agent-session-mutation-envelope' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { agentSessionProviderHandleChainHead } from '../../../shared/agent-session-provider-handle' -import { agentSessionJournalCloseRetries } from '../agent-session-journal/journal-close-retry' -import { journalDirectoryFor } from '../agent-session-journal/journal-paths' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { reconcileJournalSubmissionsAgainstHistory } from '../agent-session-journal/journal-restart-reconciliation' import type { ProviderHistoryWindow } from '../agent-session-journal/journal-submission-reconciler' -import { - openAgentSessionJournalWithRecovery, - type AgentSessionJournalRecovery -} from './agent-session-journal-recovery' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import { structuredAgentSessionRefusalMessage } from './structured-agent-session-refusal-message' @@ -165,16 +159,13 @@ export function journalIdentityFor( export type AttachedJournal = { journal: AgentSessionJournal - recovery: AgentSessionJournalRecovery | null - /** Submissions still `unknown` after this open: the crash boundary settled - * them there and provider history could not decide them either. */ + /** Submissions the crash boundary left `unknown` that provider history could not decide. */ unconfirmedClientMessageIds: string[] } /** - * Open the session's journal, recovering it when the stored one is unusable, - * settle every submission left in flight by a previous process, then let - * provider history decide the ones it can prove. + * The conversation's journal — opened, and its crash boundary settled, by the conversation's own + * open — with provider history deciding the submissions that boundary could only doubt. * * Why the reconciliation belongs HERE and nowhere else: this runs after the * record store handed this host the lease and before `onAttached` starts a @@ -182,54 +173,44 @@ export type AttachedJournal = { * is read, and the window stays valid until the resume consumes it. Every other * settlement site — a proven child exit — runs while the host * may still start another child, and a read there could be overtaken before it - * is acted on. Orca still never re-sends: this decides state only. + * is acted on. Orca still never re-sends: this decides state only. A queued + * submission is left alone: it was never handed over, so history cannot hold it. */ export async function attachJournal(input: { record: AgentSessionRecord params: AgentSessionAttachParams journalRoot: string adapter: StructuredAgentSessionAdapter + /** The host's open conversation, whose journal the attach adopts. */ + openConversation: (record: AgentSessionRecord) => Promise /** Provider history sampled before a new child is acquired. `null` means the * adapter had no usable history; omit to read lazily for direct callers. */ providerHistoryWindow?: ProviderHistoryWindow | null }): Promise { const identity = journalIdentityFor(input.record, input.params) const fence = input.record.lease.runtimeFence - const historyFilePath = input.adapter.historyFilePath - ? await input.adapter.historyFilePath({ identity }) - : null - const opened = await openAgentSessionJournalWithRecovery({ + const journal = await input.openConversation(input.record) + const settled = await reconcileAgainstProviderHistory({ + adapter: input.adapter, identity, - journalDir: journalDirectoryFor(input.journalRoot, { - workspaceId: identity.workspaceId, - sessionId: identity.sessionId - }), + journal, fence, - historyFilePath + accountHome: input.record.accountHome, + ...(Object.hasOwn(input, 'providerHistoryWindow') + ? { history: input.providerHistoryWindow } + : {}) }) - try { - // That await is a WRITE. A failure in it leaves the journal with no caller - // holding a reference to close it. - const unconfirmed = await opened.journal.markPendingSubmissionsUnknown(fence) - const settled = await reconcileAgainstProviderHistory({ - adapter: input.adapter, - identity, - journal: opened.journal, - fence, - accountHome: input.record.accountHome, - ...(Object.hasOwn(input, 'providerHistoryWindow') - ? { history: input.providerHistoryWindow } - : {}) - }) - return { - ...opened, - unconfirmedClientMessageIds: unconfirmed.filter((id) => !settled.includes(id)) - } - } catch (error) { - // A rejected close leaves the handle open, so the journal is retained for a - // later retry rather than dropped along with the only reference to it. - await agentSessionJournalCloseRetries.closeOrRetain(opened.journal) - throw error + return { + journal, + unconfirmedClientMessageIds: journal + .submissions() + .filter( + (entry) => + entry.dispatchState === 'unknown' && + entry.recovered === true && + !settled.includes(entry.clientMessageId) + ) + .map((entry) => entry.clientMessageId) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts index 5d922d72350..8594b6d58ba 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts @@ -12,6 +12,7 @@ import type { StructuredAgentSessionHostDeps, StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' export class StructuredAgentSessionBackgroundTaskChannel { constructor( @@ -48,7 +49,7 @@ export class StructuredAgentSessionBackgroundTaskChannel { return this.subscribers.open({ ...input, journal: session.journal, - fence: this.deps.store.getRecord(input.sessionId)?.lease.runtimeFence ?? 0, + fence: structuredAgentSessionConversationFence(this.deps.store, input.sessionId), ...(backgroundTasks !== undefined ? { backgroundTasks } : {}) }) } @@ -57,7 +58,11 @@ export class StructuredAgentSessionBackgroundTaskChannel { const session = this.sessions.get(sessionId) const state = publishedState !== undefined ? publishedState : this.state(sessionId) if (session && state !== undefined) { - this.subscribers.backgroundTasks(sessionId, state, session.fence) + this.subscribers.backgroundTasks( + sessionId, + state, + structuredAgentSessionConversationFence(this.deps.store, sessionId) + ) this.onPublished(sessionId) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts index a73e46b898e..537a2ac695f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts @@ -103,10 +103,11 @@ describe('Claude root-exit eviction', () => { { journal, params, - fence, - hasProviderChild: true, - providerChildPhase: 'ready', - acquisitionGeneration: acquisition.acquisitionGeneration ?? null + child: { + generation: acquisition.acquisitionGeneration ?? null, + fence, + phase: 'ready' + } } ] ]) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-close-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-close-retry.test.ts index b9e8f8e506d..303ecbe92fc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-close-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-close-retry.test.ts @@ -78,10 +78,7 @@ function hostSession(journal: AgentSessionJournal): StructuredAgentSessionHostSe return { journal, params: {} as StructuredAgentSessionHostSession['params'], - fence: 1, - hasProviderChild: false, - providerChildPhase: 'ready', - acquisitionGeneration: null + child: null } } @@ -178,40 +175,34 @@ describe('the registry', () => { }) describe('the attach orchestration', () => { - it('ABORTS the map replacement when the previous journal will not close', async () => { - const previousDir = join(root, 'previous') - const provisionalDir = join(root, 'provisional') - const previous = flakyClose( - await journals.open({ identity: IDENTITY, journalDir: previousDir }), - 1 - ) - const provisional = await journals.open({ - identity: IDENTITY, - journalDir: provisionalDir - }) - attachFlow.journal = provisional - const sessions = new Map([[SESSION, hostSession(previous)]]) + // The attach adopts the conversation's one open journal; it never opens a second handle, so + // there is no replacement to abort and no provisional journal to close. + it('keeps the journal it adopted indexed and open when an attach succeeds', async () => { + const directory = join(root, 'adopted') + const journal = await journals.open({ identity: IDENTITY, journalDir: directory }) + attachFlow.journal = journal + const sessions = new Map([[SESSION, hostSession(journal)]]) + await attachStructuredAgentSession(attachContext(sessions), 'caller-1', attachParams) + + expect(sessions.get(SESSION)?.journal).toBe(journal) await expect( - attachStructuredAgentSession(attachContext(sessions), 'caller-1', attachParams) - ).rejects.toThrow('close rejected') - - // The live entry is UNTOUCHED: overwriting it would have left its handle - // open with nothing able to reach it again. - expect(sessions.get(SESSION)?.journal).toBe(previous) - // And the provisional journal is owned by the registry, not orphaned. + journal.appendItem( + { provider: 'orca', clientMessageId: 'after-attach' }, + { + kind: 'status', + text: 'still writable' + } + ) + ).resolves.toBeDefined() expect(agentSessionJournalCloseRetries.pendingDirectories).toEqual([]) - await expectNothingHoldsTheDirectory(provisionalDir) }) - it('retains the provisional journal when its own close rejects on the barrier path', async () => { - const provisionalDir = join(root, 'provisional-barrier') - const provisional = flakyClose( - await journals.open({ identity: IDENTITY, journalDir: provisionalDir }), - 1 - ) - attachFlow.journal = provisional - const sessions = new Map() + it('leaves the conversation indexed and open when the sink barrier fails', async () => { + const directory = join(root, 'adopted-barrier') + const journal = await journals.open({ identity: IDENTITY, journalDir: directory }) + attachFlow.journal = journal + const sessions = new Map([[SESSION, hostSession(journal)]]) const context = attachContext(sessions) const failing = { sink: {}, @@ -229,9 +220,19 @@ describe('the attach orchestration', () => { 'sink barrier failed' ) - expect(sessions.size).toBe(0) - // Retained rather than dropped, so teardown can still release the handle. - expect(agentSessionJournalCloseRetries.pendingDirectories).toEqual([provisionalDir]) + // A failed attach does not end the conversation: its queued messages and the failure row + // are written into this same journal. + expect(sessions.get(SESSION)?.journal).toBe(journal) + await expect( + journal.appendItem( + { provider: 'orca', clientMessageId: 'after-failure' }, + { + kind: 'status', + text: 'still writable' + } + ) + ).resolves.toBeDefined() + expect(agentSessionJournalCloseRetries.pendingDirectories).toEqual([]) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts new file mode 100644 index 00000000000..07d1fb6b5ff --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts @@ -0,0 +1,159 @@ +// The one way a conversation's journal becomes open on this host: for a send, for a reader, and +// for an attach that finds none open. +// +// It opens with recovery, so an unusable journal is rebuilt rather than refused, and it marks what +// an earlier host process handed over and left unanswered as in doubt, and settles what it left +// running — the crash boundary. That +// needs no lease: provider history decides such a row later, under a won lease, in the attach. A +// row an earlier process accepted and never handed over is the delivery loop's, which the open +// wakes. Nothing here starts a provider child. + +import type { AgentJournalResetReason } from '../../../shared/agent-session-journal-types' +import { journalDirectoryFor } from '../agent-session-journal/journal-paths' +import { openAgentSessionJournalWithRecovery } from './agent-session-journal-recovery' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { + attachFingerprintFields, + journalIdentityFor, + type AgentSessionAttachParams +} from './structured-agent-session-attach' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' +import type { + StructuredAgentSessionHostDeps, + StructuredAgentSessionHostSession +} from './structured-agent-session-host-types' + +export type OpenedStructuredAgentSessionConversation = { + session: StructuredAgentSessionHostSession + /** Set when the journal was rebuilt on the way; readers reload from a snapshot. */ + reset: AgentJournalResetReason | null +} + +export type StructuredAgentSessionConversationOpenDeps = { + store: Pick + adapter: Pick + journalRoot: string + onEventSinkError?: StructuredAgentSessionHostDeps['onEventSinkError'] +} + +/** An acquisition's own open: its reserve cleared the record's death evidence, so it settles + * what the gone generation left running itself, from what it read before. */ +export type StructuredAgentSessionConversationOpenOptions = { acquisition?: boolean } + +export type StructuredAgentSessionConversationOpenContext = { + deps: StructuredAgentSessionConversationOpenDeps + sessions: Map + /** Indexes a conversation that just became open; the host publishes it and wakes delivery. */ + adoptOpened: ( + sessionId: string, + opened: OpenedStructuredAgentSessionConversation + ) => Promise +} + +/** The open conversation, or null when this host has no record of it. For a caller inside the + * session's serialize, which is what makes "not open yet" exact. */ +export async function openStructuredAgentSessionConversation( + context: StructuredAgentSessionConversationOpenContext, + sessionId: string, + options: StructuredAgentSessionConversationOpenOptions = {} +): Promise { + const open = context.sessions.get(sessionId) + if (open) { + return open + } + const record = context.deps.store.getRecord(sessionId) + if (!record) { + return null + } + const opened = await openStructuredAgentSessionConversationJournal(context.deps, record, options) + await context.adoptOpened(sessionId, opened) + return opened.session +} + +/** The open itself, indexed by nobody yet: the caller adopts the result. */ +export async function openStructuredAgentSessionConversationJournal( + deps: Omit, + record: AgentSessionRecord, + options: StructuredAgentSessionConversationOpenOptions = {} +): Promise { + const { sessionId } = record + const fence = record.lease.runtimeFence + const params = attachParamsForRecord(record, { + clientOperationId: `read-restore:${sessionId}`, + expectedRuntimeFence: fence + }) + const identity = journalIdentityFor(record, params) + const opened = await openAgentSessionJournalWithRecovery({ + identity, + journalDir: journalDirectoryFor(deps.journalRoot, { + workspaceId: record.location.workspaceId, + sessionId + }), + fence, + historyFilePath: (await deps.adapter.historyFilePath?.({ identity })) ?? null + }) + try { + // A queued row found here is a leftover the delivery loop's first step rejects; a handed-over + // one is only doubt, which provider history decides under a won lease. + await opened.journal.markPendingSubmissionsUnknown(fence) + } catch (error) { + deps.onEventSinkError?.({ sessionId, error }) + } + try { + // No child in this process writes to a journal nobody had open, so whatever it shows running + // belongs to a generation that is gone, whatever the lease still claims. Settled before any + // reader or child sees it. + if (!options.acquisition) { + await settleStaleStructuredAgentSessionState({ + journal: opened.journal, + sessionId, + fence, + acquisitionGeneration: null, + deathEvidence: record.lease.deathEvidence ?? null + }) + } + } catch (error) { + // Best effort: the next acquire re-derives it. + deps.onEventSinkError?.({ sessionId, error }) + } + return { + session: { journal: opened.journal, params, child: null }, + reset: opened.recovery?.reset ?? null + } +} + +export function attachParamsForRecord( + record: AgentSessionRecord, + input: { + clientOperationId: string + expectedRuntimeFence: number + } +): AgentSessionAttachParams { + const params: AgentSessionAttachParams = { + envelope: { + sessionId: record.sessionId, + clientOperationId: input.clientOperationId, + expectedRuntimeFence: input.expectedRuntimeFence, + payloadFingerprint: '' + }, + location: record.location, + provider: record.provider, + agent: record.provider, + accountHome: record.accountHome, + runtimeKind: 'native' + } + return { + ...params, + envelope: { + ...params.envelope, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.attach', + sessionId: record.sessionId, + fields: attachFingerprintFields(params) + }) + } + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts index 202ffe50ae4..7014e394728 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts @@ -37,10 +37,7 @@ function session(journal: AgentSessionJournal) { return { journal, params: hostTestAttachParams(null), - fence: 0, - hasProviderChild: false, - providerChildPhase: 'ready' as const, - acquisitionGeneration: null + child: null } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts index d553fbaecf3..8ad51871c24 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts @@ -190,6 +190,7 @@ describe('dead structured-session generation settlement', () => { | 'submissions' | 'markPendingSubmissionsUnknown' | 'rejectPendingSubmissions' + | 'rejectQueuedSubmissions' | 'appendLifecycleBatch' > = { snapshot: () => ({ @@ -203,6 +204,9 @@ describe('dead structured-session generation settlement', () => { rejectPendingSubmissions: async () => { throw new Error('journal_closed') }, + rejectQueuedSubmissions: async () => { + throw new Error('journal_closed') + }, appendLifecycleBatch: async () => { throw new Error('journal_closed') } @@ -273,7 +277,7 @@ describe('dead structured-session generation settlement', () => { pendingSubmissionReason: 'provider_closed_before_acknowledgement', verdict: { state: 'interrupted', completedAt: 1_000 }, unexpectedExitReason: 'claude stream-json exited (code 1): not signed in', - exitedDuringStartup: true + exitedDuringStartup: { generation: 'generation-1' } }) const reason = diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index 05f7b4a7fe8..9d20d93a82f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -11,6 +11,7 @@ import { cancelledJournalPromptBody } from '../agent-session-journal/journal-prompt-body-bounds' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { structuredAgentSessionStartFailureRow } from './structured-agent-session-start-failure-row' import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' import { runningTurnLifecycleRevisions, @@ -57,6 +58,14 @@ export function providerStartupFailureOutcome(reason?: string): string { : 'The provider stopped before it finished starting.' } +/** Why a message accepted but not yet handed over was rejected when its child exited. */ +export function providerExitBeforeDeliveryRejection(reason?: string): string { + const detail = exitReasonDetail(reason) + return detail + ? `The provider stopped before this message was sent: ${detail}.` + : 'The provider stopped before this message was sent.' +} + /** Why a send a child that never started left unwritten was rejected. The child's own diagnostic is * the cause the user can act on, so it is the reason, in the words the chat row uses. */ export function providerStartupFailureRejection(cause?: unknown): string { @@ -74,7 +83,7 @@ function exitReasonDetail(reason: string | undefined): string | undefined { type DeadGenerationSubmission = Pick< ReturnType[number], - 'clientMessageId' | 'dispatchState' | 'recovered' + 'clientMessageId' | 'dispatchState' | 'recovered' | 'handoverRecorded' | 'handedOverAt' > export type DeadGenerationJournal = { @@ -144,8 +153,9 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { showUnexpectedExitOutcome?: boolean /** Why the provider stopped, when the host has it. Rendered with the outcome copy. */ unexpectedExitReason?: string - /** The provider never finished starting; the outcome says so instead of naming a response. */ - exitedDuringStartup?: boolean + /** The provider never finished starting: the start that failed, keyed by the child's + * generation. Its row is the one the delivery loop writes for the same start. */ + exitedDuringStartup?: { generation: string | null } onError?: (sessionId: string, error: unknown) => void }): Promise { try { @@ -154,9 +164,10 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { if (!showUnexpectedExitOutcome && !hasUnfinishedWork) { return true } - // A child that never proved its start accepted nothing — input is written only after it - // initializes — so every send it left unanswered is provably unwritten and is rejected with the - // child's own diagnostic. A proven child's unanswered sends stay in doubt. + // A queued message is the delivery loop's to settle: it was never handed to this child. A + // child that never proved its start accepted nothing either — input is written only after it + // initializes — so every send it was handed is rejected with the child's own diagnostic. A + // proven child's handed-over sends stay in doubt. await (input.exitedDuringStartup ? input.journal.rejectPendingSubmissions( input.fence, @@ -165,17 +176,21 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { : input.journal.markPendingSubmissionsUnknown(input.fence, input.pendingSubmissionReason)) const items = input.journal.snapshot().items const mutations: JournalLifecycleMutationInput[] = [] - if (showUnexpectedExitOutcome) { + if (showUnexpectedExitOutcome && input.exitedDuringStartup) { + // Until views stop starting children, a start can die with nothing queued for the loop. + mutations.push( + structuredAgentSessionStartFailureRow( + input.exitedDuringStartup.generation ?? input.settlementId, + providerStartupFailureOutcome(input.unexpectedExitReason) + ) + ) + } else if (showUnexpectedExitOutcome) { mutations.push({ kind: 'item', identity: { provider: 'orca', clientMessageId: input.settlementId }, body: { kind: 'status', - text: boundJournalStatusText( - input.exitedDuringStartup - ? providerStartupFailureOutcome(input.unexpectedExitReason) - : unexpectedProviderExitOutcome(input.unexpectedExitReason) - ) + text: boundJournalStatusText(unexpectedProviderExitOutcome(input.unexpectedExitReason)) } }) } @@ -298,7 +313,9 @@ function hasUnsettledSubmission(journal: DeadGenerationJournal): boolean { return submissions ? submissions.some( (submission) => - submission.dispatchState === 'pending' || + // A queued message is not work in progress: nothing has it yet. + (submission.dispatchState === 'pending' && + !(submission.handoverRecorded && submission.handedOverAt === undefined)) || (submission.dispatchState === 'unknown' && submission.recovered !== true) ) : (journal.pendingSubmissions?.().length ?? 0) > 0 diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts new file mode 100644 index 00000000000..e12acb3a12e --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts @@ -0,0 +1,248 @@ +// The one thing that starts a provider child for a send, the one thing that hands a message to +// it, and the one thing that settles a queued message because of a start, a child or a leftover. +// +// A send is accepted on its own serialized step and returns; this loop does the rest. It exists +// for a session exactly while a message is queued there — accepted, not yet handed over — and +// every step re-reads the journal and the conversation's child record to decide, so there is no +// loop state to disagree with them. Each step is its own serialized task. That is what lets a Stop +// that arrives while a start holds the queue withdraw the queued messages before the handover that +// would have written them. Stop and the conversation's close are the only other writers of a +// queued message: a child's exit only ends the child, and this loop reads why. + +import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' +import { DISPATCH_REJECTED_HOST_RESTARTED } from '../../../shared/structured-agent-session-dispatch-rejection' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { + providerExitBeforeDeliveryRejection, + providerStartupFailureOutcome +} from './structured-agent-session-dead-generation-settlement' +import type { StructuredAgentSessionResumeOutcome } from './structured-agent-session-hold-resume' +import type { + StructuredAgentSessionEndedChild, + StructuredAgentSessionHostSession, + StructuredAgentSessionProviderChildIdentity +} from './structured-agent-session-host-types' +import { + oldestQueuedSubmission, + recordStructuredAgentSessionStartFailure +} from './structured-agent-session-start-failure-row' +import { failedProviderChildStart } from './structured-agent-session-provider-child' +import { handOverSubmission } from './structured-agent-session-turns' + +export type StructuredAgentSessionDeliveryLoopDeps = { + sessions: ReadonlyMap + adapter: StructuredAgentSessionAdapter + serialize: (sessionId: string, task: () => Promise) => Promise + /** A start step, tracked from enqueue so quit waits for the child it may produce. */ + trackStart: (start: Promise) => Promise + /** Gives the session a provider child if it has none; for a caller inside `serialize`. */ + ensureProviderChild: (sessionId: string) => Promise + /** The fence the conversation's own writes carry; see `structuredAgentSessionConversationFence`. */ + conversationFence: (sessionId: string) => number + /** What the chat says when the session could not be made ready. */ + startFailureText: (sessionId: string, cause: AgentSessionWireRefusal) => string + onError: (sessionId: string, error: unknown) => void +} + +type Step = 'continue' | 'stop' + +type Prepared = + | 'stop' + | { ok: false; refusal: AgentSessionWireRefusal } + | { ok: true; awaited: StructuredAgentSessionProviderChildIdentity | null } + +type StartFailure = { startKey: string | null; text: string } + +export class StructuredAgentSessionDeliveryLoop { + private readonly running = new Set() + private disposed = false + + constructor(private readonly deps: StructuredAgentSessionDeliveryLoopDeps) {} + + isRunning(sessionId: string): boolean { + return this.running.has(sessionId) + } + + /** Quit: no step after this one starts a child or hands a message over. */ + dispose(): void { + this.disposed = true + } + + /** From inside the session's serialize, after a message was accepted or the conversation + * opened. A loop already running re-reads the journal on its next step. */ + wake(sessionId: string): void { + if (this.disposed || this.running.has(sessionId)) { + return + } + this.running.add(sessionId) + void this.run(sessionId) + } + + private async run(sessionId: string): Promise { + try { + for (;;) { + const prepared = await this.deps.trackStart( + this.deps.serialize(sessionId, () => this.prepare(sessionId)) + ) + if (prepared === 'stop') { + return + } + if (!prepared.ok) { + const text = this.deps.startFailureText(sessionId, prepared.refusal) + await this.deps.serialize(sessionId, () => this.fail(sessionId, { startKey: null, text })) + return + } + // A child published before it proved its start takes no input yet; waited for outside + // the queue so a Stop can reach it meanwhile. + const failure = await this.deps.adapter.awaitStarted?.(sessionId) + const handed = await this.deps.serialize(sessionId, () => + this.handOver(sessionId, prepared.awaited, failure || null) + ) + if (handed === 'stop') { + return + } + } + } catch (error) { + this.deps.onError(sessionId, error) + const text = this.deps.startFailureText(sessionId, { + code: 'agent_session_owner_restart_failed', + message: error instanceof Error ? error.message : String(error) + }) + await this.deps + .serialize(sessionId, () => this.fail(sessionId, { startKey: null, text })) + .catch((failure: unknown) => { + // Rows left queued are rejected by the next open, or by the next loop an accept wakes. + this.running.delete(sessionId) + this.deps.onError(sessionId, failure) + }) + } + } + + /** Settles what an earlier host process left queued, then makes the session ready. */ + private async prepare(sessionId: string): Promise { + const session = this.deps.sessions.get(sessionId) + if (!session || this.disposed) { + return this.stop(sessionId) + } + await session.journal.rejectQueuedSubmissions( + this.deps.conversationFence(sessionId), + DISPATCH_REJECTED_HOST_RESTARTED, + // A handle closes only with nothing queued, so one an earlier handle wrote is a leftover. + (submission) => session.journal.wroteBeforeOpen(submission.acceptedSequence) + ) + const oldest = oldestQueuedSubmission(session) + if (!oldest) { + return this.stop(sessionId) + } + const failedStart = startThatFailedWhileQueued(session, oldest) + if (failedStart) { + return this.fail(sessionId, failedStart) + } + const ready = await this.deps.ensureProviderChild(sessionId) + if (!ready.ok) { + return ready + } + const child = this.deps.sessions.get(sessionId)?.child + // The child this run waits on; handover checks it is still the one there. + return { + ok: true, + awaited: child ? { generation: child.generation, fence: child.fence } : null + } + } + + private async handOver( + sessionId: string, + awaited: StructuredAgentSessionProviderChildIdentity | null, + startFailure: string | null + ): Promise { + const session = this.deps.sessions.get(sessionId) + if (!session || this.disposed) { + return this.stop(sessionId) + } + // Re-derived here, not carried from the start: the child may have ended, or another may have + // taken its place, since. + const { child } = session + const awaitedChild = + child && awaited && child.generation === awaited.generation && child.fence === awaited.fence + ? child + : null + // The host's `starting` trails the adapter's `started` by one serialized step, so for the child + // waited on, the adapter's own answer decides whether its start landed. + if (!awaitedChild || (awaitedChild.phase === 'starting' && startFailure !== null)) { + // The child waited on is gone, replaced by another, or settled its start without proving it. + const ended = awaitedChild ? undefined : session.lastEndedChild + // A user's Stop is not a failure: the next step starts, or waits on, a child for what is + // queued. A host stop is: its cause is why the start did not land. + if (ended?.cause === 'user-stop') { + return 'continue' + } + return this.fail(sessionId, { + startKey: awaited?.generation ?? null, + text: ended ? endedChildRejection(ended) : (startFailure ?? providerStartupFailureOutcome()) + }) + } + const next = oldestQueuedSubmission(session) + if (!next) { + return this.stop(sessionId) + } + await handOverSubmission( + { + sessionId, + journal: session.journal, + fence: awaitedChild.fence, + adapter: this.deps.adapter, + providerChildPhase: () => this.deps.sessions.get(sessionId)?.child?.phase + }, + next + ) + return 'continue' + } + + private async fail(sessionId: string, failure: StartFailure): Promise<'stop'> { + const session = this.deps.sessions.get(sessionId) + if (session) { + await recordStructuredAgentSessionStartFailure( + { journal: session.journal, fence: this.deps.conversationFence(sessionId) }, + failure + ) + } + return this.stop(sessionId) + } + + /** Inside the serialized step that found nothing to do, so an accept after it wakes anew. */ + private stop(sessionId: string): 'stop' { + this.running.delete(sessionId) + return 'stop' + } +} + +/** A start that died while this message waited on it — a view's, say — is the message's failed + * start: settled with it, under its key, rather than started again into the same failure. */ +function startThatFailedWhileQueued( + session: StructuredAgentSessionHostSession, + oldest: NonNullable> +): StartFailure | null { + const ended = failedProviderChildStart(session) + if ( + !ended || + oldest.acceptedSequence === undefined || + ended.endedAt.epoch !== session.journal.cursor().epoch || + ended.endedAt.sequence < oldest.acceptedSequence + ) { + return null + } + return { startKey: ended.generation, text: endedChildRejection(ended) } +} + +const HOST_STOPPED_BEFORE_DELIVERY = 'Orca stopped the agent before this message was sent.' + +/** Why a queued message the child never took is rejected, in the words the chat row uses. */ +function endedChildRejection(ended: StructuredAgentSessionEndedChild): string { + if (ended.cause === 'host-stop') { + return ended.reason ?? HOST_STOPPED_BEFORE_DELIVERY + } + const reason = ended.reason ?? undefined + return ended.duringStartup + ? providerStartupFailureOutcome(reason) + : providerExitBeforeDeliveryRejection(reason) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts index 19989e10711..4905cc19257 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts @@ -42,14 +42,13 @@ export class StructuredAgentSessionEventRecovery { this.sinkFailures.add(sessionId) void this.context .serialize(sessionId, async () => { - const session = this.context.sessions.get(sessionId) + const child = this.context.sessions.get(sessionId)?.child const stop = this.context.deps.adapter.forceCloseSession ?? this.context.deps.adapter.closeSession - if (!session?.hasProviderChild || !stop) { + if (!child || !stop) { return null } - const fence = session.fence - const acquisitionGeneration = session.acquisitionGeneration + const { fence, generation: acquisitionGeneration } = child const stopped = await stopAgentSessionProviderRoot(() => stop(sessionId)) if (!stopped || !acquisitionGeneration) { return null diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts index cb0cb2fb233..8fc810de2a4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.test.ts @@ -30,8 +30,8 @@ function context(): StructuredAgentSessionEvictionContext & { order: string[] } return true }) } as unknown as StructuredAgentSessionEvictionContext['adapter'], - forget: vi.fn(async () => { - order.push('forget') + acknowledgeRelease: vi.fn(() => { + order.push('acknowledgeRelease') }), discardSink: vi.fn(() => order.push('discardSink')), settleWork: vi.fn(async () => { @@ -51,7 +51,7 @@ function runtimeState(): StructuredAgentSessionHostRuntimeState { } describe('structured agent session eviction', () => { - it('stops the child before it lets the sink go, then forgets the session', async () => { + it('stops the child before it lets the sink go, then acknowledges the release', async () => { const ctx = context() await evictStructuredAgentSession(ctx) expect(ctx.order).toEqual([ @@ -62,7 +62,7 @@ describe('structured agent session eviction', () => { 'close', 'discardSink', 'releaseLease', - 'forget' + 'acknowledgeRelease' ]) }) @@ -90,7 +90,7 @@ describe('structured agent session eviction', () => { 'close-sink', 'discard-sink', 'release-lease', - 'forget-session' + 'acknowledge-release' ]) }) @@ -114,7 +114,7 @@ describe('structured agent session eviction', () => { } }) - it('aborts after a failed drain barrier without unbinding or forgetting the session', async () => { + it('aborts after a failed drain barrier without unbinding or acknowledging the release', async () => { const ctx = context() ctx.eventSink.drained = vi.fn(async () => { ctx.order.push('drained') @@ -128,7 +128,7 @@ describe('structured agent session eviction', () => { expect(ctx.eventSink.close).not.toHaveBeenCalled() expect(ctx.discardSink).not.toHaveBeenCalled() expect(ctx.releaseLease).not.toHaveBeenCalled() - expect(ctx.forget).not.toHaveBeenCalled() + expect(ctx.acknowledgeRelease).not.toHaveBeenCalled() expect(ctx.order).toEqual(['closeSession', 'drained']) }) }) @@ -154,9 +154,9 @@ describe('rows the provider emits while closing', () => { return true } } as never, - forget: async () => {}, discardSink: () => state.discardEventSink(sessionId), - releaseLease: async () => {} + releaseLease: async () => {}, + acknowledgeRelease: () => {} }) expect(published).toEqual(['final-flush']) @@ -174,9 +174,13 @@ describe('a child that will not stop', () => { ctx.adapter.closeSession = vi.fn(async () => { throw error }) + const stopped = vi.fn() + ctx.onProviderChildStopped = stopped await evictStructuredAgentSession(ctx) + // The host ends its child on the one reading of the verdict, not a second one of its own. + expect(stopped).toHaveBeenCalledWith({ rootGone: true }) expect(ctx.order).toEqual([ 'drained', 'settleWork', @@ -184,18 +188,18 @@ describe('a child that will not stop', () => { 'close', 'discardSink', 'releaseLease', - 'forget' + 'acknowledgeRelease' ]) }) - it('aborts without forgetting the session, so the next close is a real retry', async () => { + it('aborts without acknowledging the release, so the next stop is a real retry', async () => { const ctx = context() ctx.adapter.closeSession = vi.fn(async () => false) await expect(evictStructuredAgentSession(ctx)).rejects.toMatchObject({ step: 'stop-provider-child' }) - expect(ctx.forget).not.toHaveBeenCalled() + expect(ctx.acknowledgeRelease).not.toHaveBeenCalled() expect(ctx.discardSink).not.toHaveBeenCalled() expect(ctx.order).toEqual([]) }) @@ -210,7 +214,7 @@ describe('a child that will not stop', () => { StructuredAgentSessionEvictionError ) expect(ctx.eventSink.close).not.toHaveBeenCalled() - expect(ctx.forget).not.toHaveBeenCalled() + expect(ctx.acknowledgeRelease).not.toHaveBeenCalled() }) }) @@ -225,9 +229,9 @@ describe('eviction against the real sink cache', () => { sessionId, eventSink: state.eventSinkFor(sessionId), adapter: { closeSession: async () => true } as never, - forget: async () => {}, discardSink: () => state.discardEventSink(sessionId), - releaseLease: async () => {} + releaseLease: async () => {}, + acknowledgeRelease: () => {} }) const published: string[] = [] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts index e5a1d942818..2c03143ff41 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-eviction.ts @@ -1,4 +1,4 @@ -// Releasing one structured session's resources. +// Stopping one structured session's provider child and handing its lease back. // // Teardown is a DATA list, not a method body, for the reason this file exists at all: the host // tracked which sessions were live in a map, and tore them down at three unrelated call sites @@ -22,6 +22,7 @@ import { type StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import type { DeferredStructuredAgentSessionEventSink } from './structured-agent-session-event-sink' +import type { StructuredAgentSessionStopVerdict } from './structured-agent-session-host-types' import { withTimeout } from '../../../shared/promise-timeout-fallback' export type StructuredAgentSessionEvictionContext = { @@ -29,17 +30,17 @@ export type StructuredAgentSessionEvictionContext = { hasProviderChild?: boolean eventSink: DeferredStructuredAgentSessionEventSink adapter: StructuredAgentSessionAdapter - /** Closes the session's journal handle and drops the map entry. Async and - * awaited: `close()` is ordered behind queued writes, and a delete that - * returns while the close is still queued leaves nothing to retry. */ - forget: () => Promise + /** Tells the adapter the released lease is done with, so it drops this child's route and index. + * The conversation stays: stopping the agent never closes its journal. */ + acknowledgeRelease: () => Promise | void /** Drops the cached sink so a later attach mints a fresh one. */ discardSink: () => void /** Fires right before the stop, while the child's turn and background roster are still live. A * throw is logged, never allowed to abort the stop. */ beforeProviderChildStop?: () => void - /** Fires once the adapter has PROVEN the child gone, so host bookkeeping stops claiming one. */ - onProviderChildStopped?: () => void + /** Fires with the stop's verdict once `stopAgentSessionProviderRoot` read the root gone, so host + * bookkeeping stops claiming a child. */ + onProviderChildStopped?: (verdict: StructuredAgentSessionStopVerdict) => void /** Whether this host still owes the child's wind-down. Distinct from `hasProviderChild`, which a * proven exit retires mid-run: the two disagree for exactly the steps a retry has to repeat. */ owesProviderChildWindDown?: boolean @@ -84,13 +85,13 @@ export const STRUCTURED_AGENT_SESSION_EVICTION_STEPS: readonly StructuredAgentSe } // An adapter with no close has nothing to stop; anything else must PROVE the exit. const stop = context.adapter.disposeSession ?? context.adapter.closeSession - if ( - stop && - !(await stopAgentSessionProviderRoot(() => stop.call(context.adapter, context.sessionId))) - ) { + const rootGone = stop + ? await stopAgentSessionProviderRoot(() => stop.call(context.adapter, context.sessionId)) + : true + if (!rootGone) { throw new Error('provider child exit was not proven') } - context.onProviderChildStopped?.() + context.onProviderChildStopped?.({ rootGone }) } }, { @@ -115,11 +116,11 @@ export const STRUCTURED_AGENT_SESSION_EVICTION_STEPS: readonly StructuredAgentSe // these two; eviction has to as well. { name: 'discard-sink', run: (context) => context.discardSink() }, // Why here and not last: the durable lease still names a process this host just stopped, and a - // record left claiming a live owner is one nothing can resume — the next surface to open the - // chat would find a session it may not acquire. Placed BEFORE forget so a release that cannot - // be written aborts while the session is still indexed, which is what makes the retry real. + // record left claiming a live owner is one nothing can resume — the next send would find a + // session it may not acquire. Placed BEFORE the acknowledgement so a release that cannot be + // written aborts while the adapter still routes the session, which is what makes the retry real. { name: 'release-lease', run: (context) => context.releaseLease() }, - { name: 'forget-session', run: (context) => context.forget() } + { name: 'acknowledge-release', run: (context) => context.acknowledgeRelease() } ] export class StructuredAgentSessionEvictionError extends Error { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts index 20531bd6465..1ffc08a0249 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts @@ -21,11 +21,10 @@ import type { StructuredAgentSessionAttachContext } from './structured-agent-ses import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import { StructuredAgentSessionStatusFeed } from './structured-agent-session-status-feed' -// Everything before the journal is out of scope here; what matters is that the orchestration's -// own `onAttachFailed` runs, which is the real one. +// Everything before the journal is out of scope here; what matters is what the orchestration does +// when the attach throws after acquisition. vi.mock('./structured-agent-session-attach-flow', () => ({ - performAttach: async (input: { onAttachFailed?: () => Promise }) => { - await input.onAttachFailed?.() + performAttach: async () => { throw new Error('attach failed after acquisition') } })) @@ -136,10 +135,7 @@ async function workingSession(): Promise<{ accountHome: ownerRecord().accountHome, runtimeKind: 'native' }, - fence: 1, - hasProviderChild: true, - providerChildPhase: 'ready', - acquisitionGeneration: null + child: { generation: null, fence: 1, phase: 'ready' } } ] ]) @@ -230,15 +226,19 @@ describe('a session that leaves the host without an explicit close', () => { expect(server.getStatusSnapshot()).toEqual([expect.objectContaining({ prompt: 'other host' })]) }) - it('leaves the agent-status store with it when an attach fails', async () => { + // A failed attach no longer drops the session: the conversation stays open for the failure to be + // written into, so its row stays with it and the later close forgets both together. + it('keeps the session and its status row together when an attach fails', async () => { const { server, feed, sessions } = await workingSession() + const drop = vi.spyOn(server, 'dropStructuredStatus') await expect( attachStructuredAgentSession(attachContext(sessions, feed), 'caller-1', attachParams) ).rejects.toThrow('attach failed after acquisition') - expect(sessions.has(SESSION)).toBe(false) - expect(server.getStatusSnapshot()).toEqual([]) + expect(sessions.has(SESSION)).toBe(true) + expect(drop).not.toHaveBeenCalled() + expect(server.getStatusSnapshot()).toHaveLength(1) }) // The feed's own cache deliberately retains the projection for reload history; only the store diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume-race.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume-race.test.ts index 974a69b1c3c..b1004f7e396 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume-race.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-hold-resume-race.test.ts @@ -29,6 +29,7 @@ function resumeHarness() { resume, serialize: keyedSerialize(), hasProviderChild: () => child, + lastStartFailed: () => false, hasOwedWork: () => turnActive, evict, graceMs: GRACE_MS @@ -212,6 +213,7 @@ describe('a surface leaving while its structured session resumes', () => { }, serialize: keyedSerialize(), hasProviderChild: () => child, + lastStartFailed: () => false, hasOwedWork: () => false, evict: async () => {}, graceMs: GRACE_MS @@ -245,6 +247,7 @@ describe('a surface leaving while its structured session resumes', () => { resume, serialize: keyedSerialize(), hasProviderChild: () => child, + lastStartFailed: () => false, hasOwedWork: () => false, evict: async () => {}, graceMs: GRACE_MS diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-holds.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-holds.test.ts index 68027f81887..822f36df5bf 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-holds.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-holds.test.ts @@ -184,6 +184,7 @@ describe('holds', () => { resume, serialize: keyedSerialize(), hasProviderChild: () => child, + lastStartFailed: () => false, hasOwedWork: () => false, evict: async () => {}, graceMs: 1 @@ -214,6 +215,7 @@ describe('holds', () => { resume, serialize, hasProviderChild: () => child, + lastStartFailed: () => false, hasOwedWork: () => false, evict: async () => {}, graceMs: 1 @@ -244,6 +246,7 @@ describe('holds', () => { }, serialize, hasProviderChild: () => child, + lastStartFailed: () => false, hasOwedWork: () => false, evict: async () => {}, graceMs: 60_000 @@ -262,6 +265,7 @@ describe('holds', () => { resume: async () => ({ ok: true as const }), serialize: keyedSerialize(), hasProviderChild: () => false, + lastStartFailed: () => false, hasOwedWork: () => false, evict, graceMs: 1 @@ -281,6 +285,7 @@ describe('holds', () => { resume: async () => ({ ok: true as const }), serialize: keyedSerialize(), hasProviderChild: () => false, + lastStartFailed: () => false, hasOwedWork: () => false, evict: async () => {}, graceMs: 1 @@ -296,7 +301,7 @@ describe('holds', () => { describe('the teardown deadline', () => { it('leaves the child loaded instead of forcing it, and keeps the session indexed', async () => { - const forget = vi.fn() + const acknowledgeRelease = vi.fn() const releaseLease = vi.fn(async () => {}) await expect( @@ -309,7 +314,7 @@ describe('the teardown deadline', () => { close: vi.fn() } as never, adapter: { closeSession: () => new Promise(() => {}) } as never, - forget, + acknowledgeRelease, discardSink: vi.fn(), releaseLease }, @@ -317,7 +322,7 @@ describe('the teardown deadline', () => { ) ).rejects.toMatchObject({ step: 'stop-provider-child' }) - expect(forget).not.toHaveBeenCalled() + expect(acknowledgeRelease).not.toHaveBeenCalled() expect(releaseLease).not.toHaveBeenCalled() }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-holds.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-holds.ts index 0acbccb6b55..89f28b890ed 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-holds.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-holds.ts @@ -6,7 +6,7 @@ // // A surface takes a hold when it binds and drops it when it goes away. The first hold on a session // with no child resumes it — that, and not the shape of a lease on disk, is what makes a provider -// process exist. The last hold leaving starts the idle release clock. Transport close is the BACKSTOP, +// process exist — unless that session's last start died starting: only a send retries one. The last hold leaving starts the idle release clock. Transport close is the BACKSTOP, // not the mechanism: a client that vanishes mid-flight never sends its release, so the caller // registers one against the connection and the holder set absorbs the duplicate. // @@ -33,6 +33,8 @@ export type StructuredAgentSessionHoldsDeps = { serialize: (sessionId: string, task: () => Promise) => Promise /** Whether evicting this session would actually free anything. */ hasProviderChild: (sessionId: string) => boolean + /** The last start died starting; a surface does not retry it, the next send does. */ + lastStartFailed: (sessionId: string) => boolean hasOwedWork: (sessionId: string) => boolean evict: (sessionId: string) => Promise onError?: (input: { sessionId: string; error: unknown }) => void @@ -77,7 +79,11 @@ export class StructuredAgentSessionHolds { } let resumed: StructuredAgentSessionResumeOutcome try { - resumed = await this.deps.serialize(sessionId, () => this.ensureProviderChild(sessionId)) + resumed = await this.deps.serialize(sessionId, () => + this.deps.lastStartFailed(sessionId) + ? Promise.resolve({ ok: true as const }) + : this.ensureProviderChild(sessionId) + ) } catch (error) { this.releaseFailedHold(sessionId, holderId, alreadyHeld, incarnation) throw error diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts new file mode 100644 index 00000000000..dab9ea639c6 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts @@ -0,0 +1,101 @@ +// The host's conversations: how one becomes open, and the delivery loop that hands its accepted +// messages to a provider child. Bundled because they share one invariant — a conversation open +// with a message queued has a delivery loop — and the open is where a loop for leftovers wakes. + +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import type { AgentJournalResetReason } from '../../../shared/agent-session-journal-types' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { + openStructuredAgentSessionConversation, + type OpenedStructuredAgentSessionConversation, + type StructuredAgentSessionConversationOpenOptions +} from './structured-agent-session-conversation-open' +import { StructuredAgentSessionDeliveryLoop } from './structured-agent-session-delivery-loop' +import type { StructuredAgentSessionResumeOutcome } from './structured-agent-session-hold-resume' +import type { + StructuredAgentSessionHostDeps, + StructuredAgentSessionHostSession +} from './structured-agent-session-host-types' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' +import { structuredAgentSessionStartFailureText } from './structured-agent-session-send-preparation' +import { settleInterruptedCompaction } from './structured-compaction-recovery' +import { recoverStructuredRewind } from './structured-rewind-recovery' + +export type StructuredAgentSessionConversationDelivery = { + loop: StructuredAgentSessionDeliveryLoop + /** For a caller inside the session's serialize. */ + open: ( + sessionId: string, + options?: StructuredAgentSessionConversationOpenOptions + ) => Promise + /** Indexes a conversation some other open produced, as `open` would have. */ + adoptOpened: ( + sessionId: string, + opened: OpenedStructuredAgentSessionConversation + ) => Promise +} + +export function createStructuredAgentSessionConversationDelivery(input: { + deps: StructuredAgentSessionHostDeps + sessions: Map + serialize: (sessionId: string, task: () => Promise) => Promise + trackStart: (start: Promise) => Promise + ensureProviderChild: (sessionId: string) => Promise + reset: (sessionId: string, journal: AgentSessionJournal, reset: AgentJournalResetReason) => void + publishRestored: (sessionId: string) => void +}): StructuredAgentSessionConversationDelivery { + const { deps, sessions } = input + const loop = new StructuredAgentSessionDeliveryLoop({ + sessions, + adapter: deps.adapter, + serialize: input.serialize, + trackStart: input.trackStart, + ensureProviderChild: input.ensureProviderChild, + conversationFence: (sessionId) => + structuredAgentSessionConversationFence(deps.store, sessionId), + startFailureText: (sessionId, cause) => + structuredAgentSessionStartFailureText(deps.store.getRecord(sessionId), cause), + onError: (sessionId, error) => deps.onEventSinkError?.({ sessionId, error }) + }) + const adoptOpened = async ( + sessionId: string, + opened: OpenedStructuredAgentSessionConversation + ): Promise => { + const { session, reset } = opened + sessions.set(sessionId, session) + if (reset) { + input.reset(sessionId, session.journal, reset) + } + input.publishRestored(sessionId) + await settleInterruptedCommands(deps, sessionId, session) + if (session.journal.submissions().some(isQueuedAgentJournalSubmission)) { + loop.wake(sessionId) + } + } + return { + loop, + adoptOpened, + open: (sessionId, options) => + openStructuredAgentSessionConversation({ deps, sessions, adoptOpened }, sessionId, options) + } +} + +/** + * A compaction or rewind found prepared when the conversation opens was started under a child + * this process no longer has — the open runs only when none is indexed — so nothing will finish + * it, and left alone it refuses every send until a view attaches. Settled here instead of by a + * start inside acceptance. A Codex rewind only its provider can prove stays for the attach. + */ +async function settleInterruptedCommands( + deps: StructuredAgentSessionHostDeps, + sessionId: string, + session: StructuredAgentSessionHostSession +): Promise { + const fence = structuredAgentSessionConversationFence(deps.store, sessionId) + try { + await settleInterruptedCompaction(deps.store, sessionId, session.journal, fence) + await recoverStructuredRewind(deps.store, sessionId, session.journal, fence) + } catch (error) { + deps.onEventSinkError?.({ sessionId, error }) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts index 0eed19031c7..49d8b24e644 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts @@ -7,6 +7,8 @@ import { agentChildWorkLiveness } from '../../../shared/agent-status-child-work-liveness' import { activeStructuredAgentSessionTurnId } from '../../../shared/structured-agent-session-projection' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import { DISPATCH_REJECTED_PROVIDER_CLOSED } from '../../../shared/structured-agent-session-dispatch-rejection' import { evictStructuredAgentSession, STRUCTURED_AGENT_SESSION_EVICTION_STEPS, @@ -16,9 +18,16 @@ import { withStructuredAgentSessionEvictionDeadline } from './structured-agent-s import { StructuredAgentSessionHolds } from './structured-agent-session-holds' import type { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' import type { + StructuredAgentSessionChildEndCause, StructuredAgentSessionHostDeps, - StructuredAgentSessionHostSession + StructuredAgentSessionHostSession, + StructuredAgentSessionProviderChildIdentity } from './structured-agent-session-host-types' +import { + endProviderChild, + failedProviderChildStart, + structuredAgentSessionConversationFence +} from './structured-agent-session-provider-child' import { releaseStoredStructuredAgentSessionOwner } from './structured-agent-session-lease-release' import { resumeHeldStructuredAgentSession } from './structured-agent-session-hold-resume' import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' @@ -31,6 +40,8 @@ export type StructuredAgentSessionLifetimeContext = { now: () => number /** Drops the session's row from the agent-status store; see `forgetStructuredAgentSession`. */ forgetStatus: (sessionId: string) => void + /** Re-projects the session's status after its agent stopped and the chat stays. */ + publishStatus?: (sessionId: string) => void /** Quit-only snapshot taken immediately before the provider child is stopped. */ restartWitness?: { beforeStop: (sessionId: string) => void @@ -38,13 +49,39 @@ export type StructuredAgentSessionLifetimeContext = { } } +type ConversationCloseDeps = Pick & { + store: Pick +} + +/** A conversation's handle closes with nothing queued: what is still queued when the chat closes, + * or the app quits, will not be handed over. Best effort: the next open's delivery loop rejects a + * leftover itself. */ +export async function abandonQueuedStructuredAgentSessionMessages( + deps: ConversationCloseDeps, + sessionId: string, + journal: StructuredAgentSessionHostSession['journal'] +): Promise { + await journal + .rejectQueuedSubmissions( + structuredAgentSessionConversationFence(deps.store, sessionId), + DISPATCH_REJECTED_PROVIDER_CLOSED + ) + .catch((error: unknown) => deps.onEventSinkError?.({ sessionId, error })) +} + /** Dropping a session and dropping its status row are ONE operation: the store keeps the row until * told, so a caller that only deletes strands a live-looking row no reader can ever decay. */ export async function forgetStructuredAgentSession( - context: StructuredAgentSessionLifetimeContext, + context: Pick & { + deps: ConversationCloseDeps + }, sessionId: string ): Promise { - await context.sessions.get(sessionId)?.journal.close() + const session = context.sessions.get(sessionId) + if (session) { + await abandonQueuedStructuredAgentSessionMessages(context.deps, sessionId, session.journal) + } + await session?.journal.close() context.sessions.delete(sessionId) context.forgetStatus(sessionId) } @@ -53,58 +90,79 @@ function hasProviderChild( context: StructuredAgentSessionLifetimeContext, sessionId: string ): boolean { - return context.sessions.get(sessionId)?.hasProviderChild === true + return (context.sessions.get(sessionId)?.child ?? null) !== null } /** The wind-down this host owes for the session's child. A live child always owes one, whatever a - * previous childless eviction recorded: a remembered `false` must never outrank the child in front - * of it. */ -function owesProviderChildWindDown(session: StructuredAgentSessionHostSession): boolean { - return session.hasProviderChild || session.owesProviderChildWindDown === true + * previous childless eviction recorded: a remembered tombstone must never outrank the child in + * front of it. */ +function owedProviderChildWindDown( + session: StructuredAgentSessionHostSession +): StructuredAgentSessionProviderChildIdentity | undefined { + return session.child + ? { generation: session.child.generation, fence: session.child.fence } + : session.owesProviderChildWindDown } -/** Runs the eviction steps under a deadline. A step that fails — or runs out of time — aborts the - * rest, which leaves the session indexed and the child loaded so the next close is a real retry. */ -export async function evictHeldStructuredAgentSession( +/** + * The agent goes to rest; the conversation stays. Runs the eviction steps under a deadline. A step + * that fails — or runs out of time — aborts the rest and leaves the wind-down owed, so the next + * stop is a real retry. `ending` is how the child's end is told: a user's Stop, the host stopping it + * for a cause (with its text), or an eviction whose close forgets the conversation next. + */ +export async function stopStructuredAgentSessionAgentUnderSerialize( context: StructuredAgentSessionLifetimeContext, - sessionId: string + sessionId: string, + ending: { + cause: Extract + reason?: string + } = { cause: 'user-stop' } ): Promise { const session = context.sessions.get(sessionId) if (!session) { return } - // The obligation OUTLIVES the child. `hasProviderChild` is retired the instant the adapter - // proves the exit, so a step that aborts after that point would otherwise leave the retry - // reading "no child here" and skipping the settlement and the lease release it still owes. - const owesWindDown = owesProviderChildWindDown(session) - session.owesProviderChildWindDown = owesWindDown + // The obligation OUTLIVES the child. `child` is ended the instant the adapter proves the exit, + // so a step that aborts after that point would otherwise leave the retry reading "no child + // here" and skipping the settlement and the lease release it still owes. + const owed = owedProviderChildWindDown(session) + session.owesProviderChildWindDown = owed + const stopping = session.child let settlementError: unknown const eviction: StructuredAgentSessionEvictionContext = { sessionId, // The retry must not re-stop a child the adapter already proved gone, so this stays honest. - hasProviderChild: session.hasProviderChild, - owesProviderChildWindDown: owesWindDown, + hasProviderChild: stopping !== null, + owesProviderChildWindDown: owed !== undefined, eventSink: context.runtimeState.eventSinkFor(sessionId), adapter: context.deps.adapter, ...(context.restartWitness ? { beforeProviderChildStop: () => context.restartWitness?.beforeStop(sessionId) } : {}), - // Host state must not disagree with the adapter for the seven steps in between. - onProviderChildStopped: () => { - session.hasProviderChild = false + // Host state must not disagree with the adapter for the steps in between. + onProviderChildStopped: (verdict) => { + if (stopping) { + endProviderChild(session, { + generation: stopping.generation, + fence: stopping.fence, + cause: ending.cause, + reason: ending.reason ?? null, + duringStartup: stopping.phase === 'starting', + ...verdict + }) + } context.restartWitness?.stopped(sessionId) }, - forget: async () => { - await forgetStructuredAgentSession(context, sessionId) - context.deps.adapter.acknowledgeSessionRelease?.(sessionId) - }, + acknowledgeRelease: () => context.deps.adapter.acknowledgeSessionRelease?.(sessionId), discardSink: () => context.runtimeState.discardEventSink(sessionId), settleWork: async () => { + const fence = + owed?.fence ?? structuredAgentSessionConversationFence(context.deps.store, sessionId) const settled = await settleStructuredAgentSessionDeadGeneration({ journal: session.journal, sessionId, - fence: session.fence, - settlementId: `expected-close:${sessionId}:${session.fence}:${session.acquisitionGeneration ?? 'unknown'}`, + fence, + settlementId: `expected-close:${sessionId}:${fence}:${owed?.generation ?? 'unknown'}`, pendingSubmissionReason: 'provider_closed_before_acknowledgement', verdict: { state: 'interrupted', completedAt: context.now() }, showUnexpectedExitOutcome: false, @@ -119,15 +177,22 @@ export async function evictHeldStructuredAgentSession( } }, releaseLease: async () => { - await releaseStoredStructuredAgentSessionOwner({ - store: context.deps.store, - sessionId, - hasProviderChild: owesWindDown, - expectedFence: session.fence, - now: context.now() - }) - session.owesProviderChildWindDown = false - context.forgetStatus(sessionId) + if (owed) { + await releaseStoredStructuredAgentSessionOwner({ + store: context.deps.store, + sessionId, + hasProviderChild: true, + expectedFence: owed.fence, + now: context.now() + }) + } + session.owesProviderChildWindDown = undefined + if (ending.cause === 'evict') { + context.forgetStatus(sessionId) + return + } + // The conversation stays: its readers keep their own fence, and only the status moves. + context.publishStatus?.(sessionId) } } await evictStructuredAgentSession( @@ -136,6 +201,19 @@ export async function evictHeldStructuredAgentSession( ) } +/** Ends the conversation's resources, not the conversation: its child stops, and then its handle + * closes and it leaves the map. A stop that fails throws first, leaving it indexed for a retry. */ +export async function evictHeldStructuredAgentSession( + context: StructuredAgentSessionLifetimeContext, + sessionId: string +): Promise { + if (!context.sessions.has(sessionId)) { + return + } + await stopStructuredAgentSessionAgentUnderSerialize(context, sessionId, { cause: 'evict' }) + await forgetStructuredAgentSession(context, sessionId) +} + /** Stops every provider child owned by this host while keeping failed evictions reachable. A * session whose child is already stopped but whose wind-down aborted is still in scope — that is * the retry. */ @@ -146,7 +224,7 @@ export async function evictOwnedStructuredAgentSessions( retainOnFailure: Set ): Promise { const ownedSessionIds = [...context.sessions] - .filter(([, session]) => owesProviderChildWindDown(session)) + .filter(([, session]) => owedProviderChildWindDown(session) !== undefined) .map(([sessionId]) => sessionId) // Retained up front and cleared only once an eviction settles: the quit phase is bounded, and a // timeout leaves these still running. Closing their journals underneath them is the one outcome @@ -176,7 +254,8 @@ export async function evictOwnedStructuredAgentSessions( * resume and a send's ensure-owner step are the same serialized attach with a different asker. */ export function createStructuredAgentSessionHolds( attachContext: () => StructuredAgentSessionAttachContext, - close: (sessionId: string) => Promise + close: (sessionId: string) => Promise, + deliveryActive: (sessionId: string) => boolean ): StructuredAgentSessionHolds { const context = attachContext() return new StructuredAgentSessionHolds({ @@ -197,15 +276,22 @@ export function createStructuredAgentSessionHolds( }, evict: close, hasProviderChild: (sessionId) => hasProviderChild(context, sessionId), - // A send pending while the child is still starting is held for that start; evicting would - // refuse it. Any other pending send may wait on an echo that never comes, so eviction retires it. - // Subagents, commands and monitors outlive the lead's turn inside the child, so the live roster - // the sidebar shows as working is owed too; stopping the child would end them silently. + lastStartFailed: (sessionId) => { + const session = context.sessions.get(sessionId) + return session !== undefined && failedProviderChildStart(session) !== null + }, + // A message accepted and not yet handed over is owed to this child, and so is one pending while + // the child still starts. Any other pending send may wait on an echo that never comes, so + // eviction retires it. Subagents, commands and monitors outlive the lead's turn inside the + // child, so the live roster the sidebar shows as working is owed too; stopping the child would + // end them silently. hasOwedWork: (sessionId) => { const session = context.sessions.get(sessionId) return session ? activeStructuredAgentSessionTurnId(session.journal.snapshot().items) !== null || - (session.providerChildPhase === 'starting' && + deliveryActive(sessionId) || + session.journal.submissions().some(isQueuedAgentJournalSubmission) || + (session.child?.phase === 'starting' && session.journal.pendingSubmissions().length > 0) || agentChildWorkLiveness(context.deps.adapter.backgroundTaskState?.(sessionId)?.tasks) !== null diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts index c1ae42e237c..9decd5a488a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts @@ -3,8 +3,8 @@ // // They share one shape — admit the envelope against the lease, run a plan, publish the journal — so // they share one path here rather than five copies in the host. The host keeps attach, holds and -// teardown. A send is the one mutation that may need those first: it makes sure the session has -// an owner as a step of its own serialized admission, see `structured-agent-session-send-preparation`. +// teardown. A send and a Stop are conversation writes: they open the conversation and are admitted +// without the writer lease; the session's delivery loop starts the provider child a send needs. import type { AgentJournalItemIdentity, @@ -21,15 +21,16 @@ import type { AgentSessionThreadGoalChange, AgentSessionThreadGoalResult } from '../../../shared/agent-session-wire' -import type { StructuredAgentSessionHolds } from './structured-agent-session-holds' +import { DISPATCH_REJECTED_CANCELLED } from '../../../shared/structured-agent-session-dispatch-rejection' import type { AgentSessionPromptRequest } from './structured-agent-session-turns-prompt' import { threadGoalPlan } from './structured-agent-session-thread-goal' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' import { admitAndRunAgentSessionMutation, type AgentSessionMutationRequest } from './structured-agent-session-mutation-admission' import { - prepareStructuredAgentSessionSend, + openConversationForWrite, structuredAgentSessionSendBlock } from './structured-agent-session-send-preparation' import { @@ -52,11 +53,12 @@ export type StructuredAgentSessionMutationContext = { flushStreamedEvents: (sessionId: string) => Promise requireSession: (sessionId: string) => StructuredAgentSessionHostSession serialize: (sessionId: string, task: () => Promise) => Promise - /** A send that finds the owner gone brings it back through here, inside its own serialize. */ - holds: Pick - /** Makes a closed session's journal readable again, inside the caller's serialize, for a send - * the ledger answers without an owner. */ - restoreReadable: (sessionId: string) => Promise + /** The session's conversation, opened when closed; inside the caller's serialize. */ + openConversation: (sessionId: string) => Promise + /** A message was accepted: the session's delivery loop hands it over. */ + wakeDelivery: (sessionId: string) => void + /** Stops the session's provider child, keeping its conversation; inside the caller's serialize. */ + stopAgent: (sessionId: string) => Promise now: () => number } @@ -78,7 +80,7 @@ function mutate( prepareSession, publish: (journal) => context.publish(envelope.sessionId, journal), flushStreamedEvents: context.flushStreamedEvents, - providerChildPhase: () => context.sessions.get(envelope.sessionId)?.providerChildPhase, + providerChildPhase: () => context.sessions.get(envelope.sessionId)?.child?.phase, now: () => context.now() }) ) @@ -101,12 +103,19 @@ export function sendStructuredAgentSessionTurn( params.envelope, { ...plan, - run: (ctx) => { + run: async (ctx) => { const blocked = structuredAgentSessionSendBlock(context.deps.store.getRecord(ctx.sessionId)) - return blocked ? Promise.resolve(blocked) : plan.run(ctx) + if (blocked) { + return blocked + } + const accepted = await plan.run(ctx) + if (accepted.ok) { + context.wakeDelivery(ctx.sessionId) + } + return accepted } }, - (ledger, record) => prepareStructuredAgentSessionSend(context, params.envelope, ledger, record) + () => openConversationForWrite(context.openConversation, params.envelope) ) } @@ -131,7 +140,35 @@ export function cancelStructuredAgentSessionTurn( context.serialize(`compact-cancel:${sessionId}`, task) } : context - return mutate(cancellationContext, caller, params.envelope, cancelPlan(params)) + const plan = cancelPlan(params) + if (params.scope || params.prompt) { + return mutate(cancellationContext, caller, params.envelope, plan) + } + return mutate( + cancellationContext, + caller, + params.envelope, + { + ...plan, + run: async (ctx) => { + // Stop withdraws every queued message first, whatever the start or the child is doing. + const withdrawn = await ctx.journal.rejectQueuedSubmissions( + ctx.fence, + DISPATCH_REJECTED_CANCELLED + ) + const child = context.sessions.get(ctx.sessionId)?.child + if (child?.phase === 'starting') { + // A start that may never land is the one thing here Stop has to end; the chat stays. + await context.stopAgent(ctx.sessionId) + return { ok: true, value: { turnId: params.turnId, cancelled: true } } + } + return child + ? plan.run(ctx) + : { ok: true, value: { turnId: params.turnId, cancelled: withdrawn.length > 0 } } + } + }, + () => openConversationForWrite(context.openConversation, params.envelope) + ) } export function respondToStructuredAgentSessionPrompt( @@ -169,7 +206,12 @@ export function readStructuredAgentSessionOptions( if (!context.deps.adapter.readOptions) { throw new Error('structured_agent_session_options_unsupported') } - const options = await context.deps.adapter.readOptions({ sessionId, fence: session.fence }) + const options = await context.deps.adapter.readOptions({ + sessionId, + fence: + session.child?.fence ?? + structuredAgentSessionConversationFence(context.deps.store, sessionId) + }) return { ...options, rewind: @@ -203,6 +245,7 @@ export async function settleStructuredAgentSessionLateDispatch( if (!session) { return } + const fence = structuredAgentSessionConversationFence(context.deps.store, input.sessionId) // The journal queue drains before close; the host queue would defer this past teardown. await session.journal.resolveDispatch( 'providerIdentity' in input @@ -210,13 +253,13 @@ export async function settleStructuredAgentSessionLateDispatch( clientMessageId: input.clientMessageId, state: 'accepted', providerIdentity: input.providerIdentity, - fence: session.fence + fence } : { clientMessageId: input.clientMessageId, state: 'rejected', reason: input.reason, - fence: session.fence + fence } ) } @@ -234,7 +277,9 @@ export async function settleStructuredAgentSessionLateDispatch( * it never makes a send re-deliverable, because the provider may well have run it. */ export async function releaseStructuredAgentSessionUnansweredDispatches( - context: Pick, + context: Pick & { + deps: { store: Pick } + }, input: { sessionId: string; reason: string } ): Promise { const session = context.sessions.get(input.sessionId) @@ -253,7 +298,7 @@ export async function releaseStructuredAgentSessionUnansweredDispatches( state: 'unknown', // The earlier reason names a sharper fact than this one does. reason: entry.reason ?? input.reason, - fence: session.fence, + fence: structuredAgentSessionConversationFence(context.deps.store, input.sessionId), recovered: true }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.ts index 6b1693c3dca..2f1b0f3c287 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-teardown.ts @@ -9,6 +9,7 @@ import type { AgentSessionResumeTrigger } from '../../../shared/agent-session-resume-marker' import type { StructuredAgentSessionRestartResume } from './structured-agent-session-restart-resume-host' import { + abandonQueuedStructuredAgentSessionMessages, evictOwnedStructuredAgentSessions, type StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' @@ -93,6 +94,8 @@ export async function tearDownStructuredAgentSessionHost(input: { sessions: Map retainSessionIds?: ReadonlySet acknowledgeSessionRelease?: (sessionId: string) => void + /** Quit closes every conversation, so it settles what they still queue as a close does. */ + abandonQueued?: (sessionId: string, session: StructuredAgentSessionHostSession) => Promise }): Promise { const failures: unknown[] = [] for (const phase of input.phases) { @@ -107,7 +110,12 @@ export async function tearDownStructuredAgentSessionHost(input: { ([sessionId]) => !input.retainSessionIds?.has(sessionId) ) // `allSettled`, so one rejected close cannot skip the others. - const closed = await Promise.allSettled(entries.map(([, session]) => session.journal.close())) + const closed = await Promise.allSettled( + entries.map(async ([sessionId, session]) => { + await input.abandonQueued?.(sessionId, session) + await session.journal.close() + }) + ) closed.forEach((result, index) => { const sessionId = entries[index]?.[0] if (result.status === 'fulfilled') { @@ -160,6 +168,8 @@ export async function flushStructuredAgentSessionHost( sessions: context.sessions, retainSessionIds, acknowledgeSessionRelease: (sessionId) => - context.deps.adapter.acknowledgeSessionRelease?.(sessionId) + context.deps.adapter.acknowledgeSessionRelease?.(sessionId), + abandonQueued: (sessionId, session) => + abandonQueuedStructuredAgentSessionMessages(context.deps, sessionId, session.journal) }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts index fd2c911c74e..44611300c7c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts @@ -1,4 +1,5 @@ import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionStatusSummary } from '../../../shared/agent-session-wire' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' @@ -26,25 +27,60 @@ export type StructuredAgentSessionReveal = { readable: boolean } +/** Which provider child: the adapter acquisition and the lease fence it writes at. */ +export type StructuredAgentSessionProviderChildIdentity = { + readonly generation: string | null + readonly fence: number +} + +/** The provider process behind a conversation. Written only in + * `structured-agent-session-provider-child`. */ +export type StructuredAgentSessionProviderChild = StructuredAgentSessionProviderChildIdentity & { + /** A publish-first acquire is `starting` until the adapter's `started` event; only then are its + * reported options fact. */ + phase: StructuredAgentSessionProviderChildPhase +} + +/** What ending a child established about its provider root. A stop's comes only from + * `stopAgentSessionProviderRoot`; an observed exit's root is gone by definition. */ +export type StructuredAgentSessionStopVerdict = { rootGone: boolean } + +export type StructuredAgentSessionChildEndCause = + | 'user-stop' + | 'host-stop' + | 'exit' + | 'attach-failed' + | 'evict' + +/** How the conversation's last child ended. In memory only: the delivery loop reads it to tell a + * Stop from a failure. */ +export type StructuredAgentSessionEndedChild = StructuredAgentSessionProviderChildIdentity & + StructuredAgentSessionStopVerdict & { + /** `user-stop` is a Stop the user asked for; `host-stop` is the host stopping the child for a + * cause of its own, which fails the start the delivery loop was waiting on. */ + cause: StructuredAgentSessionChildEndCause + /** Descriptive text only — the provider's diagnostic, or the host's cause. Decides nothing. */ + reason: string | null + duringStartup: boolean + /** Where the conversation's journal stood when the child ended, to order the end against a + * message's acceptance. */ + endedAt: AgentJournalCursor + } + +/** The conversation: its journal, params and readers outlive any child that serves it. */ export type StructuredAgentSessionHostSession = { /** Readonly: a new handle enters only through the session map's `set`, which binds its delivery. */ readonly journal: AgentSessionJournal params: AgentSessionAttachParams - fence: number - /** Whether THIS host generation is running the provider process behind the session. A journal - * restored for reading has none — so it may not be evicted to free a child, nor have its lease - * released as an observed exit. */ - hasProviderChild: boolean - /** Whether the child behind `hasProviderChild` has proven its start. A publish-first acquire - * is `starting` until the adapter's `started` event; only then are its reported options fact. */ - providerChildPhase: StructuredAgentSessionProviderChildPhase + /** The child THIS host generation runs for the conversation. A conversation opened for reading + * has none — so it may not be evicted to free a child, nor have its lease released as an + * observed exit. */ + child: StructuredAgentSessionProviderChild | null /** The wind-down this host still owes for a child it started: settling that generation's work - * and handing the lease back. A separate fact from `hasProviderChild`, which goes false the - * moment the adapter proves the exit — an eviction that aborts after that point must still be - * able to finish the wind-down on the next close. */ - owesProviderChildWindDown?: boolean - /** Exact adapter acquisition behind `hasProviderChild`; retained after exit to fence recovery. */ - acquisitionGeneration: string | null + * and handing the lease back. Outlives `child`, which ends the moment the adapter proves the + * exit — an eviction that aborts after that point must still finish it on the next close. */ + owesProviderChildWindDown?: StructuredAgentSessionProviderChildIdentity + lastEndedChild?: StructuredAgentSessionEndedChild } export type StructuredAgentSessionHostDeps = { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts index ad2ac96da7a..3d0f5c9d427 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts @@ -788,9 +788,19 @@ describe('subscribe', () => { body }) - expect(result).toMatchObject({ ok: true, value: { submission: { dispatchState: 'accepted' } } }) - expect(dispatch).toHaveBeenCalledTimes(1) - expect(events.some((event) => event.type === 'batch')).toBe(true) + expect(result).toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) + // The failed transport does not stop the delivery loop either: the handover still lands and + // reaches the live subscriber. + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1)) + await vi.waitFor(() => + expect( + events.some( + (event) => + event.type === 'batch' && + event.batch.submissions?.some((entry) => entry.dispatchState === 'accepted') + ) + ).toBe(true) + ) }) it('resets a subscriber whose epoch is gone', async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index 40bbc38238d..6462fbe5ad7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -20,6 +20,7 @@ import { attachStructuredAgentSession } from './structured-agent-session-attach- import { createStructuredAgentSessionHolds, evictHeldStructuredAgentSession, + stopStructuredAgentSessionAgentUnderSerialize, type StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' import type { @@ -50,6 +51,8 @@ import { type StructuredAgentSessionRestartResume } from './structured-agent-session-restart-resume-host' import { structuredAgentSessionRestartResumeSurfaces } from './structured-agent-session-restart-resume-wiring' +import { createStructuredAgentSessionConversationDelivery } from './structured-agent-session-host-delivery' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' export type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' export class StructuredAgentSessionHost { @@ -75,6 +78,9 @@ export class StructuredAgentSessionHost { ) => Promise private readonly restore: ReturnType private readonly holds: StructuredAgentSessionHolds + private readonly conversationDelivery: ReturnType< + typeof createStructuredAgentSessionConversationDelivery + > private readonly eventRecovery: StructuredAgentSessionEventRecovery private readonly backgroundTasks: StructuredAgentSessionBackgroundTaskChannel /** Public because the RPC surface addresses it directly; see the restart-resume collaborator. */ @@ -97,9 +103,26 @@ export class StructuredAgentSessionHost { ...(deps.probeOwners ? { probeMany: deps.probeOwners } : {}), now: () => this.now() }) + this.conversationDelivery = createStructuredAgentSessionConversationDelivery({ + deps, + sessions: this.sessions, + serialize: (sessionId, task) => this.serialize(sessionId, task), + // Quit drains a delivery start before it evicts, so the child it produces is stopped. + trackStart: (start) => this.tasks.trackAttach(start), + ensureProviderChild: (sessionId) => this.holds.ensureProviderChild(sessionId), + reset: (sessionId, journal, reset) => + this.subscribers.reset( + sessionId, + journal, + reset, + structuredAgentSessionConversationFence(deps.store, sessionId) + ), + publishRestored: this.clientDelivery.publishRestored + }) this.holds = createStructuredAgentSessionHolds( () => this.attachContext(), - (sessionId) => this.close(sessionId) + (sessionId) => this.close(sessionId), + (sessionId) => this.conversationDelivery.loop.isRunning(sessionId) ) this.restore = createStructuredAgentSessionHostRestore(deps, { reconcile: this.reconcileLeases, @@ -108,10 +131,7 @@ export class StructuredAgentSessionHost { hasSession: this.hasSession, // Site 10: cannot overwrite a live entry — the restorer returns early on // `hasSession` inside the same serialized step as this `set`. - onReadable: (sessionId, restored) => { - this.sessions.set(sessionId, restored) - this.clientDelivery.publishRestored(sessionId) - } + onReadable: this.conversationDelivery.adoptOpened }) this.eventRecovery = new StructuredAgentSessionEventRecovery({ deps, @@ -119,7 +139,11 @@ export class StructuredAgentSessionHost { sessions: this.sessions, flushLifecycle: (sessionId) => this.runtimeState.lifecycleBarrier(sessionId), publishFence: (sessionId, session) => - this.subscribers.snapshot(sessionId, session.journal, session.fence), + this.subscribers.snapshot( + sessionId, + session.journal, + structuredAgentSessionConversationFence(deps.store, sessionId) + ), publishStatus: this.clientDelivery.publishStatusAndSettlement, hasResumeCapableHolder: (sessionId) => this.holds.hasResumeCapableHolder(sessionId), restartReleaseGrace: (sessionId) => this.holds.renew(sessionId), @@ -143,7 +167,8 @@ export class StructuredAgentSessionHost { isHeld = (sessionId: string): boolean => this.holds.isHeld(sessionId) /** A surface bound to this session and wants it live. The FIRST hold on a session with no - * provider child is what resumes one; a retained hold (a subscription) only keeps it. */ + * provider child is what resumes one, unless its last start failed; a retained hold (a + * subscription) only keeps it. */ hold = ( sessionId: string, holderId: string, @@ -162,7 +187,8 @@ export class StructuredAgentSessionHost { runtimeState: this.runtimeState, sessions: this.sessions, now: () => this.now(), - forgetStatus: this.clientDelivery.forgetStatus + forgetStatus: this.clientDelivery.forgetStatus, + publishStatus: this.clientDelivery.publishStatus } } @@ -174,18 +200,21 @@ export class StructuredAgentSessionHost { tasks: this.tasks, reconcileLeases: (sessionId) => this.reconcileLeases(sessionId), serialize: (sessionId, task) => this.serialize(sessionId, task), - publishStatus: this.clientDelivery.publishStatus + publishStatus: this.clientDelivery.publishStatus, + openConversation: this.conversationDelivery.open } } /** Releases a session's resources without ending the conversation: the record and journal stay * on disk, so the same session can be attached again. */ close(sessionId: string): Promise { - return this.serialize(sessionId, async () => { - await evictHeldStructuredAgentSession(this.lifetimeContext(), sessionId) - this.clientDelivery.closeSession(sessionId) - // The holders now look at a session that is gone; a failed eviction throws above, keeping them. - this.holds.forget(sessionId) - }) + return this.serialize(sessionId, () => this.closeUnderSerialize(sessionId)) + } + + private async closeUnderSerialize(sessionId: string): Promise { + await evictHeldStructuredAgentSession(this.lifetimeContext(), sessionId) + this.clientDelivery.closeSession(sessionId) + // The holders now look at a session that is gone; a failed eviction throws above, keeping them. + this.holds.forget(sessionId) } supportsCreate = (location: AgentSessionExecutionLocation, agent: string): boolean => @@ -227,6 +256,7 @@ export class StructuredAgentSessionHost { // Trigger inlined rather than imported: `AgentSessionResumeTrigger` in shared is the canonical // type, and this file has no line budget left for the import. async flushAllStreamedEvents(options?: { trigger?: 'quit' | 'update' }): Promise { + this.conversationDelivery.loop.dispose() await flushStructuredAgentSessionHost({ ...this.lifetimeContext(), holds: this.holds, @@ -245,8 +275,12 @@ export class StructuredAgentSessionHost { flushStreamedEvents: this.flushStreamedEvents, requireSession: (sessionId) => this.requireSession(sessionId), serialize: (sessionId, task) => this.serialize(sessionId, task), - holds: this.holds, - restoreReadable: (sessionId) => this.restore.restoreReadableUnderSerialize(sessionId), + openConversation: this.conversationDelivery.open, + wakeDelivery: (sessionId) => this.conversationDelivery.loop.wake(sessionId), + stopAgent: (sessionId) => + stopStructuredAgentSessionAgentUnderSerialize(this.lifetimeContext(), sessionId, { + cause: 'user-stop' + }), now: () => this.now() } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-handles.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-handles.test.ts index ef08e1c7e93..942c9ef6152 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-handles.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-handles.test.ts @@ -15,11 +15,7 @@ import { journalDatabaseFile } from '../agent-session-journal/journal-paths' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { createTrackedJournalOpener } from '../agent-session-journal/journal-store-test-open' import { openAgentSessionJournalWithRecovery } from './agent-session-journal-recovery' -import { - evictStructuredAgentSession, - STRUCTURED_AGENT_SESSION_EVICTION_STEPS, - type StructuredAgentSessionEvictionContext -} from './structured-agent-session-eviction' +import { forgetStructuredAgentSession } from './structured-agent-session-host-lifetime' import { tearDownStructuredAgentSessionHost } from './structured-agent-session-host-teardown' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' @@ -73,30 +69,12 @@ function hostSession(journal: AgentSessionJournal): StructuredAgentSessionHostSe return { journal, params: {} as StructuredAgentSessionHostSession['params'], - fence: 1, - hasProviderChild: false, - providerChildPhase: 'ready', - acquisitionGeneration: null + child: null } } -function evictionContext( - overrides: Partial -): StructuredAgentSessionEvictionContext { - return { - sessionId: SESSION, - hasProviderChild: false, - eventSink: { - drained: async () => ({ ok: true }) as const, - unbind: () => undefined, - close: () => undefined - } as unknown as StructuredAgentSessionEvictionContext['eventSink'], - adapter: {} as StructuredAgentSessionEvictionContext['adapter'], - forget: async () => undefined, - discardSink: () => undefined, - releaseLease: async () => undefined, - ...overrides - } +function forgetContext(sessions: Map) { + return { deps: { store: { getRecord: () => null } }, sessions, forgetStatus: () => undefined } } beforeEach(async () => { @@ -140,46 +118,37 @@ describe('site 6: recovery rehydration', () => { }) }) -describe('sites 9 and 10: the delete and overwrite callbacks', () => { +describe('sites 9 and 10: closing a conversation handle', () => { it('awaits the journal close before dropping the map entry', async () => { const journal = await journals.open({ identity: IDENTITY, journalDir }) const sessions = new Map([[SESSION, hostSession(journal)]]) const order: string[] = [] + const close = journal.close.bind(journal) + journal.close = async () => { + order.push('close-started') + await close() + order.push(sessions.has(SESSION) ? 'closed' : 'dropped-before-close') + } - await evictStructuredAgentSession( - evictionContext({ - forget: async () => { - order.push('close-started') - await sessions.get(SESSION)?.journal.close() - order.push('closed') - sessions.delete(SESSION) - order.push('forgotten') - } - }), - STRUCTURED_AGENT_SESSION_EVICTION_STEPS - ) + await forgetStructuredAgentSession(forgetContext(sessions), SESSION) - expect(order).toEqual(['close-started', 'closed', 'forgotten']) + expect(order).toEqual(['close-started', 'closed']) expect(sessions.size).toBe(0) await expectNothingHoldsTheDirectory(journalDir) }) - it('aborts the eviction with the session still indexed when the close rejects', async () => { + it('keeps the session indexed when the close rejects', async () => { const journal = await journals.open({ identity: IDENTITY, journalDir }) const sessions = new Map([[SESSION, hostSession(journal)]]) + const close = journal.close.bind(journal) + journal.close = () => Promise.reject(new Error('close rejected')) - await expect( - evictStructuredAgentSession( - evictionContext({ - forget: async () => { - await Promise.reject(new Error('close rejected')) - } - }), - STRUCTURED_AGENT_SESSION_EVICTION_STEPS - ) - ).rejects.toMatchObject({ step: 'forget-session' }) + await expect(forgetStructuredAgentSession(forgetContext(sessions), SESSION)).rejects.toThrow( + 'close rejected' + ) // Still indexed, so the next close is a real retry. expect(sessions.has(SESSION)).toBe(true) + journal.close = close }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-late-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-late-settlement.test.ts index 1874f7ee1c6..6bda0b66189 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-late-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-late-settlement.test.ts @@ -71,6 +71,18 @@ function journal(): AgentSessionJournal { ).sessions.get(SESSION)!.journal } +/** A send is accepted first; this waits for the delivery loop to hand it to the provider. */ +async function handedOver(clientMessageId: string): Promise { + await vi.waitFor(() => { + expect( + journal() + .submissions() + .find((entry) => entry.clientMessageId === clientMessageId)?.handedOverAt + ).toBeDefined() + expect(dispatch).toHaveBeenCalled() + }) +} + beforeEach(async () => { root = await mkdtemp(join(tmpdir(), 'orca-wire-late-settle-')) resetHostTestOperationIds() @@ -152,10 +164,13 @@ describe('settling a send the provider proves it received after the ack window', finishDispatch({ state: 'unknown', reason: 'ack timeout' }) unsubscribe() } - await expect(pending).resolves.toMatchObject({ - ok: true, - value: { submission: { dispatchState: 'accepted' } } - }) + await expect(pending).resolves.toMatchObject({ ok: true }) + // The late `unknown` from the handover does not reopen the proven acceptance. + await vi.waitFor(() => + expect(submissions()).toMatchObject([ + { clientMessageId: params.envelope.clientOperationId, dispatchState: 'accepted' } + ]) + ) await expect(host.send(CALLER, { ...params, retryUnknown: true })).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'accepted' } } @@ -167,6 +182,8 @@ describe('settling a send the provider proves it received after the ack window', dispatch.mockResolvedValueOnce({ state: 'unknown', reason: 'ack timeout' }) const params = sendParams('received just before shutdown') await host.send(CALLER, params) + await handedOver(params.envelope.clientOperationId) + await vi.waitFor(() => expect(submissions()).toMatchObject([{ dispatchState: 'unknown' }])) let settlement: Promise | undefined closeSession.mockImplementationOnce(async () => { settlement = host.settleLateDispatch({ @@ -188,8 +205,9 @@ describe('settling a send the provider proves it received after the ack window', it('moves a durable unknown to accepted so nothing offers to send it again', async () => { dispatch.mockRejectedValueOnce(new Error('socket closed')) const params = sendParams('sent while a turn was running') - const first = await host.send(CALLER, params) - expect(first).toMatchObject({ ok: true, value: { submission: { dispatchState: 'unknown' } } }) + await host.send(CALLER, params) + await handedOver(params.envelope.clientOperationId) + await vi.waitFor(() => expect(submissions()).toMatchObject([{ dispatchState: 'unknown' }])) await host.settleLateDispatch({ sessionId: SESSION, @@ -209,6 +227,7 @@ describe('settling a send the provider proves it received after the ack window', dispatch.mockResolvedValueOnce({ state: 'admitted' }) const params = sendParams('queued behind the active turn') await host.send(CALLER, params) + await handedOver(params.envelope.clientOperationId) await host.settleLateDispatch({ sessionId: SESSION, @@ -230,6 +249,7 @@ describe('settling a send the provider proves it received after the ack window', dispatch.mockResolvedValueOnce({ state: 'admitted' }) const params = sendParams('settle from provider echo') await host.send(CALLER, params) + await handedOver(params.envelope.clientOperationId) vi.spyOn(journal(), 'resolveDispatch').mockRejectedValueOnce( new Error('direct settlement write failed') ) @@ -259,6 +279,7 @@ describe('settling a send the provider proves it received after the ack window', it('leaves an already accepted send alone', async () => { const params = sendParams('ordinary send') await host.send(CALLER, params) + await vi.waitFor(() => expect(submissions()).toMatchObject([{ dispatchState: 'accepted' }])) await host.settleLateDispatch({ sessionId: SESSION, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-launch-send-after-create.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-launch-send-after-create.test.ts index 300f1423cb3..e7bd9a9ac85 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-launch-send-after-create.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-launch-send-after-create.test.ts @@ -57,12 +57,20 @@ async function launchAndDeliver(): Promise<{ text: 'fix the failing test' }) const sent = await send.mock.results[0]!.value - return { - messageId, - dispatchState: sent.ok - ? sent.value.submission.dispatchState - : `refused:${sent.refusal.code}:${sent.refusal.message}` + if (!sent.ok) { + return { messageId, dispatchState: `refused:${sent.refusal.code}:${sent.refusal.message}` } } + // Accepted first; the delivery loop hands it over, and that outcome is what reached the agent. + let dispatchState = sent.value.submission.dispatchState + await vi.waitFor(() => { + dispatchState = + host + .journalSnapshot(created.value.sessionId) + .submissions.find((entry) => entry.clientMessageId === sent.value.clientMessageId) + ?.dispatchState ?? 'missing' + expect(dispatchState).not.toBe('pending') + }) + return { messageId, dispatchState } } beforeEach(() => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts index 5e0d7c83616..f61dd491054 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts @@ -25,12 +25,18 @@ import { const CALLER = { callerKey: 'client-1' } +/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */ +function eventually(assertion: () => void): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} + let root: string let store: AgentSessionRecordStore let host: StructuredAgentSessionHost let acquire: Mock let probe: Mock<() => Promise> let stopOwnerProcess: Mock<(pid: number, signal: 'SIGTERM' | 'SIGKILL') => void> +let dispatch: Mock function openHost(): void { host = new StructuredAgentSessionHost({ @@ -39,7 +45,7 @@ function openHost(): void { acquire, closeSession: vi.fn(async () => true), releaseAcquisition: vi.fn(async () => true), - dispatch: vi.fn(async () => ({ state: 'admitted' as const })), + dispatch, cancelTurn: vi.fn(async () => ({ cancelled: false })), answerPrompt: vi.fn(async () => undefined), setOption: vi.fn(async () => undefined) @@ -71,6 +77,24 @@ async function persistFromOlderBuild(lease: OlderBuildLease): Promise { openHost() } +/** A send is accepted at once; what became of it is the submission's state once the host's + * delivery settles it — handed over, or rejected with the reason the chat shows. */ +async function delivered(text: string) { + const sent = await send(text) + expect(sent).toMatchObject({ ok: true }) + const clientMessageId = sent.ok ? sent.value.clientMessageId : '' + const submission = () => + host + .journalSnapshot(SESSION) + .submissions.find((candidate) => candidate.clientMessageId === clientMessageId) + await eventually(() => + expect( + submission()?.dispatchState !== 'pending' || submission()?.handedOverAt !== undefined + ).toBe(true) + ) + return submission() +} + async function send(text: string) { const body = hostTestMessage(text) return host.send(CALLER, { @@ -93,6 +117,7 @@ beforeEach(async () => { resetHostTestOperationIds() probe = vi.fn(async () => ({ outcome: 'pid-absent' as const })) stopOwnerProcess = vi.fn() + dispatch = vi.fn(async () => ({ state: 'admitted' as const })) acquire = vi.fn(async ({ fence, spawnToken }) => ({ process: { hostId: 'local', pid: 4242, processStartTimeMs: NOW - 1_000, spawnToken }, link: { @@ -143,7 +168,8 @@ describe('a record an older build left mid terminal handoff', () => { handoffOperationId: null }) expect(store.getRecord(SESSION)?.lease).not.toHaveProperty('settlementRetryRequired') - expect(await send('after the upgrade')).toMatchObject({ ok: true }) + expect(await delivered('after the upgrade')).toMatchObject({ dispatchState: 'pending' }) + expect(dispatch).toHaveBeenCalledOnce() expect(acquire).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ runtimeKind: 'native', @@ -175,7 +201,8 @@ describe('a record an older build left mid terminal handoff', () => { handoffStage: null, handoffOperationId: null }) - expect(await send('after the upgrade')).toMatchObject({ ok: true }) + expect(await delivered('after the upgrade')).toMatchObject({ dispatchState: 'pending' }) + expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'live' }) }) @@ -194,7 +221,8 @@ describe('a record an older build left mid terminal handoff', () => { handoffStage: null, claimStatus: 'released' }) - expect(await send('after the upgrade')).toMatchObject({ ok: true }) + expect(await delivered('after the upgrade')).toMatchObject({ dispatchState: 'pending' }) + expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ runtimeKind: 'native', claimStatus: 'live' @@ -216,18 +244,26 @@ describe('a record an older build left mid terminal handoff', () => { claimStatus: 'conflicted', handoffStage: 'recovering' }) - // Sending and opening the chat both say what frees it: quitting that terminal agent. + // Sending and opening the chat both say what frees it: quitting that terminal agent. A send is + // accepted, then rejected by the start that cannot take the lease, and the chat's row says why. const quitTerminal = 'This chat is still open in a terminal agent (process 4242). Quit that agent to continue the chat here.' - expect(await send('while the terminal still runs')).toMatchObject({ - ok: false, - refusal: { code: 'agent_session_conflict', message: quitTerminal } + expect(await delivered('while the terminal still runs')).toMatchObject({ + dispatchState: 'rejected' }) + expect( + host + .journalSnapshot(SESSION) + .items.flatMap((item) => + item.body.kind === 'status' && item.body.tone === 'error' ? [item.body.text] : [] + ) + ).toEqual([expect.stringContaining(quitTerminal)]) const fence = store.getRecord(SESSION)?.lease.runtimeFence ?? null expect(await host.attach(CALLER, hostTestAttachParams(fence))).toMatchObject({ ok: false, refusal: { code: 'agent_session_conflict', message: quitTerminal } }) + expect(dispatch).not.toHaveBeenCalled() expect(stopOwnerProcess).not.toHaveBeenCalled() expect(acquire).not.toHaveBeenCalled() @@ -236,7 +272,10 @@ describe('a record an older build left mid terminal handoff', () => { await host.hold(SESSION, 'surface-1') expect(stopOwnerProcess).not.toHaveBeenCalled() - expect(await send('after the terminal closed')).toMatchObject({ ok: true }) + expect(await delivered('after the terminal closed')).toMatchObject({ + dispatchState: 'pending' + }) + expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ runtimeKind: 'native', claimStatus: 'live', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts index 62be8900159..24f341d28da 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts @@ -107,7 +107,8 @@ export async function admitAndRunAgentSessionMutation( envelope, hostFingerprint, now: request.now(), - ...(plan.operationIdScope ? { operationIdScope: plan.operationIdScope } : {}) + ...(plan.operationIdScope ? { operationIdScope: plan.operationIdScope } : {}), + ...(plan.conversationWrite ? { conversationWrite: true } : {}) }) if (!admitted) { return refuseAgentSessionMutation(AGENT_SESSION_NOT_ATTACHED) @@ -141,7 +142,8 @@ export async function admitAndRunAgentSessionMutation( envelope, hostFingerprint, ledger: { decision: 'admit', row: admission.row }, - lease: record.lease + lease: record.lease, + ...(plan.conversationWrite ? { conversationWrite: true } : {}) }) if (rerun.decision === 'refused') { return refuseAgentSessionMutation(rerun.refusal) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-plans.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-plans.ts index 226881d38bf..fd56944f3bd 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-plans.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-plans.ts @@ -29,8 +29,9 @@ export type MutationPlan = { method: string fields: Record operationIdScope?: 'global' + /** Admitted without the writer lease: see `admitAgentSessionMutation`. */ + conversationWrite?: true markUnknownBeforeRun?: boolean - beforeRun?: () => void run: (ctx: AgentSessionTurnContext) => Promise> replay: (ctx: AgentSessionTurnContext, outcome: AgentSessionOperationOutcome) => TValue | null rerunWhenReplayMissing?: (ctx: AgentSessionTurnContext) => boolean @@ -50,19 +51,22 @@ export function sendPlan(params: { return { method: 'agentSession.send', operationIdScope: 'global', + conversationWrite: true, markUnknownBeforeRun: true, // A control signal is not payload; it cannot alter durable replay. fields: { body: params.body }, - ...(params.beforeRun ? { beforeRun: params.beforeRun } : {}), recoverUnknownFromDurableState: true, // `retryUnknown` is a compatibility-only client signal. A recorded send // always replays and never reaches the provider twice. - run: (ctx) => - performSend(ctx, { + run: (ctx) => { + // Asked at acceptance: a send accepted after this one is queued behind it. + params.beforeRun?.() + return performSend(ctx, { clientMessageId, payloadFingerprint: params.envelope.payloadFingerprint, body: params.body - }), + }) + }, replay: (ctx, outcome) => { const submission = ctx.journal .submissions() @@ -101,6 +105,8 @@ export function cancelPlan(params: { }): MutationPlan { return { method: 'agentSession.cancel', + // Stop is a conversation write; a prompt or background-task cancel needs the live child. + ...(params.scope || params.prompt ? {} : { conversationWrite: true as const }), fields: { turnId: params.turnId, ...(params.scope ? { scope: params.scope } : {}), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.test.ts index b1eb5704f01..094ddffdcb2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.test.ts @@ -2,7 +2,6 @@ import { join } from 'node:path' import { afterEach, expect, it, vi } from 'vitest' import { AgentSessionPreDispatchError, - AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS, runSettledAgentSessionMutation } from './structured-agent-session-operation-settlement' import { @@ -42,6 +41,9 @@ async function context(): Promise { } } +/** Longer than any bookkeeping bound: a refusal must already be answered by then. */ +const SETTLED_WAIT_MS = 2_000 + afterEach(() => { vi.useRealTimers() vi.restoreAllMocks() @@ -80,7 +82,7 @@ it('returns a pre-dispatch refusal without waiting on redundant uncertainty pers }) try { await refusing.promise - await vi.advanceTimersByTimeAsync(AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS) + await vi.advanceTimersByTimeAsync(SETTLED_WAIT_MS) expect(returned).toBe(true) expect(writes).toHaveBeenCalledOnce() expect(hostTestState().dispatch).not.toHaveBeenCalled() @@ -129,9 +131,10 @@ it.each([1, 2])( } ) -// The pre-dispatch check judges only what the journal already holds; the send path never waits on -// the provider's stream barrier, so a sink that stalls or fails cannot delay or double a send. -it('dispatches without touching the event-stream barrier', async () => { +// A send's plan only accepts: it records the submission and never waits on the provider's stream +// barrier, so a sink that stalls or fails cannot delay or double a send. Handing it over is the +// delivery loop's. +it('accepts without touching the event-stream barrier or the provider', async () => { const ctx = await context() const { store } = hostTestState() vi.spyOn(store, 'recordOperationOutcome').mockResolvedValue() @@ -149,42 +152,33 @@ it('dispatches without touching the event-stream barrier', async () => { }) expect(result).toMatchObject({ ok: true }) expect(beforeRun).toHaveBeenCalledOnce() - expect(hostTestState().dispatch).toHaveBeenCalledOnce() - expect(ctx.journal.submissions()[0]?.dispatchState).toBe('accepted') + expect(hostTestState().dispatch).not.toHaveBeenCalled() + expect(ctx.journal.submissions()[0]).toMatchObject({ + dispatchState: 'pending', + handoverRecorded: true + }) expect(barrier).not.toHaveBeenCalled() }) -it('refuses a superseded send without waiting on a stalled refusal write, and never dispatches late', async () => { +it('refuses a superseded send at acceptance, recording and dispatching nothing', async () => { const ctx = await context() const { store } = hostTestState() vi.spyOn(store, 'recordOperationOutcome').mockResolvedValue() - const pending = Promise.withResolvers() - const refusing = Promise.withResolvers() - vi.spyOn(ctx.journal, 'resolveDispatch').mockImplementationOnce(() => { - refusing.resolve() - return pending.promise.then(() => ctx.journal.cursor()) - }) - vi.spyOn(console, 'warn').mockImplementation(() => {}) const beforeRun = vi.fn(() => { throw new AgentSessionPreDispatchError('agent_session_restart_work_superseded') }) const body = hostTestMessage('Continue the interrupted work') const operation = envelope('agentSession.send', { body }) vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) - const result = runSettledAgentSessionMutation({ + const result = await runSettledAgentSessionMutation({ store, operationCallerKey: 'test', envelope: operation, context: ctx, plan: sendPlan({ envelope: operation, body, beforeRun }) }).catch((error: unknown) => error) - await refusing.promise - await vi.advanceTimersByTimeAsync(AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS) - expect(await result).toBeInstanceOf(AgentSessionPreDispatchError) - expect(hostTestState().dispatch).not.toHaveBeenCalled() - expect(ctx.journal.submissions()[0]?.dispatchState).toBe('pending') - pending.resolve() - await vi.advanceTimersByTimeAsync(0) + expect(result).toBeInstanceOf(AgentSessionPreDispatchError) + expect(ctx.journal.submissions()).toEqual([]) expect(hostTestState().dispatch).not.toHaveBeenCalled() expect(vi.getTimerCount()).toBe(0) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.ts index baf871aef65..5d5009aa918 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-operation-settlement.ts @@ -11,8 +11,6 @@ export class AgentSessionPreDispatchError extends Error { } } -export const AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS = 2_000 - export async function runSettledAgentSessionMutation(input: { store: AgentSessionRecordStore operationCallerKey: string @@ -33,10 +31,7 @@ export async function runSettledAgentSessionMutation(input: { if (input.plan.markUnknownBeforeRun) { await settle({ status: 'unknown' }) } - outcome = await input.plan.run({ - ...input.context, - ...(input.plan.beforeRun ? { beforeDispatch: input.plan.beforeRun } : {}) - }) + outcome = await input.plan.run(input.context) await settle( outcome.ok ? (input.plan.settledOutcome?.(outcome.value) ?? { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-option-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-option-settlement.test.ts index 7957ed326a8..853fa5784a0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-option-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-option-settlement.test.ts @@ -152,6 +152,11 @@ describe('structured session options and close', () => { envelope: envelope('agentSession.send', { body }), body }) + // Accepted, then handed over by the delivery loop; the status is read once it answered. + await vi.waitFor(() => expect(dispatchedModels).toEqual([DEFAULT_MODEL])) + await vi.waitFor(() => + expect(host.journalSnapshot(SESSION).submissions[0]?.dispatchState).toBe('accepted') + ) const events: AgentSessionStatusEvent[] = [] host.subscribeStatus({ id: 'session-list', emit: (event) => events.push(event) }) expect(events).toEqual([ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-owed-work-release.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-owed-work-release.test.ts index d2d5d746469..459cd60bcc3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-owed-work-release.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-owed-work-release.test.ts @@ -2,9 +2,11 @@ // child that still owes the user work. // // A Claude chat is published before its CLI answers initialize, and a message sent in that window -// is held until it does; evicting then refuses a message the user already sent. And a lead whose -// turn has settled can leave subagents, commands and monitors running inside the child; evicting -// then ends them silently. +// is accepted and stays queued until it does; the delivery loop hands it over once startup lands. +// Switching away from the chat starts the release clock; the clock must treat that queued message +// as work still owed, exactly as it treats a running turn, or it evicts the session and rejects a +// message the user already sent. And a lead whose turn has settled can leave subagents, commands +// and monitors running inside the child; evicting then ends them silently. import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -65,7 +67,7 @@ beforeEach(async () => { lifecycle.push(host.handleAdapterEvent(mapped)) } }, - // As the runtime wires it: a held prompt's outcome reaches the journal out of band. + // As the runtime wires it: an admitted prompt's outcome reaches the journal out of band. onDispatchSettledLate: (settlement) => void host.settleLateDispatch(settlement), // Initialize answers only when the test says so. openConnection: async (launch, handlers) => { @@ -139,13 +141,21 @@ function dispatchState(clientMessageId: string): string | undefined { .submissions.find((entry) => entry.clientMessageId === clientMessageId)?.dispatchState } +/** The delivery loop hands a message over on its own serialized steps after startup lands; this + * yields to them without advancing the (possibly faked) release clock. */ +async function untilSent(connection: { sent: unknown[] }): Promise { + for (let turn = 0; turn < 2000 && connection.sent.length === 0; turn += 1) { + await new Promise((resolve) => setImmediate(resolve)) + } +} + /** Long enough for several grace windows to elapse, so "not evicted" means the clock declined. */ function waitOutSeveralGraceWindows(): Promise { return new Promise((resolve) => setTimeout(resolve, GRACE_MS * 20)) } describe('a chat left while its Claude CLI is still starting', () => { - it('keeps the session for a message it is holding, and delivers it once startup lands', async () => { + it('keeps the session for a message still queued, and delivers it once startup lands', async () => { await attachStarting() const held = await send('sent while starting') @@ -157,9 +167,12 @@ describe('a chat left while its Claude CLI is still starting', () => { expect(dispatchState(held)).toBe('pending') landInit() - await adapter.drainStartup(SESSION) + await adapter.awaitStarted(SESSION) - expect(claude.connections[0].sent).toEqual([expect.objectContaining({ type: 'user' })]) + await vi.waitFor( + () => expect(claude.connections[0].sent).toEqual([expect.objectContaining({ type: 'user' })]), + { timeout: 3000 } + ) await vi.waitFor(() => expect(dispatchState(held)).toBe('accepted')) }) @@ -178,8 +191,9 @@ describe('a chat left while its Claude CLI is still starting', () => { // Startup lands just before the clock's next tick. landInit() - await adapter.drainStartup(SESSION) + await adapter.awaitStarted(SESSION) await Promise.all(lifecycle) + await untilSent(connection) expect(connection.sent).toEqual([expect.objectContaining({ type: 'user' })]) await vi.advanceTimersByTimeAsync(GRACE_MS - 1) @@ -195,8 +209,9 @@ describe('a chat left while its Claude CLI is still starting', () => { it('is released after the grace once its turn has finished', async () => { await attachStarting() landInit() - await adapter.drainStartup(SESSION) - await send('answered', 'accepted') + await adapter.awaitStarted(SESSION) + const answered = await send('answered') + await vi.waitFor(() => expect(dispatchState(answered)).toBe('accepted')) claude.connections[0].handlers.onMessage?.({ type: 'result', subtype: 'success', @@ -232,8 +247,9 @@ describe('a chat left while its settled lead still has background work running', async function settleTurnLeavingTask(taskType: string): Promise { await attachStarting() landInit() - await adapter.drainStartup(SESSION) - await send('fan out', 'accepted') + await adapter.awaitStarted(SESSION) + const fanOut = await send('fan out') + await vi.waitFor(() => expect(dispatchState(fanOut)).toBe('accepted')) frame({ type: 'system', subtype: 'task_started', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-processless-reservation.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-processless-reservation.test.ts index e0fa13b3362..d1ab41ccaef 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-processless-reservation.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-processless-reservation.test.ts @@ -12,6 +12,7 @@ import { attachFingerprintFields, type AgentSessionAttachParams } from './structured-agent-session-attach' +import { openTestAttachConversation } from './structured-agent-session-attach-test-conversation' import { performAttach } from './structured-agent-session-attach-flow' const NOW = 1_800_000_000_000 @@ -86,6 +87,7 @@ describe('processless structured session reservation', () => { store, adapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -133,6 +135,7 @@ describe('processless structured session reservation', () => { store, adapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -172,6 +175,7 @@ describe('processless structured session reservation', () => { store, adapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-drift', claimKeyId: 'key-1', @@ -224,6 +228,7 @@ describe('processless structured session reservation', () => { store, adapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', @@ -290,6 +295,7 @@ describe('processless structured session reservation', () => { store, adapter, journalRoot: root, + openConversation: openTestAttachConversation(root!), authority: { spawnToken: 'spawn-a', claimKeyId: 'key-1', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts new file mode 100644 index 00000000000..10d60791781 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts @@ -0,0 +1,691 @@ +// The provider child is its own record on the conversation: stopping it, losing it or failing to +// start it ends the child, never the conversation. Against the real host, store and journal, with a +// live subscriber opened before each action. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' +import type { + AgentSessionStatusSummary, + AgentSessionSubscribeEvent +} from '../../../shared/agent-session-wire' +import { + DISPATCH_REJECTED_CANCELLED, + DISPATCH_REJECTED_PROVIDER_CLOSED +} from '../../../shared/structured-agent-session-dispatch-rejection' +import { + providerStartupFailureOutcome, + unexpectedProviderExitOutcome +} from './structured-agent-session-dead-generation-settlement' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' +import { journalDirectoryFor } from '../agent-session-journal/journal-paths' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { stopStructuredAgentSessionAgentUnderSerialize } from './structured-agent-session-host-lifetime' +import { + HOST_TEST_LOCATION, + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +const CALLER = { callerKey: 'client-1' } + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost +let acquire: Mock +let dispatch: Mock +let adapterExtras: Partial + +function eventually(assertion: () => void | Promise): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} + +function generation(): string { + return `generation-${acquire.mock.calls.length}` +} + +const spawnChild: StructuredAgentSessionAdapter['acquire'] = async ({ fence, spawnToken }) => ({ + process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, + acquisitionGeneration: generation(), + link: { + linkId: `link-${fence}`, + handle: { provider: 'codex' as const, threadId: THREAD }, + origin: store.getRecord(SESSION)?.providerHandleChain.length + ? ('resumed' as const) + : ('created' as const), + mintedAtFence: fence, + observedAt: NOW + } +}) + +/** A Claude-shaped child: published at spawn, so it is `starting` until `started`. */ +const spawnStartingChild: StructuredAgentSessionAdapter['acquire'] = async (input) => ({ + ...(await spawnChild(input)), + providerChildPhase: 'starting' as const +}) + +function startHost(): void { + host = new StructuredAgentSessionHost({ + store, + adapter: { + acquire, + dispatch, + closeSession: vi.fn(async () => true), + releaseAcquisition: vi.fn(async () => true), + cancelTurn: vi.fn(async () => ({ cancelled: false })), + answerPrompt: vi.fn(async () => undefined), + setOption: vi.fn(async () => undefined), + ...adapterExtras + }, + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => `spawn-${acquire.mock.calls.length}`, + releaseGraceMs: 60_000, + now: () => NOW + }) +} + +async function restartHost(): Promise { + await host.flushAllStreamedEvents() + startHost() +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-child-record-')) + resetHostTestOperationIds() + adapterExtras = {} + acquire = vi.fn(spawnChild) + dispatch = vi.fn(async (input) => ({ + state: 'accepted' as const, + providerIdentity: { + provider: 'codex' as const, + threadId: THREAD, + turnId: `turn-${input.clientMessageId}`, + ordinal: dispatch.mock.calls.length + } + })) + store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) + startHost() + expect(await host.attach(CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true }) + await host.close(SESSION) +}) + +afterEach(async () => { + await host.flushAllStreamedEvents() + await rm(root, { recursive: true, force: true }) +}) + +function sendParams(text: string) { + const body = hostTestMessage(text) + return { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: 1, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + }, + body + } +} + +async function accept(text: string): Promise { + const params = sendParams(text) + const sent = await host.send(CALLER, params) + expect(sent).toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) + return params.envelope.clientOperationId +} + +function stop() { + const turnId = 'turn-none' + return host.cancel(CALLER, { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: null, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.cancel', + sessionId: SESSION, + fields: { turnId } + }) + }, + turnId + }) +} + +function submission(id: string): AgentJournalSubmission | undefined { + return host.journalSnapshot(SESSION).submissions.find((entry) => entry.clientMessageId === id) +} + +function statusRows(): { itemId: string; text: string; tone?: string }[] { + return host.journalSnapshot(SESSION).items.flatMap((item) => + item.body.kind === 'status' + ? [ + { + itemId: item.itemId, + text: item.body.text, + ...(item.body.tone ? { tone: item.body.tone } : {}) + } + ] + : [] + ) +} + +/** The child the conversation has now, as its lifecycle events name it. */ +function currentChild() { + const child = conversation()?.child + if (!child?.generation) { + throw new Error('no child indexed') + } + return { sessionId: SESSION, fence: child.fence, acquisitionGeneration: child.generation } +} + +function exit(child: ReturnType, reason: string, startupUnproven?: true) { + return host.handleAdapterEvent({ + type: 'ended', + ...child, + reason, + cause: 'unexpected-exit', + ...(startupUnproven ? { startupUnproven } : {}) + }) +} + +function rejectedIn(events: AgentSessionSubscribeEvent[], id: string): boolean { + return events.some( + (event) => + event.type === 'batch' && + event.batch.submissions.some( + (entry) => entry.clientMessageId === id && entry.dispatchState === 'rejected' + ) + ) +} + +function conversation() { + return host['sessions'].get(SESSION) +} + +function subscribe(): AgentSessionSubscribeEvent[] { + const events: AgentSessionSubscribeEvent[] = [] + host.subscribe({ + id: 'sub-1', + sessionId: SESSION, + emit: (event) => events.push(structuredClone(event)) + }) + return events +} + +/** The chat's status row as a session list sees it, frame by frame. */ +function watchStatus(): AgentSessionStatusSummary[] { + const frames: AgentSessionStatusSummary[] = [] + host.subscribeStatus({ + id: 'list-1', + emit: (event) => { + if (event.type === 'status' && event.session.sessionId === SESSION) { + frames.push(event.session) + } + } + }) + return frames +} + +function deferred() { + let resolve!: (value: T) => void + const promise = new Promise((next) => { + resolve = next + }) + return { promise, resolve } +} + +describe('Stop on a child still proving its start', () => { + it('ends the child and keeps the conversation, its holders and its readers (R1)', async () => { + const ended = deferred() + adapterExtras = { + awaitStarted: vi.fn(() => ended.promise), + closeSession: vi.fn(async () => { + ended.resolve() + return true + }) + } + await restartHost() + acquire.mockImplementationOnce(spawnStartingChild) + await host.hold(SESSION, 'surface-1', { resume: false }) + const first = await accept('hello') + const journal = conversation()?.journal + const events = subscribe() + const frames = watchStatus() + await eventually(() => expect(conversation()?.child?.phase).toBe('starting')) + + expect(await stop()).toMatchObject({ ok: true, value: { cancelled: true } }) + + // The same conversation: no reopen, the holder kept, and the chat told it is idle again. + expect(conversation()?.journal).toBe(journal) + expect(conversation()?.child).toBeNull() + expect(conversation()?.lastEndedChild).toMatchObject({ cause: 'user-stop', rootGone: true }) + expect(host.isHeld(SESSION)).toBe(true) + expect(frames.at(-1)).not.toHaveProperty('hostExecutionPhase') + expect(frames.at(-1)).not.toHaveProperty('hostExecutionOwned') + expect(submission(first)).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_CANCELLED + }) + // A Stop is not a failure: no row, and the loop is gone. + expect(statusRows()).toEqual([]) + await eventually(() => expect(host['conversationDelivery'].loop.isRunning(SESSION)).toBe(false)) + + const next = await accept('after stop') + await eventually(() => expect(submission(next)?.dispatchState).toBe('accepted')) + expect(conversation()?.journal).toBe(journal) + expect(dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([next]) + // The reader opened before the Stop saw the next message delivered on the same stream. + expect( + events.some( + (event) => + event.type === 'batch' && + event.batch.submissions.some( + (entry) => entry.clientMessageId === next && entry.dispatchState === 'accepted' + ) + ) + ).toBe(true) + }) +}) + +describe('settling an earlier child before the next one takes its message', () => { + it("settles the earlier child's turn from its death evidence and leaves the queued message to the new child (R1)", async () => { + // The earlier child exited mid-turn; the released lease keeps only its death evidence. + await store.transitionHandoff(SESSION, (record) => ({ + ...record, + lease: { + ...record.lease, + deathEvidence: { kind: 'exit-observed', detail: 'provider exited', observedAt: NOW - 1_000 } + } + })) + const releasedFence = store.getRecord(SESSION)!.lease.runtimeFence + const journal = await openAgentSessionJournal({ + identity: { + sessionId: SESSION, + workspaceId: HOST_TEST_LOCATION.workspaceId, + hostId: HOST_TEST_LOCATION.executionHostId, + agent: 'codex', + providerHandle: { kind: 'codex', threadId: THREAD } + }, + journalDir: journalDirectoryFor(root, { + workspaceId: HOST_TEST_LOCATION.workspaceId, + sessionId: SESSION + }) + }) + await journal.appendItem( + { provider: 'codex', threadId: THREAD, turnId: 'earlier-turn', ordinal: 0 }, + { kind: 'turn', turnId: 'earlier-turn', state: 'running', startedAt: NOW - 5_000 }, + { fence: releasedFence } + ) + await journal.close() + const id = await accept('for the next child') + + await eventually(() => expect(submission(id)?.dispatchState).toBe('accepted')) + // The exit was observed, so its receipt ends the turn, and the chat says why it stopped. + const items = conversation()!.journal.snapshot().items + expect(items.map((item) => readAgentJournalTurn(item.body)).filter(Boolean)).toContainEqual( + expect.objectContaining({ + turnId: 'earlier-turn', + state: 'interrupted', + completedAt: NOW - 1_000 + }) + ) + expect( + items.flatMap((item) => (item.body.kind === 'status' ? [item.body.text] : [])) + ).toContain(unexpectedProviderExitOutcome('provider exited')) + // Handed over at the new child's fence, which the attach reserved after settling. + const newFence = store.getRecord(SESSION)!.lease.runtimeFence + expect(newFence).toBeGreaterThan(releasedFence) + expect(submission(id)?.fence).toBe(newFence) + expect(conversation()?.child).toMatchObject({ generation: generation(), fence: newFence }) + }) +}) + +describe('a published child that dies while it proves its start', () => { + const EXIT = 'claude stream-json exited (code 1)' + const TEXT = providerStartupFailureOutcome(EXIT) + + it.each([['the loop sees the start fail first'], ['the exit is processed first']])( + 'leaves one error row keyed by the start, and every queued message rejected with it: %s (R2)', + async (order) => { + const settled = deferred() + adapterExtras = { awaitStarted: vi.fn(() => settled.promise) } + await restartHost() + acquire.mockImplementation(spawnStartingChild) + const first = await accept('first') + const events = subscribe() + const second = await accept('second') + await eventually(() => expect(adapterExtras.awaitStarted).toHaveBeenCalled()) + const child = currentChild() + + if (order === 'the exit is processed first') { + await exit(child, EXIT, true) + settled.resolve(TEXT) + } else { + settled.resolve(TEXT) + await eventually(() => expect(submission(second)?.dispatchState).toBe('rejected')) + await exit(child, EXIT, true) + } + + await eventually(() => expect(submission(second)?.dispatchState).toBe('rejected')) + expect(statusRows()).toEqual([ + { + itemId: `orca:${encodeURIComponent(`start-failure:${child.acquisitionGeneration}`)}`, + text: TEXT, + tone: 'error' + } + ]) + expect(submission(first)).toMatchObject({ dispatchState: 'rejected', reason: TEXT }) + expect(submission(second)).toMatchObject({ dispatchState: 'rejected', reason: TEXT }) + expect(rejectedIn(events, second)).toBe(true) + expect(dispatch).not.toHaveBeenCalled() + expect(acquire).toHaveBeenCalledTimes(2) + } + ) +}) + +describe("a view's start that dies while a sent message waits on it", () => { + const EXIT = 'claude stream-json exited (code 1)' + const TEXT = providerStartupFailureOutcome(EXIT) + + it("is the message's own failed start: one error row, the message rejected, no second start (R2)", async () => { + adapterExtras = { awaitStarted: vi.fn(async () => TEXT) } + await restartHost() + acquire.mockImplementation(spawnStartingChild) + // Opening the tab: the view's hold starts a child that has not proven its start. + await host.hold(SESSION, 'surface-1') + const viewChild = currentChild() + const events = subscribe() + const params = sendParams('hello') + + // Accepted first; the view's child's exit is settled before the loop's first step. + const sent = host.send(CALLER, params) + const exited = exit(viewChild, EXIT, true) + expect(await sent).toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'pending' } } + }) + await exited + const id = params.envelope.clientOperationId + + await eventually(() => expect(submission(id)?.dispatchState).toBe('rejected')) + await settleLoop() + expect(submission(id)?.reason).toBe(TEXT) + expect(statusRows()).toEqual([ + { + itemId: `orca:${encodeURIComponent(`start-failure:${viewChild.acquisitionGeneration}`)}`, + text: TEXT, + tone: 'error' + } + ]) + expect(rejectedIn(events, id)).toBe(true) + // The setup's child and the view's: nothing started again into the same failure. + expect(acquire).toHaveBeenCalledTimes(2) + expect(dispatch).not.toHaveBeenCalled() + }) + + it('leaves a message sent after that start failed to a fresh start (R2)', async () => { + await restartHost() + acquire.mockImplementationOnce(spawnStartingChild) + await host.hold(SESSION, 'surface-1') + await exit(currentChild(), EXIT, true) + expect(statusRows()).toHaveLength(1) + + const id = await accept('after the failure') + + await eventually(() => expect(submission(id)?.dispatchState).toBe('accepted')) + expect(acquire).toHaveBeenCalledTimes(3) + }) + + it('starts again for a message whose proven child crashed: only a failed start settles it (R2)', async () => { + await restartHost() + await host.hold(SESSION, 'surface-1') + const params = sendParams('hello') + + const sent = host.send(CALLER, params) + const exited = exit(currentChild(), 'codex app-server crashed') + await sent + await exited + const id = params.envelope.clientOperationId + + await eventually(() => expect(submission(id)?.dispatchState).not.toBe('pending')) + expect(submission(id)?.dispatchState).toBe('accepted') + expect(acquire).toHaveBeenCalledTimes(3) + }) + + it('waits on a child started since the failed one, not on the failure (R2)', async () => { + adapterExtras = { awaitStarted: vi.fn(async () => undefined) } + await restartHost() + acquire.mockImplementation(spawnStartingChild) + await host.hold(SESSION, 'surface-1') + const params = sendParams('hello') + + // A client's attach lands after the first child's exit, before the loop's first step: a view + // no longer starts a child whose last start failed, but an attach still does. + const sent = host.send(CALLER, params) + const exitedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + const exited = exit(currentChild(), EXIT, true) + const attached = host.attach(CALLER, hostTestAttachParams(exitedFence + 1)) + await sent + await exited + await expect(attached).resolves.toMatchObject({ ok: true }) + const id = params.envelope.clientOperationId + + await eventually(() => expect(submission(id)?.dispatchState).not.toBe('pending')) + expect(submission(id)?.dispatchState).toBe('accepted') + expect(acquire).toHaveBeenCalledTimes(3) + }) +}) + +describe('a child that ends before its message is handed over', () => { + it('starts one child for the message, then rejects it and stops (R2)', async () => { + // The child the loop starts exits between its start step and its handover step. + adapterExtras = { + awaitStarted: vi.fn(async () => { + await exit(currentChild(), 'codex app-server crashed') + }) + } + await restartHost() + const id = await accept('hello') + const events = subscribe() + + await eventually(() => expect(submission(id)?.dispatchState).toBe('rejected')) + expect(submission(id)?.reason).toContain('codex app-server crashed') + expect(statusRows()).toEqual([ + { itemId: expect.any(String), text: submission(id)?.reason, tone: 'error' } + ]) + expect(rejectedIn(events, id)).toBe(true) + await eventually(() => expect(host['conversationDelivery'].loop.isRunning(SESSION)).toBe(false)) + expect(acquire).toHaveBeenCalledTimes(2) + expect(dispatch).not.toHaveBeenCalled() + }) +}) + +describe('another child indexed while the loop waits on the one it started', () => { + it('hands nothing over until the child now there has proven its start (R2)', async () => { + const starts = new Map>>() + const startOf = (generation: string) => { + const start = starts.get(generation) ?? deferred() + starts.set(generation, start) + return start + } + adapterExtras = { + awaitStarted: vi.fn(() => startOf(currentChild().acquisitionGeneration).promise), + // The stop ends the child without settling the start the loop is waiting on. + closeSession: vi.fn(async () => true) + } + await restartHost() + acquire.mockImplementation(spawnStartingChild) + const first = await accept('first') + await eventually(() => expect(adapterExtras.awaitStarted).toHaveBeenCalledTimes(1)) + const stopped = currentChild() + expect(await stop()).toMatchObject({ ok: true }) + const second = await accept('second') + // A view's hold starts its own child before the loop's handover step runs. + await host.hold(SESSION, 'surface-1') + const replacement = currentChild() + expect(replacement.acquisitionGeneration).not.toBe(stopped.acquisitionGeneration) + + startOf(stopped.acquisitionGeneration).resolve() + await eventually(() => expect(adapterExtras.awaitStarted).toHaveBeenCalledTimes(2)) + expect(dispatch).not.toHaveBeenCalled() + + await host.handleAdapterEvent({ + type: 'started', + ...replacement, + reportedOptions: { model: 'sonnet' }, + restoreSkippedOptions: [] + }) + startOf(replacement.acquisitionGeneration).resolve() + + await eventually(() => expect(submission(second)?.dispatchState).toBe('accepted')) + expect(dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([second]) + expect(submission(first)).toMatchObject({ reason: DISPATCH_REJECTED_CANCELLED }) + }) +}) + +describe('a quit with a message still queued', () => { + /** Read by the next launch, through the same open any reader takes. */ + async function afterRelaunch(id: string): Promise { + startHost() + await host.revealSession(SESSION) + return submission(id) + } + + it('settles a message no child ever had the way a chat close does (R2)', async () => { + // Quit has begun — its first step stops the delivery loops — when this message is accepted. + host['conversationDelivery'].loop.dispose() + const id = await accept('hello') + expect(conversation()?.child).toBeNull() + await host.flushAllStreamedEvents() + + expect(await afterRelaunch(id)).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_PROVIDER_CLOSED + }) + }) + + it('waits for the start already in flight and stops the child it produced (R2)', async () => { + const starting = deferred() + const closeSession = vi.fn(async () => true) + adapterExtras = { closeSession } + await restartHost() + // The loop's start step is under way, but has not reached its attach yet. + const resolveRecovery = host['runtimeState'].resolveRecovery.bind(host['runtimeState']) + const recovering = vi.spyOn(host['runtimeState'], 'resolveRecovery') + recovering.mockImplementationOnce(async (sessionId) => { + await starting.promise + return resolveRecovery(sessionId) + }) + const id = await accept('hello') + await eventually(() => expect(recovering).toHaveBeenCalled()) + + const quit = host.flushAllStreamedEvents() + starting.resolve() + await quit + + expect(closeSession).toHaveBeenCalledWith(SESSION) + expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'released' }) + expect(dispatch).not.toHaveBeenCalled() + expect(await afterRelaunch(id)).toMatchObject({ + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_PROVIDER_CLOSED + }) + }) +}) + +describe('a send whose start failed, sent again with the same operation id', () => { + it('replays the recorded rejection and starts no second agent (R2)', async () => { + acquire.mockRejectedValueOnce(new Error('spawn claude ENOENT')) + const fenceBefore = store.getRecord(SESSION)!.lease.runtimeFence + const params = sendParams('hello') + // A failed start is a rejected message, never a refused send. + expect(await host.send(CALLER, params)).toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'pending' } } + }) + const id = params.envelope.clientOperationId + await eventually(() => expect(submission(id)?.dispatchState).toBe('rejected')) + // The start moved the fence while the message was out. + expect(store.getRecord(SESSION)!.lease.runtimeFence).toBeGreaterThan(fenceBefore) + const starts = acquire.mock.calls.length + + const replay = await host.send(CALLER, params) + + expect(replay).toMatchObject({ + ok: true, + replayed: true, + value: { submission: { clientMessageId: id, dispatchState: 'rejected' } } + }) + await settleLoop() + expect(acquire).toHaveBeenCalledTimes(starts) + expect(dispatch).not.toHaveBeenCalled() + }) +}) + +async function settleLoop(): Promise { + await eventually(() => expect(host['conversationDelivery'].loop.isRunning(SESSION)).toBe(false)) +} + +describe('how a stopped child ends the start its loop was waiting on', () => { + /** A child the loop waits on, whose start the stop below does not settle, so a message sent + * after the stop is queued when the loop next looks. */ + async function stoppedWhileStarting(stop: () => Promise) { + const start = deferred() + adapterExtras = { + awaitStarted: vi.fn(() => start.promise), + closeSession: vi.fn(async () => true) + } + await restartHost() + acquire.mockImplementationOnce(spawnStartingChild) + await accept('first') + await eventually(() => expect(adapterExtras.awaitStarted).toHaveBeenCalledTimes(1)) + await stop() + const second = await accept('second') + adapterExtras.awaitStarted = undefined + start.resolve() + return second + } + + it("goes on after a user's Stop and delivers what was sent since (R2)", async () => { + const second = await stoppedWhileStarting(async () => { + expect(await stop()).toMatchObject({ ok: true }) + }) + + await eventually(() => expect(submission(second)?.dispatchState).toBe('accepted')) + expect(conversation()?.lastEndedChild).toMatchObject({ cause: 'user-stop', reason: null }) + expect(statusRows()).toEqual([]) + }) + + it('fails the start after a host stop, with the stop as the reason (R2)', async () => { + const reason = 'Claude never finished starting, so Orca stopped it.' + const second = await stoppedWhileStarting(() => + host['serialize'](SESSION, () => + stopStructuredAgentSessionAgentUnderSerialize(host['lifetimeContext'](), SESSION, { + cause: 'host-stop', + reason + }) + ) + ) + + await eventually(() => expect(submission(second)?.dispatchState).toBe('rejected')) + expect(submission(second)?.reason).toBe(reason) + expect(statusRows()).toEqual([{ itemId: expect.any(String), text: reason, tone: 'error' }]) + expect(dispatch).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts new file mode 100644 index 00000000000..6c469e4dacc --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts @@ -0,0 +1,78 @@ +// The provider child behind a conversation, kept as its own record on the conversation's entry. +// +// The entry is the conversation and outlives any number of children. A child enters only when an +// attach has fully succeeded, and leaves only through `endProviderChild`, which every ending shares: +// an exit, a failed re-attach, a Stop and an eviction. Each is matched on the child's generation +// and fence, so an ending that arrives late for an older child cannot end a newer one. + +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { + StructuredAgentSessionEndedChild, + StructuredAgentSessionHostSession, + StructuredAgentSessionProviderChild, + StructuredAgentSessionProviderChildIdentity +} from './structured-agent-session-host-types' + +type ChildBearer = Pick & { + journal: Pick +} + +/** The fence a conversation write carries: the record's, which is where the next child starts. A + * child's own writes carry `child.fence`, which equals it while that child holds the lease. */ +export function structuredAgentSessionConversationFence( + store: Pick, + sessionId: string +): number { + return store.getRecord(sessionId)?.lease.runtimeFence ?? 0 +} + +/** For the end of a successful attach only: a failed one never wrote a child to take back. */ +export function indexProviderChild( + session: ChildBearer, + child: StructuredAgentSessionProviderChild +): void { + session.child = child +} + +export function markProviderChildStarted( + session: ChildBearer, + identity: StructuredAgentSessionProviderChildIdentity +): boolean { + const child = matchingChild(session, identity) + if (child) { + child.phase = 'ready' + } + return child !== null +} + +export function endProviderChild( + session: ChildBearer, + ended: Omit +): boolean { + if (!matchingChild(session, ended)) { + return false + } + session.child = null + session.lastEndedChild = { ...ended, endedAt: session.journal.cursor() } + return true +} + +/** The conversation's last start died before it proved itself, and nothing started since. Only a + * send retries it: a view or an exit recovery would respawn into the same failure, adding a row. */ +export function failedProviderChildStart( + session: Pick +): StructuredAgentSessionEndedChild | null { + const ended = session.lastEndedChild + return !session.child && ended?.duringStartup && ended.cause !== 'user-stop' ? ended : null +} + +function matchingChild( + session: ChildBearer, + identity: StructuredAgentSessionProviderChildIdentity +): StructuredAgentSessionProviderChild | null { + const { child } = session + return child && child.generation === identity.generation && child.fence === identity.fence + ? child + : null +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.test.ts index 52699fd062f..bb5d29d4dea 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.test.ts @@ -102,7 +102,7 @@ describe('a publish-first Claude create whose init is slow', () => { expect(store.getRecord(SESSION)?.options?.model).toBeUndefined() expect(lastPhase()).toBe('starting') - await adapter.drainStartup(SESSION) + await adapter.awaitStarted(SESSION) await Promise.all(lifecycle) expect(store.getRecord(SESSION)?.options?.model).toBe('claude-opus-9') @@ -116,7 +116,7 @@ describe('a publish-first Claude create whose init is slow', () => { ).resolves.toMatchObject({ ok: true }) expect(store.getRecord(SESSION)?.options?.model).toBe('opus') - await adapter.drainStartup(SESSION) + await adapter.awaitStarted(SESSION) await Promise.all(lifecycle) expect(store.getRecord(SESSION)?.options?.model).toBe('opus') @@ -126,7 +126,7 @@ describe('a publish-first Claude create whose init is slow', () => { it('keeps the picked model across a resume whose new child starts on its own default', async () => { const params = claudeParams() await host.attach(CALLER, { ...params, options: { model: 'opus' } }) - await adapter.drainStartup(SESSION) + await adapter.awaitStarted(SESSION) await Promise.all(lifecycle) await host.close(SESSION) const releasedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 @@ -138,7 +138,7 @@ describe('a publish-first Claude create whose init is slow', () => { expect(store.getRecord(SESSION)?.options?.model).toBe('opus') expect(lastPhase()).toBe('starting') - await adapter.drainStartup(SESSION) + await adapter.awaitStarted(SESSION) await Promise.all(lifecycle) expect(store.getRecord(SESSION)?.options?.model).toBe('opus') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts index 55c718f967b..6d6d910cc55 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.ts @@ -16,6 +16,7 @@ import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import { nativeSessionOptionsFromReport } from './structured-agent-session-option-restoration' +import { markProviderChildStarted } from './structured-agent-session-provider-child' export type StructuredAgentSessionProviderStartedContext = { deps: StructuredAgentSessionHostDeps @@ -35,13 +36,14 @@ export function settleStructuredAgentSessionProviderStarted( return context.serialize(event.sessionId, async () => { const session = context.sessions.get(event.sessionId) if ( - !session?.hasProviderChild || - session.fence !== event.fence || - session.acquisitionGeneration !== event.acquisitionGeneration + !session || + !markProviderChildStarted(session, { + generation: event.acquisitionGeneration, + fence: event.fence + }) ) { return } - session.providerChildPhase = 'ready' // Prompts held for the start are written now but open a turn only on their echo; a release // tick in between would stop the child before it runs them. context.restartReleaseGrace(event.sessionId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.test.ts index 34e3fb8a4cf..514008e49f9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.test.ts @@ -48,6 +48,11 @@ const store = { } as unknown as AgentSessionRecordStore let journalRoot: string +const openDeps = () => ({ + store, + journalRoot, + adapter: {} +}) const opened: AgentSessionJournal[] = [] async function writeRemnant(name: string): Promise { @@ -73,29 +78,29 @@ describe('a session whose journal is still the pre-SQLite format', () => { it('is published, carrying the message that explains it', async () => { const transcript = await writeRemnant('log.jsonl') - const restored = await restoreStructuredAgentSessionRead(store, journalRoot, SESSION_ID) + const restored = await restoreStructuredAgentSessionRead(openDeps(), SESSION_ID) expect(restored).not.toBeNull() - opened.push(restored!.journal) - const disclosed = restored!.journal + opened.push(restored!.session.journal) + const disclosed = restored!.session.journal .snapshot() .items.map((entry) => (entry.body.kind === 'status' ? entry.body.text : '')) expect(disclosed.join('')).toContain(transcript) // Publishing it costs no agent process; acquisition still waits for the user. - expect(restored!.hasProviderChild).toBe(false) + expect(restored!.session.child).toBeNull() }) it('is published for a remnant whose log is gone', async () => { await writeRemnant('snapshot.json') - const restored = await restoreStructuredAgentSessionRead(store, journalRoot, SESSION_ID) + const restored = await restoreStructuredAgentSessionRead(openDeps(), SESSION_ID) expect(restored).not.toBeNull() - opened.push(restored!.journal) + opened.push(restored!.session.journal) }) it('still drops a session with neither a journal nor a remnant', async () => { - const restored = await restoreStructuredAgentSessionRead(store, journalRoot, SESSION_ID) + const restored = await restoreStructuredAgentSessionRead(openDeps(), SESSION_ID) expect(restored).toBeNull() }) @@ -103,7 +108,7 @@ describe('a session whose journal is still the pre-SQLite format', () => { it('still drops a session with no record', async () => { await writeRemnant('log.jsonl') - const restored = await restoreStructuredAgentSessionRead(store, journalRoot, 'unknown-session') + const restored = await restoreStructuredAgentSessionRead(openDeps(), 'unknown-session') expect(restored).toBeNull() }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts index 65b913daf42..0e1441b729c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts @@ -1,104 +1,32 @@ -import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { findJournalFileFormatRemnant } from '../agent-session-journal/journal-file-format-remnant' -import { loadJournal } from '../agent-session-journal/journal-open' -import { journalDirectoryFor } from '../agent-session-journal/journal-paths' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' +import { existsSync } from 'node:fs' +import { journalDatabaseFile, journalDirectoryFor } from '../agent-session-journal/journal-paths' import { - attachFingerprintFields, - journalIdentityFor, - type AgentSessionAttachParams -} from './structured-agent-session-attach' -import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' - -export type RestoredStructuredAgentSessionRead = { - journal: AgentSessionJournal - params: AgentSessionAttachParams - fence: number - hasProviderChild: false - providerChildPhase: 'ready' - acquisitionGeneration: null -} + openStructuredAgentSessionConversationJournal, + type OpenedStructuredAgentSessionConversation, + type StructuredAgentSessionConversationOpenDeps +} from './structured-agent-session-conversation-open' +/** + * A reader's open: the conversation's own open, for a session that has a journal to read. One + * with none — never written, or gone — stays unpublished rather than founding an empty one. + * Opening can still write: the crash boundary, and the row explaining an old-format history. + */ export async function restoreStructuredAgentSessionRead( - store: AgentSessionRecordStore, - journalRoot: string, + deps: StructuredAgentSessionConversationOpenDeps, sessionId: string -): Promise { - const record = store.getRecord(sessionId) +): Promise { + const record = deps.store.getRecord(sessionId) if (!record) { return null } - const params = attachParamsForRecord(record, { - clientOperationId: `read-restore:${record.sessionId}`, - expectedRuntimeFence: record.lease.runtimeFence - }) - const journalDir = journalDirectoryFor(journalRoot, { + const journalDir = journalDirectoryFor(deps.journalRoot, { workspaceId: record.location.workspaceId, sessionId }) - const loaded = loadJournal(journalDir, sessionId) - if (loaded?.corrupt) { + // A session still in the pre-SQLite format has no `journal.db`; the open imports it. + if (!existsSync(journalDatabaseFile(journalDir)) && !findJournalFileFormatRemnant(journalDir)) { return null } - // A session still in the pre-SQLite format has no `journal.db` to load. Dropping - // it here leaves it unpublished, which is also what prunes its tab out of the - // saved workspace — so the chat disappears with nowhere to explain itself. - if (!loaded && !findJournalFileFormatRemnant(journalDir)) { - return null - } - const journal = await openAgentSessionJournal({ - identity: journalIdentityFor(record, params), - journalDir, - // Omitted, not `null`: the store reads `null` as "replay already ran and - // found nothing" and founds a fresh epoch. In process the probe above is the - // previous statement, so the window is zero-width; this holds the line for a - // database another process creates in between. - ...(loaded ? { loaded } : {}) - }) - // Read restore opens the journal and nothing else: no adapter call, so no - // provider child. Opening it can still write — a session whose history is in - // the old format founds its epoch and commits the row explaining that here. - return { - journal, - params, - fence: record.lease.runtimeFence, - hasProviderChild: false, - providerChildPhase: 'ready', - acquisitionGeneration: null - } -} - -export function attachParamsForRecord( - record: AgentSessionRecord, - input: { - clientOperationId: string - expectedRuntimeFence: number - } -): AgentSessionAttachParams { - const params: AgentSessionAttachParams = { - envelope: { - sessionId: record.sessionId, - clientOperationId: input.clientOperationId, - expectedRuntimeFence: input.expectedRuntimeFence, - payloadFingerprint: '' - }, - location: record.location, - provider: record.provider, - agent: record.provider, - accountHome: record.accountHome, - runtimeKind: 'native' - } - return { - ...params, - envelope: { - ...params.envelope, - payloadFingerprint: computeAgentSessionPayloadFingerprint({ - method: 'agentSession.attach', - sessionId: record.sessionId, - fields: attachFingerprintFields(params) - }) - } - } + return openStructuredAgentSessionConversationJournal(deps, record) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts index a788e1ebb66..d1cc9408a62 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts @@ -19,15 +19,17 @@ describe('StructuredAgentSessionReadableRestorer', () => { (sessionId) => ({ sessionId }) as AgentSessionRecord ) const restorer = new StructuredAgentSessionReadableRestorer({ - store: { listRecords: () => records } as never, - journalRoot: '/tmp/journals', + openDeps: { + store: { getRecord: () => null, listRecords: () => records }, + journalRoot: '/tmp/journals', + adapter: {} + }, supportsRecord: () => true, reconcile: async () => null, resolveRecovery: async () => undefined, serialize: async (_sessionId, task) => task(), hasSession: () => false, - onReadable: () => undefined, - settleStaleState: async () => undefined + onReadable: () => undefined }) await restorer.restore(['visible-a', 'visible-b', 'background-a', 'background-b']) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.ts index 5775ecc5401..ed10f63e44c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.ts @@ -1,10 +1,7 @@ import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' -import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' -import type { RestoredStructuredAgentSessionRead } from './structured-agent-session-read-restore' +import type { StructuredAgentSessionReadRestoreDeps } from './structured-agent-session-restart-restore' import { restoreOneStructuredAgentSessionRead, - restoreOneStructuredAgentSessionReadUnderSerialize, restoreStructuredAgentSessionsOnRestart } from './structured-agent-session-restart-restore' @@ -12,19 +9,8 @@ export class StructuredAgentSessionReadableRestorer { private restorePromise: Promise | null = null constructor( - private readonly input: { - store: AgentSessionRecordStore - journalRoot: string + private readonly input: StructuredAgentSessionReadRestoreDeps & { supportsRecord: (record: AgentSessionRecord) => boolean - reconcile: (sessionId: string) => Promise - resolveRecovery: (sessionId: string) => Promise - serialize: (sessionId: string, task: () => Promise) => Promise - hasSession: (sessionId: string) => boolean - onReadable: (sessionId: string, restored: RestoredStructuredAgentSessionRead) => void - settleStaleState: ( - sessionId: string, - restored: RestoredStructuredAgentSessionRead - ) => Promise } ) {} @@ -55,19 +41,8 @@ export class StructuredAgentSessionReadableRestorer { return this.input.hasSession(sessionId) } - /** `restoreOne` for a caller already inside the session's serialize. Reconciliation is skipped - * on purpose: a lease this host has not adjudicated is the attach's problem, and a replay - * needs only the journal. */ - async restoreOneUnderSerialize(sessionId: string): Promise { - if (!this.supports(sessionId)) { - return false - } - await restoreOneStructuredAgentSessionReadUnderSerialize(this.input, sessionId) - return this.input.hasSession(sessionId) - } - private supports(sessionId: string): boolean { - const record = this.input.store.getRecord(sessionId) + const record = this.input.openDeps.store.getRecord(sessionId) return record !== null && this.input.supportsRecord(record) } @@ -75,7 +50,7 @@ export class StructuredAgentSessionReadableRestorer { const targetOrder = sessionIds ? new Map(sessionIds.map((sessionId, index) => [sessionId, index])) : null - const records = this.input.store + const records = this.input.openDeps.store .listRecords() .filter( (record) => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts index 87f4f9e3d65..0d873389890 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts @@ -106,6 +106,7 @@ async function createHarness(options: { attached?: boolean } = {}) { async function abandonHost(host: StructuredAgentSessionHost): Promise { host['runtimeState'].stopLeaseRenewal() host['holds'].dispose() + host['conversationDelivery'].loop.dispose() await Promise.all([...host['sessions'].values()].map((session) => session.journal.close())) host['sessions'].clear() } @@ -225,16 +226,20 @@ const UNREACHABLE = new Set([ // StructuredAgentSessionHost.mutate maps an absent record to AGENT_SESSION_NOT_ATTACHED. 'agentSession.setOption:agent_session_identity_required', 'agentSession.send:agent_session_identity_required', - // No structured-agent-session host branch emits agent_session_journal_unreadable. + // Only a send opens the conversation it writes to. 'agentSession.setOption:agent_session_journal_unreadable', - 'agentSession.send:agent_session_journal_unreadable', // Send reconstructs doubt from its global tombstone instead of refusing it. 'agentSession.send:agent_session_operation_unknown', // Only a send restarts a lost owner. 'agentSession.setOption:agent_session_owner_restart_failed', // A write names its target, not an owner generation; only an attach compares fences. 'agentSession.setOption:agent_session_checkpoint_stale', - 'agentSession.send:agent_session_checkpoint_stale' + 'agentSession.send:agent_session_checkpoint_stale', + // A send is a conversation write: admitted whoever owns the lease, and a start it needs that + // fails rejects the accepted message rather than refusing the call. + 'agentSession.send:agent_session_conflict', + 'agentSession.send:execution_owner_reconciling', + 'agentSession.send:agent_session_owner_restart_failed' ]) describe('agentSessionRefusalOperationState host oracle', () => { @@ -251,7 +256,7 @@ describe('agentSessionRefusalOperationState host oracle', () => { expect(stale.setOption).toHaveBeenCalledTimes(1) const conflict = await createHarness() - for (const method of ['agentSession.setOption', 'agentSession.send'] as const) { + for (const method of ['agentSession.setOption'] as const) { await setLease(conflict, (current) => ({ ...current, lease: { ...current.lease, handoffStage: 'new-owner-proving' } @@ -331,7 +336,7 @@ describe('agentSessionRefusalOperationState host oracle', () => { record(await assertHostAgreement(unknown, optionUnknown, 'agent_session_operation_unknown')) const reconciling = await createHarness() - for (const method of ['agentSession.setOption', 'agentSession.send'] as const) { + for (const method of ['agentSession.setOption'] as const) { await setLease(reconciling, (current) => ({ ...current, lease: { ...current.lease, unreconciled: true } @@ -348,16 +353,21 @@ describe('agentSessionRefusalOperationState host oracle', () => { ) } - const unrecoverable = await createHarness() - await unrecoverable.host.close(SESSION) - unrecoverable.host.deps.adapter.acquire = async () => { - throw new Error('no provider thread to resume') + const unreadable = await createHarness() + await unreadable.host.close(SESSION) + unreadable.host.deps.adapter.historyFilePath = async () => { + throw new Error('transcript unreadable') } + const unreadableSend = { method: 'agentSession.send' as const, operationId: operationId() } record( await assertHostAgreement( - unrecoverable, - { method: 'agentSession.send', operationId: operationId() }, - 'agent_session_owner_restart_failed' + unreadable, + unreadableSend, + 'agent_session_journal_unreadable', + async () => { + delete unreadable.host.deps.adapter.historyFilePath + return { harness: unreadable, spec: unreadableSend } + } ) ) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation-wait.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation-wait.test.ts new file mode 100644 index 00000000000..8ab9bb250ad --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation-wait.test.ts @@ -0,0 +1,63 @@ +// The restart continuation is accepted like any send, so its verdict is its delivery: a cold start +// longer than the legacy client wait must not turn a continuation that went through into an +// "unconfirmed" one. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' +import { structuredAgentSessionRestartResumeSurfaces } from './structured-agent-session-restart-resume-wiring' +import { StructuredAgentSessionSendSettlement } from './structured-agent-session-send-settlement' + +const SESSION = 'session-1' +const MESSAGE = 'continuation-1' + +let submission: AgentJournalSubmission +const journal = { + submissions: (): AgentJournalSubmission[] => [submission], + cursor: () => ({ epoch: 'epoch-1', sequence: 1 }) +} + +beforeEach(() => { + vi.useFakeTimers() + submission = { + clientMessageId: MESSAGE, + fence: 2, + payloadFingerprint: 'fingerprint', + dispatchState: 'pending', + providerItemId: null, + reason: null, + submittedAt: 1, + resolvedAt: null, + handoverRecorded: true + } +}) + +afterEach(() => { + vi.useRealTimers() +}) + +describe('the restart continuation waits for its delivery (W18)', () => { + it('reaches accepted after a 40 s cold start, with nothing filed as unconfirmed', async () => { + const settlement = new StructuredAgentSessionSendSettlement(() => journal) + const surfaces = structuredAgentSessionRestartResumeSurfaces( + { + revealSession: async () => ({ readable: true }), + hold: async () => undefined, + release: () => undefined, + send: async () => { + throw new Error('not used') + }, + waitForSendSettlement: settlement.wait + }, + () => 0 + ) + + const verdict = surfaces.awaitSendSettlement(SESSION, MESSAGE) + await vi.advanceTimersByTimeAsync(40_000) + submission = { ...submission, dispatchState: 'accepted', providerItemId: 'item-1' } + settlement.publish(SESSION, journal) + + await expect(verdict).resolves.toMatchObject({ + value: { submission: { dispatchState: 'accepted' } } + }) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation.ts index c025c0dc46a..87a56efb7b9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-continuation.ts @@ -44,7 +44,9 @@ export type StructuredAgentSessionContinuationOutcome = { /** The slice of the host one continuation needs. Structural so this module never imports the host. */ export type StructuredAgentSessionContinuationHost = { - sessions: ReadonlyMap + sessions: ReadonlyMap + /** The fence a conversation write carries; null when the session is not open. */ + conversationFence: (sessionId: string) => number | null send: (input: { envelope: AgentSessionMutationEnvelope body: AgentJournalMessageItem @@ -72,7 +74,7 @@ export function restartContinuationDeps( marker: AgentSessionResumeMarker ): StructuredAgentSessionContinuationDeps { return { - currentFence: (sessionId) => host.sessions.get(sessionId)?.fence ?? null, + currentFence: host.conversationFence, send: (input) => host.send({ ...input, @@ -107,17 +109,18 @@ export function noteRestartReattachFailed( /** Writes a host-authored status note into the chat. */ function restartNoteWriter( - host: Pick + host: Pick ): StructuredAgentSessionContinuationDeps['note'] { return async (sessionId, text, tone) => { const session = host.sessions.get(sessionId) - if (!session) { + const fence = host.conversationFence(sessionId) + if (!session || fence === null) { return } await session.journal.appendItem( { provider: 'orca', clientMessageId: `restart-continuation:${sessionId}:${host.now()}` }, { kind: 'status', text, ...(tone ? { tone } : {}) }, - { fence: session.fence } + { fence } ) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-failure-filing.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-failure-filing.test.ts index 5dcc3ca7690..97830212b0a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-failure-filing.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-failure-filing.test.ts @@ -6,7 +6,6 @@ import { AGENT_SESSION_RESTART_NOT_CONNECTED_NOTE } from '../../../shared/agent-session-restart-continuation' import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key' -import { AgentSessionJournal } from '../agent-session-journal/journal-store' import { latestStructuredAgentSessionUserItem } from '../../../shared/structured-agent-session-projection' import { StructuredAgentSessionResumeAdmission } from './structured-agent-session-restart-resume-runner' import { @@ -39,17 +38,17 @@ it('files a continuation superseded by a replayed message, lists it and says so await host.restartResume.list() await host.hold(SESSION, 'pane') const events = providerEvents(acquire) - const append = AgentSessionJournal.prototype.appendSubmission - vi.spyOn(AgentSessionJournal.prototype, 'appendSubmission').mockImplementationOnce( - async function (this: AgentSessionJournal, input) { - const cursor = await append.call(this, input) - events.appendItem( - { provider: 'codex', threadId: THREAD, turnId: 'replayed-turn', ordinal: 1 }, - hostTestMessage('A queued notification the provider replayed') - ) - return cursor - } - ) + // The replay lands after the reattach and just before the continuation is accepted, which is + // where a send asks whether its offer still stands. + const send = host.send + vi.spyOn(host, 'send').mockImplementationOnce(async (caller, params) => { + events.appendItem( + { provider: 'codex', threadId: THREAD, turnId: 'replayed-turn', ordinal: 1 }, + hostTestMessage('A queued notification the provider replayed') + ) + await host.flushStreamedEvents(SESSION) + return send(caller, params) + }) const result = await host.restartResume.continueAfterRestart([SESSION], 'modal') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts index 1bcb27ca4a2..1d7f0033a15 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts @@ -10,7 +10,6 @@ import { } from '../../runtime/agent-session-recovery-capsule' import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { parseAgentSessionResumeMarker } from '../../../shared/agent-session-resume-marker' -import { AgentSessionJournal } from '../agent-session-journal/journal-store' import { StructuredAgentSessionHost } from './structured-agent-session-host' import { StructuredAgentSessionResumeAdmission } from './structured-agent-session-restart-resume-runner' import { @@ -52,6 +51,8 @@ export async function interruptedRestart( previous.dispatch.mockResolvedValueOnce({ state: 'admitted' }) const body = hostTestMessage('Perform the original task') await previous.host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) + // Accepted first, handed over after: the work in flight is a send the provider took. + await vi.waitFor(() => expect(previous.dispatch).toHaveBeenCalledOnce()) } else if (work === 'children') { events.appendItem( { provider: 'codex', threadId: THREAD, turnId: 'settled-turn', ordinal: 1 }, @@ -150,15 +151,17 @@ export async function supersededRefusal(userAnswers?: 'before' | 'after') { if (!events) { throw new Error('missing resumed provider event sink') } - const append = AgentSessionJournal.prototype.appendSubmission - const writing = vi.spyOn(AgentSessionJournal.prototype, 'appendSubmission') - writing.mockImplementationOnce(async function (this: AgentSessionJournal, input) { - const cursor = await append.call(this, input) + // The newer message lands after the reattach and just before the continuation is accepted, + // which is where a send asks whether its offer still stands. + const send = host.send + const writing = vi.spyOn(host, 'send') + writing.mockImplementationOnce(async (caller, params) => { events.appendItem( { provider: 'codex', threadId: THREAD, turnId: 'newer-turn', ordinal: 1 }, hostTestMessage('A newer task from another client') ) - return cursor + await host.flushStreamedEvents(SESSION) + return send(caller, params) }) const admit = StructuredAgentSessionResumeAdmission.prototype.run const admitting = vi.spyOn(StructuredAgentSessionResumeAdmission.prototype, 'run') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-ownership.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-ownership.test.ts index c3301861c01..ca5ba68dee0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-ownership.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-ownership.test.ts @@ -178,9 +178,8 @@ it('refuses at send admission once the user has sent a newer message', async () { outcome: 'refused', reason: 'agent_session_restart_work_superseded' } ]) expect(dispatch).not.toHaveBeenCalled() - expect(host.journalSnapshot(SESSION).submissions).toMatchObject([ - { dispatchState: 'rejected', reason: 'agent_session_restart_work_superseded' } - ]) + // Refused where it would have been accepted, so the chat records no continuation at all. + expect(host.journalSnapshot(SESSION).submissions).toEqual([]) host.release(SESSION, 'pane') expect(host.isHeld(SESSION)).toBe(false) }) @@ -249,6 +248,8 @@ it.each([false, true])( hostTestMessage('A newer task from another client'), { lifecycle: true } ) + // Journaled before the continuation's acceptance asks whether its offer still stands. + await host.flushStreamedEvents(SESSION) }) const result = await host.restartResume.continueAfterRestart([SESSION], 'modal') @@ -257,8 +258,8 @@ it.each([false, true])( { outcome: 'refused', reason: 'agent_session_restart_work_superseded' } ]) expect(dispatch).not.toHaveBeenCalled() - expect(host.journalSnapshot(SESSION).submissions).toHaveLength(2) - expect(host.journalSnapshot(SESSION).submissions[1]?.dispatchState).toBe('rejected') + // Refused before acceptance: only the interrupted send is in the journal. + expect(host.journalSnapshot(SESSION).submissions).toHaveLength(1) host.release(SESSION, 'pane') expect(host.isHeld(SESSION)).toBe(false) // The offer is spent, but the refusal is kept as a durable failure: a retry finds it, and the @@ -560,7 +561,7 @@ it('fails closed on corrupt recovery storage while ordinary hold and send still expect( await host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) ).toMatchObject({ ok: true }) - expect(dispatch).toHaveBeenCalledTimes(1) + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1)) // list; the action's read of offers and of failures; the post-action refresh of both. expect(warning).toHaveBeenCalledTimes(5) warning.mockRestore() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts index 56d5e0b2317..491a2670d0d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts @@ -15,6 +15,12 @@ import { restoreStructuredAgentSessionsOnRestart } from './structured-agent-session-restart-restore' +const NO_OPEN_DEPS = { + store: { getRecord: () => null, listRecords: () => [] }, + journalRoot: '/tmp/journals', + adapter: {} +} + describe('restart journal restoration', () => { beforeEach(() => restoreRead.mockReset()) @@ -22,17 +28,19 @@ describe('restart journal restoration', () => { const gate = Promise.withResolvers() let active = 0 let peak = 0 - restoreRead.mockImplementation(async (_store, _root, sessionId: string) => { + restoreRead.mockImplementation(async (_deps, sessionId: string) => { active += 1 peak = Math.max(peak, active) await gate.promise active -= 1 return { - journal: {}, - params: { location: { workspaceId: 'workspace-1' }, provider: 'codex' }, - fence: 1, - hasProviderChild: false, - sessionId + session: { + journal: {}, + params: { location: { workspaceId: 'workspace-1' }, provider: 'codex' }, + child: null, + sessionId + }, + reset: null } }) const records = Array.from( @@ -41,15 +49,13 @@ describe('restart journal restoration', () => { ) const restoration = restoreStructuredAgentSessionsOnRestart({ - store: {} as never, - journalRoot: '/tmp/journals', + openDeps: NO_OPEN_DEPS, records, reconcile: async () => null, resolveRecovery: async () => undefined, serialize: async (_sessionId, task) => task(), hasSession: () => false, - onReadable: () => undefined, - settleStaleState: async () => undefined + onReadable: () => undefined }) await vi.waitFor(() => expect(active).toBe(4)) @@ -82,61 +88,53 @@ describe('restart journal restoration', () => { runtimeKind: 'native' } const restored = { - journal: {}, - params, - fence: 4, - hasProviderChild: false, - acquisitionGeneration: null + session: { journal: {}, params, child: null }, + reset: null } - restoreRead.mockResolvedValue(restored) + // The open is what settles: it runs after recovery resolution and before the publish. + restoreRead.mockImplementation(async () => { + calls.push('open') + return restored + }) await restoreOneStructuredAgentSessionRead( { - store: {} as never, - journalRoot: '/tmp/journals', + openDeps: NO_OPEN_DEPS, reconcile: async () => null, resolveRecovery: async () => { calls.push('resolveRecovery') }, serialize: async (_sessionId, task) => task(), hasSession: () => false, - onReadable: () => { - calls.push('onReadable') - }, - settleStaleState: async (_sessionId, settled) => { - calls.push(settled === restored ? 'settleStaleState:restored' : 'settleStaleState') + onReadable: (_sessionId, readable) => { + calls.push(readable === restored ? 'onReadable:restored' : 'onReadable') } }, 'session-1' ) - expect(calls).toEqual(['resolveRecovery', 'settleStaleState:restored', 'onReadable']) + expect(calls).toEqual(['resolveRecovery', 'open', 'onReadable:restored']) }) it('does not settle again when a second restore finds the session already open', async () => { - const settleStaleState = vi.fn(async () => undefined) restoreRead.mockResolvedValue({ - journal: {}, - params: {}, - fence: 4, - hasProviderChild: false, - acquisitionGeneration: null + session: { journal: {}, params: {}, child: null }, + reset: null }) await restoreOneStructuredAgentSessionRead( { - store: {} as never, - journalRoot: '/tmp/journals', + openDeps: NO_OPEN_DEPS, reconcile: async () => null, resolveRecovery: async () => undefined, serialize: async (_sessionId, task) => task(), hasSession: () => true, - onReadable: () => undefined, - settleStaleState + onReadable: () => undefined }, 'session-1' ) - expect(settleStaleState).not.toHaveBeenCalled() + // The open is where the settlement runs, and a session already open is not opened again. + expect(restoreRead).not.toHaveBeenCalled() }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts index 1b5d8864ae5..789eb37f45c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts @@ -13,28 +13,28 @@ import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' -import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { mapWithConcurrency } from '../../../shared/map-with-concurrency' -import { - restoreStructuredAgentSessionRead, - type RestoredStructuredAgentSessionRead -} from './structured-agent-session-read-restore' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { + OpenedStructuredAgentSessionConversation, + StructuredAgentSessionConversationOpenDeps +} from './structured-agent-session-conversation-open' +import { restoreStructuredAgentSessionRead } from './structured-agent-session-read-restore' const JOURNAL_RESTORE_CONCURRENCY = 4 export type StructuredAgentSessionReadRestoreDeps = { - store: AgentSessionRecordStore - journalRoot: string + openDeps: StructuredAgentSessionConversationOpenDeps & { + store: Pick + } reconcile: (sessionId: string) => Promise resolveRecovery: (sessionId: string) => Promise serialize: (sessionId: string, task: () => Promise) => Promise hasSession: (sessionId: string) => boolean - onReadable: (sessionId: string, restored: RestoredStructuredAgentSessionRead) => void - /** Settles what a previous generation left running. Best effort: the next acquire re-derives it. */ - settleStaleState: ( + onReadable: ( sessionId: string, - restored: RestoredStructuredAgentSessionRead - ) => Promise + opened: OpenedStructuredAgentSessionConversation + ) => Promise | void } /** @@ -61,28 +61,19 @@ export async function restoreOneStructuredAgentSessionRead( /** The serialized half of the restore, for a caller already inside the session's serialize — a * send replaying into a session this host has closed, which needs the journal and no child. */ export async function restoreOneStructuredAgentSessionReadUnderSerialize( - input: Pick< - StructuredAgentSessionReadRestoreDeps, - 'store' | 'journalRoot' | 'hasSession' | 'onReadable' | 'settleStaleState' - >, + input: Pick, sessionId: string ): Promise { if (input.hasSession(sessionId)) { // A surface that took a hold mid-restore already attached this one. return } - const restored = await restoreStructuredAgentSessionRead( - input.store, - input.journalRoot, - sessionId - ) - if (!restored) { + const opened = await restoreStructuredAgentSessionRead(input.openDeps, sessionId) + if (!opened) { return } - // No child in this process writes to a journal with no map entry, so anything it shows running - // belongs to a generation that is gone. Settled before it is published, so no reader sees it run. - await input.settleStaleState(sessionId, restored) - input.onReadable(sessionId, restored) + // The open settled what a gone generation left running, so no reader sees it run. + await input.onReadable(sessionId, opened) } export async function restoreStructuredAgentSessionsOnRestart( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-host.ts index 29453e74652..0fa4ee8534d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-host.ts @@ -41,8 +41,9 @@ import { type StructuredAgentSessionContinuationOutcome } from './structured-agent-session-restart-continuation' import { createStructuredAgentSessionRestartWitnesses } from './structured-agent-session-restart-witnesses' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' -type LiveSession = { journal: AgentSessionJournal; hasProviderChild: boolean; fence: number } +type LiveSession = { journal: AgentSessionJournal; child: { fence: number } | null } export type StructuredAgentSessionRestartResumeSurfaces = { revealSession: (sessionId: string) => Promise<{ readable: boolean }> @@ -152,6 +153,10 @@ export function createStructuredAgentSessionRestartResume( const continuationHost: StructuredAgentSessionContinuationHost = { ...surfaces, sessions, + conversationFence: (sessionId) => + sessions.has(sessionId) + ? structuredAgentSessionConversationFence(deps.store, sessionId) + : null, stillResumable: (marker, options) => derive([marker], 'may-be-held', options).candidates.length === 1 } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts index e5b65009634..68823c078dc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts @@ -145,7 +145,7 @@ export type HarnessJournal = { appendItem: (envelope: unknown, body: { kind: string; text: string }) => Promise } -export type HarnessSession = { journal: HarnessJournal; hasProviderChild: boolean; fence?: number } +export type HarnessSession = { journal: HarnessJournal; child: { fence: number } | null } export function journal( items: AgentJournalRenderItem[], diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-wiring.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-wiring.ts index 07ec6b81599..66461a0c873 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-wiring.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-wiring.ts @@ -11,7 +11,9 @@ import type { AgentSessionMutationResult, AgentSessionSendResult } from '../../../shared/agent-session-wire' +import { MAX_TIMER_DELAY_MS } from '../../../shared/timer-delay' import type { StructuredAgentSessionRestartResumeSurfaces } from './structured-agent-session-restart-resume-host' +import type { SendSettlementWaitOptions } from './structured-agent-session-send-settlement' /** The caller key the continuation sends under, so its writes are attributable to Orca itself. */ export const STRUCTURED_AGENT_SESSION_RESTART_CONTINUATION_CALLER = @@ -34,7 +36,8 @@ type RestartResumeHostBindings = { /** The host's existing settlement waiter; a send returns while its dispatch is still pending. */ waitForSendSettlement: ( sessionId: string, - clientMessageId: string + clientMessageId: string, + options: SendSettlementWaitOptions ) => Promise<{ value: AgentSessionSendResult } | undefined> } @@ -48,7 +51,10 @@ export function structuredAgentSessionRestartResumeSurfaces( release: host.release, send: (params) => host.send({ callerKey: STRUCTURED_AGENT_SESSION_RESTART_CONTINUATION_CALLER }, params), - awaitSendSettlement: host.waitForSendSettlement, + // Accepted like any send, so its verdict is its delivery, however long the start takes; the + // wait ends when the submission settles or the session closes. + awaitSendSettlement: (sessionId, clientMessageId) => + host.waitForSendSettlement(sessionId, clientMessageId, { budgetMs: MAX_TIMER_DELAY_MS }), onNoteFailed: () => console.warn('[structured-agent-session] restart continuation attribution failed'), now diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts index 1d6a8a1fc5e..5dc939d7e66 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts @@ -48,7 +48,7 @@ describe('deriving what was working at teardown', () => { it('marks a session this host was running a turn for', () => { const markers = markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'running')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'running')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => undefined, @@ -74,7 +74,7 @@ describe('deriving what was working at teardown', () => { it('carries the update trigger so the surface can say the restart was not the user choice', () => { const [recorded] = markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'running')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'running')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => undefined, @@ -89,7 +89,7 @@ describe('deriving what was working at teardown', () => { it('marks nothing for an idle session', () => { expect( markersAtTeardown({ - sessions: new Map([[SESSION, { journal: journal([]), hasProviderChild: true }]]), + sessions: new Map([[SESSION, { journal: journal([]), child: { fence: 1 } }]]), getRecord: () => record(), backgroundTasks: () => undefined, trigger: 'quit', @@ -103,7 +103,7 @@ describe('deriving what was working at teardown', () => { expect( markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => undefined, @@ -115,12 +115,12 @@ describe('deriving what was working at teardown', () => { }) // The user's stated fear. A journal restored for READING carries whatever `running` row an older - // crash left behind, and it is the live `hasProviderChild` — not that row — that decides. + // crash left behind, and it is the live `child` — not that row — that decides. it('marks nothing for a stale running row this host was not executing', () => { expect( markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'running')]), hasProviderChild: false }] + [SESSION, { journal: journal([turnItem('turn-1', 'running')]), child: null }] ]), getRecord: () => record(), backgroundTasks: () => undefined, @@ -141,7 +141,7 @@ describe('deriving what was working at teardown', () => { SESSION, { journal: journal([turnItem('turn-1', 'running'), pendingApproval()]), - hasProviderChild: true + child: { fence: 1 } } ] ]), @@ -165,7 +165,7 @@ describe('deriving what was working at teardown', () => { it('marks a settled lead whose subagent was still running, anchored on its last turn', () => { const markers = markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => [ @@ -188,7 +188,7 @@ describe('deriving what was working at teardown', () => { it('records only the live rows of the roster, bounded', () => { const [recorded] = markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => [ @@ -213,7 +213,7 @@ describe('deriving what was working at teardown', () => { it('marks a settled lead whose only live work is a monitor', () => { const markers = markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => [{ id: 'task-m', kind: 'monitor', name: 'ci-watch' }], @@ -230,7 +230,7 @@ describe('deriving what was working at teardown', () => { expect( markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'completed')]), child: { fence: 1 } }] ]), getRecord: () => record(), backgroundTasks: () => [ @@ -248,7 +248,7 @@ describe('deriving what was working at teardown', () => { it('records the identity root so an advancing Claude leaf cannot invalidate the marker', () => { const [recorded] = markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'running')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'running')]), child: { fence: 1 } }] ]), getRecord: () => claudeRecord(null), backgroundTasks: () => undefined, @@ -264,7 +264,7 @@ describe('deriving what was working at teardown', () => { expect( markersAtTeardown({ sessions: new Map([ - [SESSION, { journal: journal([turnItem('turn-1', 'running')]), hasProviderChild: true }] + [SESSION, { journal: journal([turnItem('turn-1', 'running')]), child: { fence: 1 } }] ]), getRecord: () => record({ chain: [] }), backgroundTasks: () => undefined, @@ -285,7 +285,7 @@ describe('deriving what was working at teardown', () => { SESSION, { journal: journal([], false, [submission('msg-1', 'pending')]), - hasProviderChild: true + child: { fence: 1 } } ] ]), @@ -310,7 +310,7 @@ describe('deriving what was working at teardown', () => { journal: journal([turnItem('turn-0', 'completed')], false, [ submission('msg-1', 'pending') ]), - hasProviderChild: true + child: { fence: 1 } } ] ]), @@ -324,6 +324,67 @@ describe('deriving what was working at teardown', () => { expect(recorded?.work).toEqual({ kind: 'submission', id: 'msg-1' }) }) + // Accepted while the agent was starting and never handed over: the chat shows working, but no + // agent had the message, and quit rejects it as never sent. + it('marks nothing for a session whose only work is a message still queued', () => { + const queued = { ...submission('msg-1', 'pending'), handoverRecorded: true as const } + expect( + markersAtTeardown({ + sessions: new Map([ + [ + SESSION, + { + journal: journal([turnItem('turn-0', 'completed')], false, [queued]), + child: { fence: 1 } + } + ] + ]), + getRecord: () => claudeRecord(null), + backgroundTasks: () => undefined, + trigger: 'quit', + teardownId: TEARDOWN_CURRENT, + now: NOW + }) + ).toEqual([]) + }) + + it('marks a handed-over message over a newer queued one, and a turn a queued one waits behind', () => { + const handedOver = { + ...submission('msg-1', 'pending'), + handoverRecorded: true as const, + handedOverAt: NOW + } + const queued = { ...submission('msg-2', 'pending'), handoverRecorded: true as const } + const markers = markersAtTeardown({ + sessions: new Map([ + [ + SESSION, + { + journal: journal([turnItem('turn-0', 'completed')], false, [handedOver, queued]), + child: { fence: 1 } + } + ], + [ + 'session-running', + { + journal: journal([turnItem('turn-1', 'running')], false, [queued]), + child: { fence: 1 } + } + ] + ]), + getRecord: () => claudeRecord(null), + backgroundTasks: () => undefined, + trigger: 'quit', + teardownId: TEARDOWN_CURRENT, + now: NOW + }) + + expect(markers.map((entry) => entry.work)).toEqual([ + { kind: 'submission', id: 'msg-1' }, + { kind: 'turn', id: 'turn-1' } + ]) + }) + // Once a turn exists it is the better identity: it is what eviction rewrites, so it is what the // journal can be asked about at launch. it('prefers the running turn over the send that opened it', () => { @@ -335,7 +396,7 @@ describe('deriving what was working at teardown', () => { journal: journal([turnItem('turn-1', 'running')], false, [ submission('msg-1', 'accepted') ]), - hasProviderChild: true + child: { fence: 1 } } ] ]), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts index 2e031b9f071..93a7962d4a2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts @@ -11,7 +11,7 @@ import { agentSessionLeaseIsReleased } from '../../../shared/agent-session-lease import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { randomUUID } from 'node:crypto' import type { AgentSessionAttachParams } from './structured-agent-session-attach' -import { attachParamsForRecord } from './structured-agent-session-read-restore' +import { attachParamsForRecord } from './structured-agent-session-conversation-open' export function isResumableStructuredAgentSessionRecord(record: AgentSessionRecord): boolean { return agentSessionLeaseIsReleased(record.lease) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.test.ts index 1172e993b96..7ba1c716769 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.test.ts @@ -46,23 +46,31 @@ function harness( return task() } const restorer = new StructuredAgentSessionReadableRestorer({ - store: { - getRecord: (sessionId: string) => records.find((r) => r.sessionId === sessionId) ?? null, - listRecords: () => records - } as never, - journalRoot: '/journals', + openDeps: { + store: { + getRecord: (sessionId: string) => records.find((r) => r.sessionId === sessionId) ?? null, + listRecords: () => records + }, + journalRoot: '/journals', + adapter: {} + }, supportsRecord: options.supports ?? (() => true), reconcile: async () => null, resolveRecovery: async () => undefined, serialize, hasSession: (sessionId) => live.has(sessionId), - onReadable: (sessionId, restored) => live.set(sessionId, restored), - settleStaleState: async () => undefined + onReadable: (sessionId, restored) => { + live.set(sessionId, restored) + } }) return { restorer, live, serializedIds } } -const readable = { journal: {}, params: {}, fence: 1 } as never +const readable = { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the restorer only indexes the session it is handed; no member of it is read here. + session: { journal: {}, params: {}, fence: 1 } as never, + reset: null +} afterEach(() => { vi.restoreAllMocks() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index 097cd6b479a..c6ea9c1f83b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -18,7 +18,6 @@ import type { StructuredAgentSessionHostDeps, StructuredAgentSessionReveal } from './structured-agent-session-host-types' -import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' /** Throws its refusal as the code itself, matching `resumeHeldStructuredAgentSession`. */ export async function revealStructuredAgentSession( @@ -58,31 +57,15 @@ export function createStructuredAgentSessionHostRestore( deps: StructuredAgentSessionHostDeps, wiring: Omit< ConstructorParameters[0], - 'store' | 'journalRoot' | 'supportsRecord' | 'settleStaleState' + 'openDeps' | 'supportsRecord' > ): { restoreReadableSessions: (sessionIds?: readonly string[]) => Promise revealSession: (sessionId: string) => Promise - /** One session, for a caller already inside its serialize. */ - restoreReadableUnderSerialize: (sessionId: string) => Promise } { const restorer = new StructuredAgentSessionReadableRestorer({ - store: deps.store, - journalRoot: deps.journalRoot, + openDeps: deps, supportsRecord: (record) => adapterSupportsRecord(deps.adapter, record), - settleStaleState: async (sessionId, restored) => { - try { - await settleStaleStructuredAgentSessionState({ - journal: restored.journal, - sessionId, - fence: restored.fence, - acquisitionGeneration: null, - deathEvidence: deps.store.getRecord(sessionId)?.lease.deathEvidence ?? null - }) - } catch (error) { - deps.onEventSinkError?.({ sessionId, error }) - } - }, ...wiring }) const gate = new StructuredAgentSessionRestartRestoreGate() @@ -91,7 +74,6 @@ export function createStructuredAgentSessionHostRestore( revealSession: (sessionId) => revealStructuredAgentSession(deps, sessionId, wiring.hasSession, (id) => restorer.restoreOne(id) - ), - restoreReadableUnderSerialize: (sessionId) => restorer.restoreOneUnderSerialize(sessionId) + ) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts index b82248cb06f..32b05b08077 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts @@ -139,6 +139,14 @@ async function seed(acceptedSubmissions = false) { } }) ).toMatchObject({ ok: true }) + // Accepted into the conversation first; the delivery loop hands it over after. + await vi.waitFor(() => + expect( + host + .journalSnapshot(HOST_TEST_SESSION) + .submissions.find((entry) => entry.clientMessageId === clientOperationId)?.dispatchState + ).toBe('accepted') + ) if (i === 1) { selectedItemId = agentJournalSubmissionKey(clientOperationId) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-idempotency.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-idempotency.test.ts index 0f75c9a3322..2494535e019 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-idempotency.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-idempotency.test.ts @@ -120,11 +120,16 @@ describe('structured send idempotency', () => { await performSend(context, input) const replay = await performSend(context, input) + // Acceptance records the one submission; the reused id answers with it and writes nothing. + // Handing it over is the delivery loop's, never a second accept's. expect(replay).toMatchObject({ ok: true, - value: { clientMessageId: 'shared-send-id', submission: { dispatchState: 'accepted' } } + value: { + clientMessageId: 'shared-send-id', + submission: { dispatchState: 'pending', handoverRecorded: true } + } }) - expect(dispatch).toHaveBeenCalledOnce() + expect(dispatch).not.toHaveBeenCalled() expect(journal.submissions()).toHaveLength(1) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts new file mode 100644 index 00000000000..ffd06ae4dac --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts @@ -0,0 +1,137 @@ +// A send can be what opens a conversation this process has not read yet: a chat nobody has on +// screen after the app died, sent to from a phone or the CLI. Whatever that journal shows running +// belongs to a generation that is gone, so it is settled when the journal opens, not only when a +// new child starts: a start that then fails would leave the turn running for every reader. + +import { cp, rm } from 'node:fs/promises' +import { join } from 'node:path' +import { afterEach, expect, it, vi } from 'vitest' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + adapter, + attach, + CALLER, + envelope, + hostTestState, + replaceHostTestState +} from './structured-agent-session-host-test-harness' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, + hostTestMessage +} from './structured-agent-session-host-test-data' + +/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */ +function eventually(assertion: () => unknown): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} + +const relaunchedRoots: string[] = [] + +afterEach(async () => { + await Promise.all( + relaunchedRoots.splice(0).map((dir) => rm(dir, { recursive: true, force: true })) + ) +}) + +/** A host that dies mid-turn, relaunched over a copy of its files taken at the crash. */ +async function relaunchAfterCrashMidTurn(probe: AgentSessionOwnerProbe) { + const dying = hostTestState() + await attach() + const events = dying.acquire.mock.calls[0]?.[0].events + if (!events) { + throw new Error('missing provider event sink') + } + events.appendItem( + { provider: 'codex', threadId: THREAD, turnId: 'crashed-turn', ordinal: 1 }, + { kind: 'turn', turnId: 'crashed-turn', state: 'running' } + ) + await dying.host.flushStreamedEvents(SESSION) + // An empty renewal queues behind every record write, so they are on disk. + await dying.store.renewLeases([]) + const relaunched = `${dying.root}-relaunched` + relaunchedRoots.push(relaunched) + // A dead process holds no lock. + await cp(dying.root, relaunched, { + recursive: true, + filter: (source) => !source.includes('.lock') + }) + const store = await AgentSessionRecordStore.open({ + directory: join(relaunched, 'store'), + hostId: 'local' + }) + const acquire = vi.fn(async () => { + throw new Error('claude: command not found') + }) + const host = new StructuredAgentSessionHost({ + store, + adapter: { ...adapter(), acquire }, + journalRoot: relaunched, + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-next', + probeOwner: async () => probe, + now: () => NOW + }) + await host.reconcileRestartLeases() + replaceHostTestState({ store, host }) + return { host, acquire } +} + +/** Neither proof of the old owner's death is an observed exit, so the turn ends `unverifiable`. */ +function turnStates(host: StructuredAgentSessionHost) { + return host + .journalSnapshot(SESSION) + .items.flatMap((item) => readAgentJournalTurn(item.body) ?? []) + .map((turn) => turn.state) +} + +// A host that cannot prove the old owner gone leaves its lease in recovery, still claimed, until +// the next acquire resolves it: the open is not that acquire, and the turn is no less gone. +const PROBES = [ + ['the old owner is proven gone', { outcome: 'pid-absent' }], + [ + 'nothing proves the old owner gone', + { outcome: 'indeterminate', reason: 'This host cannot probe structured session owners.' } + ] +] as const satisfies readonly (readonly [string, AgentSessionOwnerProbe])[] + +it.each(PROBES)( + 'settles a turn a dead generation left running when a send opens the chat and its start fails, when %s', + async (_when, probe) => { + const { host, acquire } = await relaunchAfterCrashMidTurn(probe) + expect(host.hasSession(SESSION)).toBe(false) + + const body = hostTestMessage('sent to a chat nobody has open') + const sendEnvelope = envelope('agentSession.send', { body }) + await expect(host.send(CALLER, { envelope: sendEnvelope, body })).resolves.toMatchObject({ + ok: true + }) + await eventually(() => + expect( + host + .journalSnapshot(SESSION) + .submissions.find((entry) => entry.clientMessageId === sendEnvelope.clientOperationId) + ).toMatchObject({ dispatchState: 'rejected' }) + ) + + expect(acquire).toHaveBeenCalledOnce() + expect(turnStates(host)).toEqual(['unverifiable']) + await host.flushAllStreamedEvents() + } +) + +it.each(PROBES)( + 'settles the same turn when a reader opens the chat, when %s', + async (_when, probe) => { + const { host } = await relaunchAfterCrashMidTurn(probe) + + await host.revealSession(SESSION) + + expect(turnStates(host)).toEqual(['unverifiable']) + await host.flushAllStreamedEvents() + } +) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.test.ts index a93a64b0dc2..8c8dfec04ce 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.test.ts @@ -1,11 +1,11 @@ -// A send, or a hold, that finds the session's provider child gone, against the real host. +// A send, or a hold, that finds the session's provider child gone, against the real host. The +// send is accepted at once; its delivery restarts the child, or rejects it with the reason. import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest' import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' -import { agentSessionRefusalOperationState } from '../../../shared/agent-session-refusal-retry' import type { AgentSessionMutationEnvelope, AgentSessionSubscribeEvent @@ -24,6 +24,11 @@ import { } from './structured-agent-session-host-test-data' const CALLER = { callerKey: 'client-1' } + +/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */ +function eventually(assertion: () => void): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} // Long enough that no release fires mid-test; whether one is pending is asserted directly. const GRACE_MS = 60_000 @@ -105,15 +110,39 @@ function sendParams(text: string, operationId = hostTestOperationId()) { return { envelope, body } } -/** Every status row the chat shows, oldest first; none when the session is not even readable. */ -function journalStatuses(): string[] { - if (!host.hasSession(SESSION)) { - return [] - } - const history = host.history({ sessionId: SESSION, direction: 'tail' }) - return history.ok - ? history.page.items.flatMap((item) => (item.body.kind === 'status' ? [item.body.text] : [])) - : [] +/** Accepted at once, before any owner exists for it; answers the message id. */ +async function accept(params: ReturnType): Promise { + const result = await host.send(CALLER, params) + expect(result, JSON.stringify(result)).toMatchObject({ + ok: true, + replayed: false, + value: { submission: { dispatchState: 'pending', handoverRecorded: true } } + }) + return params.envelope.clientOperationId +} + +function submission(clientMessageId: string) { + return host + .journalSnapshot(SESSION) + .submissions.find((entry) => entry.clientMessageId === clientMessageId) +} + +/** The submission once delivery is done with it: handed over, or rejected unwritten. */ +async function settled(clientMessageId: string) { + await eventually(() => { + const current = submission(clientMessageId) + expect(current?.dispatchState !== 'pending' || current.handedOverAt !== undefined).toBe(true) + }) + return submission(clientMessageId) +} + +/** The failure rows a start the chat needed left, oldest first. */ +function errorStatuses(): string[] { + return host + .journalSnapshot(SESSION) + .items.flatMap((item) => + item.body.kind === 'status' && item.body.tone === 'error' ? [item.body.text] : [] + ) } /** The child timed out or exited: its lease is handed back and the host holds no session. */ @@ -130,15 +159,14 @@ describe('a send with no live owner', () => { it('restarts the owner once and delivers against it', async () => { await loseOwner() - const result = await host.send(CALLER, sendParams('after the child died')) + await accept(sendParams('after the child died')) - expect(result).toMatchObject({ ok: true, replayed: false }) + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) expect(acquire).toHaveBeenCalledOnce() - expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('live') }) - it('restarts the owner before the send is admitted, so the send is admitted once', async () => { + it('accepts the send before anything restarts, and the restart hands it over', async () => { await loseOwner() const order: string[] = [] const spawnChild = acquire.getMockImplementation()! @@ -152,21 +180,18 @@ describe('a send with no live owner', () => { return admit(args) }) - await expect(host.send(CALLER, sendParams('ensure first'))).resolves.toMatchObject({ - ok: true, - replayed: false - }) + await accept(sendParams('accept first')) + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) - expect(order).toEqual(['acquire', 'admit']) + expect(order).toEqual(['admit', 'acquire']) }) it('leaves a live owner alone', async () => { acquire.mockClear() - await expect(host.send(CALLER, sendParams('owner is live'))).resolves.toMatchObject({ - ok: true - }) + await accept(sendParams('owner is live')) + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) expect(acquire).not.toHaveBeenCalled() }) @@ -184,36 +209,35 @@ describe('a send with no live owner', () => { } })) - await host.send(CALLER, sendParams('into a cleared chat')) + await expect(host.send(CALLER, sendParams('into a cleared chat'))).resolves.toMatchObject({ + ok: false, + refusal: { code: 'agent_session_operation_invalid' } + }) expect(acquire).not.toHaveBeenCalled() }) it('renews the idle window on journal activity in an unheld session', async () => { await loseOwner() - await expect(host.send(CALLER, sendParams('restart'))).resolves.toMatchObject({ ok: true }) + await accept(sendParams('restart')) + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) const arm = vi.spyOn(host['holds']['clock'], 'arm') - await expect(host.send(CALLER, sendParams('more activity'))).resolves.toMatchObject({ - ok: true - }) + await accept(sendParams('more activity')) - expect(arm).toHaveBeenCalledWith(SESSION) + await eventually(() => expect(arm).toHaveBeenCalledWith(SESSION)) }) it('releases the restarted child on the usual clock only when no surface holds it', async () => { await loseOwner() - await expect(host.send(CALLER, sendParams('nobody is watching'))).resolves.toMatchObject({ - ok: true - }) - expect(host['holds'].isReleasePending(SESSION)).toBe(true) + await accept(sendParams('nobody is watching')) + await eventually(() => expect(host['holds'].isReleasePending(SESSION)).toBe(true)) await host.close(SESSION) // A reading surface that does not itself restart the agent. await host.hold(SESSION, 'desktop-chat:1', { resume: false }) - await expect(host.send(CALLER, sendParams('the chat is open'))).resolves.toMatchObject({ - ok: true - }) + await accept(sendParams('the chat is open')) + await eventually(() => expect(dispatch).toHaveBeenCalledTimes(2)) expect(host['holds'].isReleasePending(SESSION)).toBe(false) }) @@ -231,15 +255,15 @@ describe('a send with no live owner', () => { expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') acquire.mockClear() - await expect(host.send(CALLER, sendParams('after an exit'))).resolves.toMatchObject({ - ok: true - }) + await accept(sendParams('after an exit')) + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) expect(acquire).toHaveBeenCalledOnce() }) it('restarts nothing for a resend the journal already answers', async () => { const params = sendParams('sent once') - await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: false }) + await accept(params) + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) // The child died during startup: the lease is handed back, the fence moves, and the session // stays readable. The client resends against the new fence. await host.handleAdapterEvent({ @@ -268,12 +292,9 @@ describe('a send with no live owner', () => { expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('released') // Retry rotates the id: a genuinely new send restarts the owner once. - await expect(host.send(CALLER, sendParams('sent once'))).resolves.toMatchObject({ - ok: true, - replayed: false - }) + await accept(sendParams('sent once')) + await eventually(() => expect(dispatch).toHaveBeenCalledTimes(2)) expect(acquire).toHaveBeenCalledOnce() - expect(dispatch).toHaveBeenCalledTimes(2) }) it('restarts nothing for a send the ledger holds but the journal never saw', async () => { @@ -320,7 +341,7 @@ describe('a send with no live owner', () => { expect(dispatch).not.toHaveBeenCalled() }) - it('admits a send that arrives after the restart has already claimed the lease', async () => { + it('accepts a send that arrives while a restart holds the queue, and hands both over in order', async () => { await loseOwner() const lostFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 let claimed = () => {} @@ -334,18 +355,22 @@ describe('a send with no live owner', () => { return spawnChild!(input) }) - const first = host.send(CALLER, sendParams('first')) + const first = await accept(sendParams('first')) await claim expect(store.getRecord(SESSION)?.lease.runtimeFence).toBe(lostFence + 1) + // Written against the lost owner's fence, which admits it: a send is a conversation write. const late = sendParams('second') late.envelope.expectedRuntimeFence = lostFence const second = host.send(CALLER, late) release() - expect(await first).toMatchObject({ ok: true }) expect(await second).toMatchObject({ ok: true }) + await eventually(() => expect(dispatch).toHaveBeenCalledTimes(2)) expect(acquire).toHaveBeenCalledOnce() - expect(dispatch).toHaveBeenCalledTimes(2) + expect(dispatch.mock.calls.map(([input]) => input.clientMessageId)).toEqual([ + first, + late.envelope.clientOperationId + ]) }) it('shares one restart between concurrent sends', async () => { @@ -358,8 +383,8 @@ describe('a send with no live owner', () => { ]) expect(results.map((result) => result.ok)).toEqual([true, true, true]) + await eventually(() => expect(dispatch).toHaveBeenCalledTimes(3)) expect(acquire).toHaveBeenCalledOnce() - expect(dispatch).toHaveBeenCalledTimes(3) }) it('shares one restart between a hold and a send that arrive in the same gap', async () => { @@ -372,21 +397,22 @@ describe('a send with no live owner', () => { expect(held).toMatchObject({ status: 'fulfilled' }) expect(sent).toMatchObject({ status: 'fulfilled', value: { ok: true } }) + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) expect(acquire).toHaveBeenCalledOnce() - expect(dispatch).toHaveBeenCalledOnce() expect(host['holds'].isHeld(SESSION)).toBe(true) expect(host['holds'].isReleasePending(SESSION)).toBe(false) }) it('replays into a closed session without spawning anything', async () => { const params = sendParams('sent once') - await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: false }) + await accept(params) + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) await loseOwner() await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: true }) await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: true }) - // The journal was made readable for the answer; the record's lease was left as it was. + // The conversation was opened for the answer; the record's lease was left as it was. expect(host.hasSession(SESSION)).toBe(true) expect(acquire).not.toHaveBeenCalled() expect(dispatch).toHaveBeenCalledOnce() @@ -394,104 +420,83 @@ describe('a send with no live owner', () => { expect(host['holds'].isReleasePending(SESSION)).toBe(false) }) - it("refuses with the restart's own cause, in the answer and in the chat", async () => { + it("rejects the accepted message with the restart's own cause, and says so in the chat once", async () => { await loseOwner() acquire.mockRejectedValue(new Error('Not signed in. Run codex login')) const params = sendParams('while signed out') - - const result = await host.send(CALLER, params) - - expect(result).toEqual({ - ok: false, - refusal: { - code: 'agent_session_owner_restart_failed', - message: "Codex couldn't restart: Not signed in. Run codex login.", - // The failed attach proved its child gone: nothing runs for this session. - ownerVerdict: 'exited' - } - }) - expect(dispatch).not.toHaveBeenCalled() - expect(hostErrors).not.toEqual([]) - expect(agentSessionRefusalOperationState('agent_session_owner_restart_failed')).toBe( - 'settled-rejected' - ) - // Refused before admission: the ledger holds nothing a resend would replay. - expect(store.getOperationRow(CALLER.callerKey, params.envelope.clientOperationId)).toBeNull() - // The same status row a failed start leaves, so the reason outlives the error strip. - expect(journalStatuses()).toEqual([ + const cause = 'The provider stopped before it finished starting: Not signed in. Run codex login.' - ]) + + const id = await accept(params) + + expect(await settled(id)).toMatchObject({ dispatchState: 'rejected', reason: cause }) + expect(dispatch).not.toHaveBeenCalled() + // Accepted, so the ledger answers a resend with the rejection rather than a second attempt. + expect( + store.getOperationRow(CALLER.callerKey, params.envelope.clientOperationId) + ).toMatchObject({ outcome: { status: 'succeeded' } }) + // One row, in the error tone, so the reason outlives the error strip. + expect(errorStatuses()).toEqual([cause]) }) - it('restarts again for a Retry of the refused send, under its own id or a new one', async () => { + it('restarts again for a Retry under a new id, and replays a resend of the same id', async () => { await loseOwner() acquire.mockRejectedValue(new Error('Not signed in')) const params = sendParams('while signed out') - await expect(host.send(CALLER, params)).resolves.toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } - }) + await settled(await accept(params)) + expect(acquire).toHaveBeenCalledTimes(1) - // A client that resends the same id gets another attempt, and the chat no second row. + // A client that resends the same id gets the recorded rejection, and the chat no second row. await expect(host.send(CALLER, params)).resolves.toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } + ok: true, + replayed: true, + value: { submission: { dispatchState: 'rejected' } } + }) + expect(acquire).toHaveBeenCalledTimes(1) + expect(errorStatuses()).toHaveLength(1) + + // The outbox's Retry rotates the id: a fresh attempt, with its own row. + expect(await settled(await accept(sendParams('while signed out')))).toMatchObject({ + dispatchState: 'rejected' }) expect(acquire).toHaveBeenCalledTimes(2) - expect(journalStatuses()).toHaveLength(1) - - // The outbox's Retry rotates the id: also a fresh attempt. - await expect(host.send(CALLER, sendParams('while signed out'))).resolves.toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } - }) - expect(acquire).toHaveBeenCalledTimes(3) + expect(errorStatuses()).toHaveLength(2) expect(dispatch).not.toHaveBeenCalled() }) it('restarts and delivers a later send once the cause clears', async () => { await loseOwner() acquire.mockRejectedValueOnce(new Error('Not signed in')) - await expect(host.send(CALLER, sendParams('while signed out'))).resolves.toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } + expect(await settled(await accept(sendParams('while signed out')))).toMatchObject({ + dispatchState: 'rejected' }) // The user signed in; nothing about the failed attempt is remembered. - await expect(host.send(CALLER, sendParams('signed in now'))).resolves.toMatchObject({ - ok: true, - replayed: false - }) + await accept(sendParams('signed in now')) + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) expect(acquire).toHaveBeenCalledTimes(2) - expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('live') }) it('suggests a new chat only when this host has nothing to restart the chat from', async () => { await loseOwner() acquire.mockRejectedValue(new Error('Not signed in')) - const failed = await host.send(CALLER, sendParams('restart fails')) - expect(failed).toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } - }) - expect(failed.ok ? '' : failed.refusal.message).not.toMatch(/new chat/) + const failed = await settled(await accept(sendParams('restart fails'))) + expect(failed).toMatchObject({ dispatchState: 'rejected' }) + expect(failed?.reason).not.toMatch(/new chat/) // The adapter cannot run this record where it lives: no retry would bring it back. host.deps.adapter.supportsLocation = () => false - const unresumable = await host.send(CALLER, sendParams('cannot resume here')) + const unresumable = await settled(await accept(sendParams('cannot resume here'))) expect(unresumable).toMatchObject({ - ok: false, - refusal: { - code: 'agent_session_owner_restart_failed', - message: - "Codex couldn't restart: This execution host cannot resume the requested structured agent session. Start a new chat to continue." - } + dispatchState: 'rejected', + reason: + "Codex couldn't restart: This execution host cannot resume the requested structured agent session. Start a new chat to continue." }) }) - it('runs the send as the lease stands when the restart met a lease someone else is settling', async () => { + it('rejects the message with the cause when the restart met a lease someone else is settling', async () => { await loseOwner() vi.spyOn(host['holds'], 'ensureProviderChild').mockResolvedValueOnce({ ok: false, @@ -501,33 +506,30 @@ describe('a send with no live owner', () => { } }) - const result = await host.send(CALLER, sendParams('owner being settled')) + const id = await accept(sendParams('owner being settled')) - // The ordinary lease check answers, retryably; nothing terminal and nothing in the chat. - expect(result).toMatchObject({ - ok: false, - refusal: { code: 'agent_session_ownership_unknown' } - }) + const cause = "Codex couldn't restart: Another runtime is still adjudicating this lease." + expect(await settled(id)).toMatchObject({ dispatchState: 'rejected', reason: cause }) expect(acquire).not.toHaveBeenCalled() - expect(journalStatuses()).toEqual([]) + expect(errorStatuses()).toEqual([cause]) }) - it('runs the send as the lease stands when the restart itself faults', async () => { + it('rejects the message, and reports the fault, when the restart itself faults', async () => { await loseOwner() vi.spyOn(host['holds'], 'ensureProviderChild').mockRejectedValueOnce( new Error('spawn-token mint failed') ) - const result = await host.send(CALLER, sendParams('bookkeeping failed')) + const id = await accept(sendParams('bookkeeping failed')) - expect(result).toMatchObject({ - ok: false, - refusal: { code: 'agent_session_ownership_unknown' } + expect(await settled(id)).toMatchObject({ + dispatchState: 'rejected', + reason: "Codex couldn't restart: spawn-token mint failed." }) expect(hostErrors).toContainEqual( expect.objectContaining({ message: 'spawn-token mint failed' }) ) - expect(journalStatuses()).toEqual([]) + expect(errorStatuses()).toHaveLength(1) }) it('keeps a second surface holder taken during an auto-restart, and starts nothing for it', async () => { @@ -663,20 +665,22 @@ describe('a send with no live owner', () => { }) }) - it('leaves a lease it cannot adjudicate alone', async () => { + it('adjudicates a lease this host has not reconciled before its delivery resumes it', async () => { await loseOwner() await store.transitionHandoff(SESSION, (current) => ({ ...current, lease: { ...current.lease, unreconciled: true } })) - const result = await host.send(CALLER, sendParams('owner unverifiable')) + // The send's start is the same serialized resume a hold runs, reconciliation first. + await accept(sendParams('owner unverified')) - expect(result).toMatchObject({ - ok: false, - refusal: { code: 'agent_session_ownership_unknown' } + await eventually(() => expect(dispatch).toHaveBeenCalledOnce()) + expect(acquire).toHaveBeenCalledOnce() + expect(store.getRecord(SESSION)?.lease).toMatchObject({ + unreconciled: false, + claimStatus: 'live' }) - expect(acquire).not.toHaveBeenCalled() }) }) @@ -689,20 +693,17 @@ describe('a write fenced to an owner the pane has not seen replaced', () => { const params = sendParams('after the release') params.envelope.expectedRuntimeFence = seenFence - expect(await host.send(CALLER, params)).toMatchObject({ - ok: true, - replayed: false, - value: { submission: { dispatchState: 'accepted' } } - }) + const id = await accept(params) + expect(await settled(id)).toMatchObject({ dispatchState: 'accepted' }) expect(acquire).toHaveBeenCalledOnce() expect(dispatch).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease.runtimeFence).toBeGreaterThan(seenFence + 1) }) - // Clients resend a refused message when the fence they hold moves, and a refused restart leaves - // no ledger row, so a frame carrying each failed attempt's fence would resend it forever. - it("keeps the pane's fence on the row a failed restart publishes", async () => { + // Clients resend a refused message when the fence they hold moves. A failed start is a + // rejected message now, never a refused send, and the pane keeps the fence it subscribed under. + it("keeps the pane's fence on the rows a failed start publishes, and rejects the message", async () => { const seenFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 const frames: AgentSessionSubscribeEvent[] = [] host.subscribe({ id: 'pane', sessionId: SESSION, emit: (event) => frames.push(event) }) @@ -710,11 +711,12 @@ describe('a write fenced to an owner the pane has not seen replaced', () => { acquire.mockRejectedValueOnce(new Error('Not signed in')) const subscribed = frames.length - expect(await host.send(CALLER, sendParams('while signed out'))).toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } - }) + const id = await accept(sendParams('while signed out')) + expect(await settled(id)).toMatchObject({ + dispatchState: 'rejected', + reason: expect.stringContaining('Not signed in') + }) expect(store.getRecord(SESSION)?.lease.runtimeFence).toBeGreaterThan(seenFence + 1) const published = frames.slice(subscribed) expect(published.length).toBeGreaterThan(0) @@ -736,7 +738,8 @@ describe('a write fenced to an owner the pane has not seen replaced', () => { return spawnChild(input) }) - const first = host.send(CALLER, sendParams('starts the agent')) + const firstParams = sendParams('starts the agent') + const first = host.send(CALLER, firstParams) await claim const cancelFields = { turnId: 'turn-1' } const stop = host.cancel(CALLER, { @@ -760,7 +763,14 @@ describe('a write fenced to an owner the pane has not seen replaced', () => { expect(await first).toMatchObject({ ok: true }) expect(await stop).toMatchObject({ ok: true, replayed: false }) expect(await second).toMatchObject({ ok: true, replayed: false }) + // The Stop withdrew the message its start held; the one typed after it is delivered. + expect(await settled(late.envelope.clientOperationId)).toMatchObject({ + dispatchState: 'accepted' + }) + expect(submission(firstParams.envelope.clientOperationId)).toMatchObject({ + dispatchState: 'rejected' + }) expect(acquire).toHaveBeenCalledOnce() - expect(dispatch).toHaveBeenCalledTimes(2) + expect(dispatch).toHaveBeenCalledTimes(1) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.ts index 09986f2e129..1fa923e971f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.ts @@ -1,23 +1,8 @@ -// What a send needs from the session before its lease is checked. -// -// A provider child that exits or fails to start hands its lease back. Before this, a send to that -// session was refused `agent_session_ownership_unknown` — which a client reads as "not admitted -// yet" and resends forever — and only a surface hold could ever make a new child. Now the send -// makes sure it has an owner as a step of its own serialized admission: a released lease where -// resume is allowed gets a child first; anything else runs as it is and meets the lease check. -// A restart that fails refuses with a code the client stops auto-retrying on, carrying the -// restart's own cause, and writes that cause into the chat the way a start that failed does, so -// the user sees why. A manual Retry or a new send is a fresh attempt: a refusal before admission -// leaves no ledger row behind. -// -// The ledger's answer comes first, so a send it already holds a row for restarts nothing: -// admission replays or refuses it whoever owns the session now, and a closed session is made -// readable for that, never given a child. Otherwise each resend of a message whose child died at -// startup would spawn another child that dies the same way. -// -// A child that has not proven its start is still the owner: the send is admitted against it and -// the adapter holds the message until startup lands, or rejects it with the child's own reason -// when the child dies first. The exit settlement writes that reason into the chat. +// What a send or a Stop needs from the session before the ledger places its row: the +// conversation open. Nothing here needs an owner — a send is accepted into the conversation and +// the delivery loop makes the session ready — so a refusal before acceptance is only one the +// conversation itself makes: a rewind or conversation command in doubt, a cleared conversation, +// or a journal that cannot be opened. import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { @@ -25,45 +10,39 @@ import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' import type { AgentSessionWireRefusalCode } from '../../../shared/agent-session-wire-refusals' -import { boundJournalStatusText } from '../agent-session-journal/journal-prompt-body-bounds' import { TUI_AGENT_DISPLAY_NAMES } from '../../../shared/tui-agent-display-names' import { ownerRestartFailedOutcome, providerStartupFailureOutcome } from './structured-agent-session-dead-generation-settlement' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' -import type { StructuredAgentSessionMutationContext } from './structured-agent-session-host-mutations' -import type { AgentSessionMutationSessionPreparation } from './structured-agent-session-mutation-admission' -import { isResumableStructuredAgentSessionRecord } from './structured-agent-session-resume-eligibility' +import { + AGENT_SESSION_NOT_ATTACHED, + type AgentSessionMutationSessionPreparation +} from './structured-agent-session-mutation-admission' import { rewindRefusal } from './structured-rewind-refusal' /** - * What a refused resume means for the send that ran it. `transient`: the resume met a lease - * someone else is settling, which is not proof it cannot resume — the send runs as the lease - * stands and admission reports it. `failed`: the restart itself failed; the send answers with the - * cause and stops the client's retry loop, and the user may clear the cause and retry. - * `unresumable`: this host has nothing to restart the chat from — no record, or none it can run — - * so only a new chat continues. A new wire code does not compile until it is classified here. + * Whether a refused start leaves the chat anything to start again from. `unresumable`: this host + * has nothing to restart it from — no record, or none it can run — so only a new chat continues. + * A new wire code does not compile until it is classified here. */ -const RESUME_REFUSAL_OUTCOME: Record< - AgentSessionWireRefusalCode, - 'transient' | 'failed' | 'unresumable' -> = { - execution_owner_reconciling: 'transient', - agent_session_conflict: 'transient', - agent_session_checkpoint_stale: 'transient', - agent_session_ownership_unknown: 'transient', - agent_session_operation_capacity: 'transient', - structured_agent_session_unsupported: 'unresumable', - agent_session_operation_conflict: 'failed', - agent_session_operation_expired: 'failed', - agent_session_operation_invalid: 'failed', - agent_session_operation_unknown: 'failed', - agent_session_item_revision_stale: 'failed', - agent_session_already_resolved: 'failed', - agent_session_identity_required: 'unresumable', - agent_session_journal_unreadable: 'failed', - agent_session_owner_restart_failed: 'failed' +const START_REFUSAL_RESUMABLE: Record = { + execution_owner_reconciling: true, + agent_session_conflict: true, + agent_session_checkpoint_stale: true, + agent_session_ownership_unknown: true, + agent_session_operation_capacity: true, + structured_agent_session_unsupported: false, + agent_session_operation_conflict: true, + agent_session_operation_expired: true, + agent_session_operation_invalid: true, + agent_session_operation_unknown: true, + agent_session_item_revision_stale: true, + agent_session_already_resolved: true, + agent_session_identity_required: false, + agent_session_journal_unreadable: true, + agent_session_owner_restart_failed: true } /** Why the record refuses any send right now, whoever owns it; null when a send may run. */ @@ -93,133 +72,41 @@ export function structuredAgentSessionSendBlock( return null } -/** Whether this send is the one that must bring the owner back: no child, a lease handed back - * cleanly, and nothing on the record that refuses the send anyway. Live, unverifiable, still - * reserved, or handed off: that lease is not this send's to replace. */ -export function structuredAgentSessionSendNeedsOwner( - session: StructuredAgentSessionHostSession | undefined, - record: AgentSessionRecord -): boolean { - return ( - session?.hasProviderChild !== true && - isResumableStructuredAgentSessionRecord(record) && - structuredAgentSessionSendBlock(record) === null - ) -} - -type SendPreparationContext = Pick< - StructuredAgentSessionMutationContext, - 'deps' | 'sessions' | 'holds' | 'restoreReadable' -> - -export async function prepareStructuredAgentSessionSend( - context: SendPreparationContext, - envelope: AgentSessionMutationEnvelope, - ledger: 'admit' | 'replay', - record: AgentSessionRecord +/** The conversation a send or a Stop writes to, opened when this host holds it closed. */ +export async function openConversationForWrite( + openConversation: (sessionId: string) => Promise, + envelope: AgentSessionMutationEnvelope ): Promise { - const { sessionId } = record - if (ledger !== 'admit') { - if (!context.sessions.has(sessionId)) { - await context.restoreReadable(sessionId) + try { + if (await openConversation(envelope.sessionId)) { + return { ok: true } } - return { ok: true } - } - if (structuredAgentSessionSendNeedsOwner(context.sessions.get(sessionId), record)) { - const refusal = await restartOwnerForSend(context, envelope, record) - if (refusal) { - return { ok: false, refusal } + return { ok: false, refusal: AGENT_SESSION_NOT_ATTACHED } + } catch (error) { + return { + ok: false, + refusal: { + code: 'agent_session_journal_unreadable', + message: `The conversation could not be opened: ${ + error instanceof Error ? error.message : String(error) + }` + } } } - return { ok: true } } -/** One restart attempt. Answers with the refusal that ends the send, or null when the send goes - * on to admission — after a child, after a transient refusal, or after a fault in the restart's - * own bookkeeping, which is reported and never gates the user's action. */ -async function restartOwnerForSend( - context: SendPreparationContext, - envelope: AgentSessionMutationEnvelope, - record: AgentSessionRecord -): Promise { - const { sessionId } = envelope - let resumed: Awaited> - try { - resumed = await context.holds.ensureProviderChild(sessionId) - } catch (error) { - context.deps.onEventSinkError?.({ sessionId, error }) - return null +/** What the chat says, in its row and on every message it rejects, when the delivery loop could + * not make the session ready. A child that died starting reads as any start that died does. */ +export function structuredAgentSessionStartFailureText( + record: AgentSessionRecord | null, + cause: AgentSessionWireRefusal +): string { + if (cause.ownerVerdict === 'exited') { + return providerStartupFailureOutcome(cause.message) } - const outcome = resumed.ok ? null : RESUME_REFUSAL_OUTCOME[resumed.refusal.code] - if (resumed.ok || outcome === 'transient') { - return null - } - const refusal = ownerRestartFailedRefusal(record, resumed.refusal, outcome !== 'unresumable') - context.deps.onEventSinkError?.({ - sessionId, - error: new Error(`${resumed.refusal.code}: ${resumed.refusal.message}`) + return ownerRestartFailedOutcome({ + agentName: record ? TUI_AGENT_DISPLAY_NAMES[record.provider] : 'The agent', + reason: cause.message, + resumable: START_REFUSAL_RESUMABLE[cause.code] }) - // A restart whose child died starting leaves the row any start that died leaves, so the chat - // reads the same whether the send met that death before admission or after it. - await recordFailedRestart( - context, - envelope, - resumed.refusal.ownerVerdict === 'exited' - ? providerStartupFailureOutcome(resumed.refusal.message) - : refusal.message - ) - return refusal -} - -/** The client stops on the code; the message carries the restart's own cause, and the verdict — - * when the failed attach proved its child gone — tells a client nothing runs for the session. */ -function ownerRestartFailedRefusal( - record: AgentSessionRecord, - cause: AgentSessionWireRefusal, - resumable: boolean -): AgentSessionWireRefusal { - return { - code: 'agent_session_owner_restart_failed', - message: ownerRestartFailedOutcome({ - agentName: TUI_AGENT_DISPLAY_NAMES[record.provider], - reason: cause.message, - resumable - }), - ...(cause.ownerVerdict ? { ownerVerdict: cause.ownerVerdict } : {}) - } -} - -/** The same status row a start that failed leaves in the chat, so the reason outlives the error - * strip. The journal is made readable for it when the failed attach left none behind. Keyed by - * the send, not the clock: a resend of the same id that fails again adds no second row. */ -async function recordFailedRestart( - context: SendPreparationContext, - envelope: AgentSessionMutationEnvelope, - text: string -): Promise { - const { sessionId } = envelope - try { - if (!context.sessions.has(sessionId)) { - await context.restoreReadable(sessionId) - } - const session = context.sessions.get(sessionId) - if (!session) { - return - } - const settlementId = `failed-restart:${envelope.clientOperationId}` - await session.journal.appendLifecycleBatch({ - settlementId, - fence: session.fence, - recovered: true, - mutations: [ - { - kind: 'item', - identity: { provider: 'orca', clientMessageId: settlementId }, - body: { kind: 'status', text: boundJournalStatusText(text) } - } - ] - }) - } catch (error) { - context.deps.onEventSinkError?.({ sessionId, error }) - } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-restarts-failed-start.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-restarts-failed-start.test.ts index e2e2fb81102..211d8684e15 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-restarts-failed-start.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-restarts-failed-start.test.ts @@ -1,12 +1,11 @@ // A session that published and then lost its child before startup (not signed in, say) keeps a // released lease and a chat the user can still type into. The send is the user asking for the -// child back: the host restarts it before admitting the write and delivers against the new owner, -// instead of parking the message behind a lease nothing would ever re-acquire. +// child back: the host accepts the message, and its delivery restarts the child and hands the +// message to the new owner, instead of parking it behind a lease nothing would ever re-acquire. // // A child is published before it has proven its start, and it owns the send from that moment: the -// message is admitted against it and the adapter holds it for the start. When the child exits -// first, the exit settlement rejects the message and writes the cause into the chat, once, and -// the next send is a fresh restart. +// message is handed to it. When the child exits first, the exit settlement rejects the message and +// writes the cause into the chat, once, and the next send is a fresh restart. import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -28,6 +27,11 @@ import { } from './structured-agent-session-host-test-data' const CALLER = { callerKey: 'client-1' } + +/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */ +function eventually(assertion: () => void): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} const EXIT_REASON = 'Claude Code is not signed in. Sign in with the Claude CLI' let root: string @@ -53,7 +57,7 @@ function sendEnvelope( } } -/** A send is admitted against the child it meets, proven or not; the adapter holds the rest. */ +/** A send is accepted at once and handed to the child delivery finds or starts. */ async function send( text: string, fence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 @@ -63,9 +67,11 @@ async function send( expect(sent, JSON.stringify(sent)).toMatchObject({ ok: true, replayed: false, - value: { submission: { dispatchState: 'pending' } } + value: { submission: { dispatchState: 'pending', handoverRecorded: true } } }) - return sent.ok ? sent.value.clientMessageId : '' + const clientMessageId = sent.ok ? sent.value.clientMessageId : '' + await eventually(() => expect(submission(clientMessageId)?.handedOverAt).toBeDefined()) + return clientMessageId } /** The child of the current acquisition, as the adapter would identify it in a lifecycle event. */ @@ -166,8 +172,7 @@ describe('a send into a published session whose child ended before startup', () await send('hello again', releasedFence) - // A send still fenced to the lost owner is admitted once against the restarted one, with no - // stale round trip. + // Accepted at the lost owner's fence and handed to the child delivery started, once. expect(acquire).toHaveBeenCalledTimes(2) expect(store.getRecord(SESSION)?.lease.claimStatus).toBe('live') expect(dispatch).toHaveBeenCalledOnce() @@ -207,7 +212,7 @@ describe('a send into a published session whose child ended before startup', () // One spawn per user action: nothing restarted it a second time. expect(acquire).toHaveBeenCalledTimes(2) - // Retry is a fresh action: it restarts once and is admitted against the new child. + // Retry is a fresh action: it restarts once and is handed to the new child. await send('signed in now') expect(acquire).toHaveBeenCalledTimes(3) expect(dispatch).toHaveBeenCalledTimes(2) @@ -216,7 +221,7 @@ describe('a send into a published session whose child ended before startup', () }) describe('a send while the child of the first start is still proving itself', () => { - it('is admitted against the starting child, and nothing restarts it', async () => { + it('is handed to the starting child, and nothing restarts it', async () => { await send('hello') expect(dispatch).toHaveBeenCalledOnce() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts index b461d508f42..ff9765d1cee 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts @@ -61,7 +61,7 @@ describe('structured send settlement compatibility wait', () => { it('removes an abandoned wait on transport cancellation', async () => { const settlements = new StructuredAgentSessionSendSettlement(() => journal('pending')) const controller = new AbortController() - const pending = settlements.wait('session-1', 'client-1', controller.signal) + const pending = settlements.wait('session-1', 'client-1', { signal: controller.signal }) controller.abort(new Error('transport closed')) await expect(pending).rejects.toThrow('transport closed') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts index 6150e3412a6..bc265833c42 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts @@ -3,8 +3,12 @@ import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' import type { AgentSessionSendResult } from '../../../shared/agent-session-wire' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +/** What a settlement read needs of a journal. */ +type SendSettlementJournal = Pick + type SettledSend = { cursor: AgentJournalCursor value: AgentSessionSendResult @@ -12,8 +16,19 @@ type SettledSend = { type SendSettlement = SettledSend | 'pending' | 'missing' +/** What ends a wait: the provider's answer, or only the host handing the message over. */ +export type SendSettlementPoint = 'answered' | 'handed-over' + +export type SendSettlementWaitOptions = { + signal?: AbortSignal + /** How long to observe; unanswered by then resolves undefined. */ + budgetMs?: number + until?: SendSettlementPoint +} + type SendSettlementWaiter = { clientMessageId: string + until: SendSettlementPoint resolve: (result: SettledSend | undefined) => void reject: (error: Error) => void timer: ReturnType @@ -23,12 +38,15 @@ type SendSettlementWaiter = { // Known legacy clients abandon the RPC after 15s without cancelling its socket dispatch. const SEND_SETTLEMENT_WAIT_TIMEOUT_MS = 30_000 +/** Long enough for a cold provider start; a longer one replays through the same wait. */ +export const STRUCTURED_AGENT_SESSION_START_WAIT_MS = 120_000 const MAX_SEND_SETTLEMENT_WAITERS_PER_SESSION = 64 const MAX_SEND_SETTLEMENT_WAITERS = 1_024 function settledSend( - journal: AgentSessionJournal, + journal: SendSettlementJournal, clientMessageId: string, + until: SendSettlementPoint, submission: AgentJournalSubmission | undefined = journal .submissions() .find((candidate) => candidate.clientMessageId === clientMessageId) @@ -36,9 +54,11 @@ function settledSend( if (!submission) { return 'missing' } - return submission.dispatchState === 'pending' - ? 'pending' - : { cursor: journal.cursor(), value: { clientMessageId, submission } } + const waiting = + until === 'handed-over' + ? isQueuedAgentJournalSubmission(submission) + : submission.dispatchState === 'pending' + return waiting ? 'pending' : { cursor: journal.cursor(), value: { clientMessageId, submission } } } function abortError(signal: AbortSignal): Error { @@ -52,17 +72,19 @@ export class StructuredAgentSessionSendSettlement { private readonly waiters = new Map>() private waiterCount = 0 - constructor(private readonly journalFor: (sessionId: string) => AgentSessionJournal) {} + constructor(private readonly journalFor: (sessionId: string) => SendSettlementJournal) {} wait = ( sessionId: string, clientMessageId: string, - signal?: AbortSignal + options: SendSettlementWaitOptions = {} ): Promise => { + const { signal } = options + const until = options.until ?? 'answered' if (signal?.aborted) { return Promise.reject(abortError(signal)) } - const immediate = settledSend(this.journalFor(sessionId), clientMessageId) + const immediate = settledSend(this.journalFor(sessionId), clientMessageId, until) if (immediate === 'missing') { return Promise.reject(new Error('agent session send disappeared before settlement')) } @@ -79,12 +101,13 @@ export class StructuredAgentSessionSendSettlement { return new Promise((resolve, reject) => { const waiter: SendSettlementWaiter = { clientMessageId, + until, resolve, reject, timer: setTimeout(() => { this.remove(sessionId, waiter) resolve(undefined) - }, SEND_SETTLEMENT_WAIT_TIMEOUT_MS) + }, options.budgetMs ?? SEND_SETTLEMENT_WAIT_TIMEOUT_MS) } waiter.timer.unref?.() const session = existingSession ?? new Set() @@ -106,7 +129,7 @@ export class StructuredAgentSessionSendSettlement { }) } - publish(sessionId: string, journal: AgentSessionJournal): void { + publish(sessionId: string, journal: SendSettlementJournal): void { const waiters = this.waiters.get(sessionId) if (!waiters) { return @@ -118,6 +141,7 @@ export class StructuredAgentSessionSendSettlement { const result = settledSend( journal, waiter.clientMessageId, + waiter.until, submissions.get(waiter.clientMessageId) ) if (result !== 'pending') { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send.test.ts index 638d8a8679f..1a6e5606c1c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send.test.ts @@ -32,6 +32,28 @@ beforeEach(() => { ;({ store, host, dispatch } = hostTestState()) }) +function hostJournal(): AgentSessionJournal { + return ( + host as unknown as { sessions: Map } + ).sessions.get(SESSION)!.journal +} + +/** A send is accepted, then the session's delivery loop hands it over: wait for the handover's + * outcome, or with `handedOver`, only for the handover itself (an admitted send stays pending). */ +async function delivered(clientMessageId: string, options: { handedOver?: true } = {}) { + let submission: ReturnType[number] | undefined + await vi.waitFor(() => { + submission = hostJournal() + .submissions() + .find((entry) => entry.clientMessageId === clientMessageId) + expect(submission?.handedOverAt).toBeDefined() + if (!options.handedOver) { + expect(submission?.dispatchState).not.toBe('pending') + } + }) + return submission! +} + describe('send', () => { it('writes the submission before dispatching and resolves it accepted', async () => { await attach() @@ -43,7 +65,15 @@ describe('send', () => { if (!result.ok) { throw new Error(`expected a send, got ${result.refusal.code}`) } - expect(result.value.submission.dispatchState).toBe('accepted') + // Answered once accepted; the delivery loop hands it over after. + expect(result.value.submission).toMatchObject({ + dispatchState: 'pending', + handoverRecorded: true + }) + expect(result.value.submission.handedOverAt).toBeUndefined() + await expect(delivered(result.value.clientMessageId)).resolves.toMatchObject({ + dispatchState: 'accepted' + }) expect(dispatch).toHaveBeenCalledTimes(1) const page = host.history({ sessionId: SESSION, direction: 'tail' }) expect(page.ok && page.page.items).toHaveLength(1) @@ -75,11 +105,11 @@ describe('send', () => { await attach() dispatch.mockRejectedValueOnce(new Error('socket closed')) const body = hostTestMessage('add a retry') - const result = await host.send(CALLER, { - envelope: envelope('agentSession.send', { body }), - body + const params = { envelope: envelope('agentSession.send', { body }), body } + await host.send(CALLER, params) + await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'unknown' }) - expect(result).toMatchObject({ ok: true, value: { submission: { dispatchState: 'unknown' } } }) }) it('replays a retried send from the journal without dispatching twice', async () => { @@ -87,6 +117,7 @@ describe('send', () => { const body = hostTestMessage('add a retry') const params = { envelope: envelope('agentSession.send', { body }), body } await host.send(CALLER, params) + await delivered(params.envelope.clientOperationId) const retry = await host.send(CALLER, params) expect(retry).toMatchObject({ ok: true, replayed: true }) expect(dispatch).toHaveBeenCalledTimes(1) @@ -98,10 +129,9 @@ describe('send', () => { const body = hostTestMessage('possibly delivered') const params = { envelope: envelope('agentSession.send', { body }), body } - const first = await host.send(CALLER, params) - expect(first).toMatchObject({ - ok: true, - value: { submission: { dispatchState: 'unknown' } } + await host.send(CALLER, params) + await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'unknown' }) // A thrown adapter call is indistinguishable from a lost reply, so Retry // replays the recorded outcome. @@ -129,6 +159,7 @@ describe('send', () => { const params = { envelope: envelope('agentSession.send', { body }), body } await host.send(CALLER, params) + await delivered(params.envelope.clientOperationId) await expect(host.send(CALLER, { ...params, retryUnknown: true })).resolves.toMatchObject({ ok: true, value: { @@ -151,21 +182,18 @@ describe('send', () => { const body = hostTestMessage('never written') const params = { envelope: envelope('agentSession.send', { body }), body } - await expect(host.send(CALLER, params)).resolves.toMatchObject({ - ok: true, - value: { - submission: { dispatchState: 'rejected', reason: 'provider_write_failed: broken pipe' } - } + await host.send(CALLER, params) + await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'rejected', + reason: 'provider_write_failed: broken pipe' }) // What the user's Retry does with a rejection: a fresh client message id, // which is a first delivery by construction and cannot duplicate the frame // that never left the process. - await expect( - host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) - ).resolves.toMatchObject({ - ok: true, - replayed: false, - value: { submission: { dispatchState: 'accepted' } } + const rotated = { envelope: envelope('agentSession.send', { body }), body } + await expect(host.send(CALLER, rotated)).resolves.toMatchObject({ ok: true, replayed: false }) + await expect(delivered(rotated.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'accepted' }) expect(dispatch).toHaveBeenCalledTimes(2) const state = host.history({ sessionId: SESSION, direction: 'tail' }) @@ -183,10 +211,9 @@ describe('send', () => { const body = hostTestMessage('a message the provider may already hold') const params = { envelope: envelope('agentSession.send', { body }), body } - const first = await host.send(CALLER, params) - expect(first).toMatchObject({ - ok: true, - value: { submission: { dispatchState: 'unknown' } } + await host.send(CALLER, params) + await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'unknown' }) // No `unknown` is re-delivered under its own id, whatever its reason says, // so Retry replays the recorded outcome instead of writing again. @@ -203,6 +230,7 @@ describe('send', () => { const body = hostTestMessage('settled for good') const params = { envelope: envelope('agentSession.send', { body }), body } await host.send(CALLER, params) + await delivered(params.envelope.clientOperationId) const journal = ( host as unknown as { sessions: Map } ).sessions.get(SESSION)!.journal @@ -223,7 +251,7 @@ describe('send', () => { expect(journal.receiptFor(params.envelope.clientOperationId)).not.toBeNull() }) - it('leaves an admitted send pending and writes no dispatch row', async () => { + it('leaves an admitted send pending once handed over', async () => { await attach() dispatch.mockImplementationOnce(async () => ({ state: 'admitted' as const })) const body = hostTestMessage('queued behind a running turn') @@ -233,9 +261,9 @@ describe('send', () => { ok: true, value: { submission: { dispatchState: 'pending', reason: null, resolvedAt: null } } }) - const journal = ( - host as unknown as { sessions: Map } - ).sessions.get(SESSION)!.journal + await delivered(params.envelope.clientOperationId, { handedOver: true }) + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1)) + const journal = hostJournal() expect(journal.pendingSubmissions()).toHaveLength(1) }) @@ -245,6 +273,8 @@ describe('send', () => { const body = hostTestMessage('written, never acknowledged') const params = { envelope: envelope('agentSession.send', { body }), body } await host.send(CALLER, params) + await delivered(params.envelope.clientOperationId, { handedOver: true }) + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1)) const journal = ( host as unknown as { sessions: Map } ).sessions.get(SESSION)!.journal @@ -281,6 +311,8 @@ describe('send', () => { const params = { envelope: envelope('agentSession.send', { body }), body } await expect(host.send(CALLER, params)).rejects.toThrow('operation settlement failed') + // The submission was recorded before the ledger write failed, so it is still delivered. + await delivered(params.envelope.clientOperationId) await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true, replayed: true, @@ -333,9 +365,8 @@ describe('send', () => { await expect(host.send(CALLER, params)).rejects.toThrow('operation settlement failed') settlement.mockRestore() - const journal = ( - host as unknown as { sessions: Map } - ).sessions.get(SESSION)!.journal + await delivered(params.envelope.clientOperationId) + const journal = hostJournal() await journal.rollEpoch('schema_unreadable', store.getRecord(SESSION)?.lease.runtimeFence ?? 1) expect(journal.submissions()).toHaveLength(0) @@ -362,6 +393,7 @@ describe('send', () => { const params = { envelope: envelope('agentSession.send', { body }), body } await host.send(CALLER, params) + await delivered(params.envelope.clientOperationId) expect(dispatch).toHaveBeenCalledTimes(1) await store.recordOperationOutcome({ callerKey: CALLER.callerKey, @@ -393,6 +425,8 @@ describe('send', () => { const body = hostTestMessage('written, then the child died') const params = { envelope: envelope('agentSession.send', { body }), body } await host.send(CALLER, params) + await delivered(params.envelope.clientOperationId, { handedOver: true }) + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1)) const journal = ( host as unknown as { sessions: Map } ).sessions.get(SESSION)!.journal @@ -411,21 +445,25 @@ describe('send', () => { it('advances an explicit retry after a ledger-unknown send is reconciled in the journal', async () => { await attach() - const journal = ( - host as unknown as { sessions: Map } - ).sessions.get(SESSION)!.journal - vi.spyOn(journal, 'resolveDispatch').mockRejectedValueOnce(new Error('journal resolve failed')) + const journal = hostJournal() + const resolve = journal.resolveDispatch.bind(journal) + // The handover row lands; the provider's answer, written after the adapter took the + // message, does not. + vi.spyOn(journal, 'resolveDispatch') + .mockImplementationOnce(resolve) + .mockRejectedValueOnce(new Error('journal resolve failed')) const body = hostTestMessage('possibly delivered before persistence failed') const params = { envelope: envelope('agentSession.send', { body }), body } - await expect(host.send(CALLER, params)).rejects.toThrow('journal resolve failed') - expect(journal.submissions()).toMatchObject([ - { clientMessageId: params.envelope.clientOperationId, dispatchState: 'unknown' } - ]) + await expect(host.send(CALLER, params)).resolves.toMatchObject({ ok: true }) + await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'unknown' + }) + // Acceptance is what the ledger answers for; delivery is the journal's to say. expect( store.listOperationRows().find((row) => row.operationId === params.envelope.clientOperationId) ?.outcome - ).toEqual({ status: 'unknown' }) + ).toMatchObject({ status: 'succeeded' }) expect(dispatch).toHaveBeenCalledTimes(1) await journal.markPendingSubmissionsUnknown(store.getRecord(SESSION)?.lease.runtimeFence ?? 1) @@ -446,7 +484,7 @@ describe('send', () => { expect(journal.submissions()).toHaveLength(1) }) - it('admits a send fenced to another generation, and replays it by id once the fence catches up', async () => { + it('admits a send fenced to another generation, delivers it once, and replays it by id', async () => { const record = await attach() const body = hostTestMessage('add a retry') const params = { @@ -457,10 +495,9 @@ describe('send', () => { ), body } - expect(await host.send(CALLER, params)).toMatchObject({ - ok: true, - replayed: false, - value: { submission: { dispatchState: 'accepted' } } + expect(await host.send(CALLER, params)).toMatchObject({ ok: true, replayed: false }) + await expect(delivered(params.envelope.clientOperationId)).resolves.toMatchObject({ + dispatchState: 'accepted' }) const retry = { ...params, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-settled-attach-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-settled-attach-retry.test.ts index f0dd60965ee..711e38b29b8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-settled-attach-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-settled-attach-retry.test.ts @@ -314,6 +314,8 @@ describe('settled attach retry', () => { } const first = await host.send(CALLER, unknownParams) expect(first).toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) + // Handed over before the host dies: that is what makes the restart's answer doubt. + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1)) await host.flushAllStreamedEvents() store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) @@ -343,7 +345,7 @@ describe('settled attach retry', () => { if (!sent.ok) { throw new Error(`unexpected restored send refusal: ${sent.refusal.message}`) } - expect(dispatch).toHaveBeenCalledTimes(2) + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(2)) const restoredHistory = host.history({ sessionId: SESSION, direction: 'tail' }) if (!restoredHistory.ok) { throw new Error(`unexpected restored history reset: ${restoredHistory.reset}`) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts new file mode 100644 index 00000000000..00594966101 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts @@ -0,0 +1,59 @@ +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' +import { boundJournalStatusText } from '../agent-session-journal/journal-prompt-body-bounds' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' + +/** + * The one row a start that failed leaves in the chat, whoever saw it fail: an error row, so the + * reason outlives any error strip. Keyed by the start — the child's generation, or the oldest + * message it was for when no child was ever published — so a second report of the same failure + * revises the row instead of adding one. + */ +export function structuredAgentSessionStartFailureRow( + startKey: string, + text: string +): JournalLifecycleMutationInput { + return { + kind: 'item', + identity: { provider: 'orca', clientMessageId: `start-failure:${startKey}` }, + body: { kind: 'status', text: boundJournalStatusText(text), tone: 'error' } + } +} + +/** + * A start the delivery loop needed and did not get: the start's row, and every queued message + * rejected with the same words. Writes nothing when nothing is still queued: a start whose + * messages Stop withdrew did not fail anyone. + */ +export async function recordStructuredAgentSessionStartFailure( + session: Pick & { fence: number }, + failure: { startKey: string | null; text: string } +): Promise { + const oldest = oldestQueuedSubmission(session) + if (!oldest) { + return + } + const startKey = failure.startKey ?? oldest.clientMessageId + await session.journal.appendLifecycleBatch({ + settlementId: `start-failure:${startKey}`, + fence: session.fence, + recovered: true, + mutations: [structuredAgentSessionStartFailureRow(startKey, failure.text)] + }) + await session.journal.rejectQueuedSubmissions(session.fence, failure.text) +} + +export function oldestQueuedSubmission( + session: Pick +): ReturnType[number] | undefined { + let oldest: ReturnType + for (const submission of session.journal.submissions()) { + if ( + isQueuedAgentJournalSubmission(submission) && + (oldest === undefined || (submission.acceptedSequence ?? 0) < (oldest.acceptedSequence ?? 0)) + ) { + oldest = submission + } + } + return oldest +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts index 0fd544ec9ee..88774c63bed 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts @@ -19,10 +19,9 @@ function startedSession(): StructuredAgentSessionUnexpectedExitSession & { journal: { appendLifecycleBatch: ReturnType } } { return { - hasProviderChild: true, - fence: 7, - acquisitionGeneration: GENERATION, + child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { + cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), // Nothing ran: the start failed before any response or acknowledged prompt. snapshot: () => ({ items: [] }), appendLifecycleBatch: vi.fn(async () => ({ epoch: 'epoch-1', sequence: 1 })), @@ -86,7 +85,9 @@ describe('a provider that ends before it finished starting', () => { expect.objectContaining({ mutations: [ expect.objectContaining({ - body: { kind: 'status', text: providerStartupFailureOutcome(REASON) } + // The same row the delivery loop writes for a failed start: an error, keyed by it. + identity: { provider: 'orca', clientMessageId: `start-failure:${GENERATION}` }, + body: { kind: 'status', text: providerStartupFailureOutcome(REASON), tone: 'error' } }) ] }) @@ -104,7 +105,10 @@ describe('a provider that ends before it finished starting', () => { }) it("reads a start that failed off the host's own phase when the provider omits the flag", async () => { - const session = { ...startedSession(), providerChildPhase: 'starting' as const } + const session = { + ...startedSession(), + child: { generation: GENERATION, fence: 7, phase: 'starting' as const } + } const ticket = await settleUnexpectedStructuredAgentSessionExit(contextFor(session), ended) @@ -113,7 +117,9 @@ describe('a provider that ends before it finished starting', () => { expect.objectContaining({ mutations: [ expect.objectContaining({ - body: { kind: 'status', text: providerStartupFailureOutcome(REASON) } + // The same row the delivery loop writes for a failed start: an error, keyed by it. + identity: { provider: 'orca', clientMessageId: `start-failure:${GENERATION}` }, + body: { kind: 'status', text: providerStartupFailureOutcome(REASON), tone: 'error' } }) ] }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-test-session.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-test-session.ts index e4fa6ebed1b..c130aaf9d59 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-test-session.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-test-session.ts @@ -2,17 +2,11 @@ import type { AgentSessionJournal } from '../agent-session-journal/journal-store export function indexedStatusFeedSession(session: { journal: AgentSessionJournal - hasProviderChild?: boolean - providerChildPhase?: 'starting' | 'ready' - fence?: number + child?: { phase: 'starting' | 'ready' } | null }) { return { journal: session.journal, - fence: session.fence ?? 1, - ...(session.hasProviderChild !== undefined - ? { hasProviderChild: session.hasProviderChild } - : {}), - ...(session.providerChildPhase ? { providerChildPhase: session.providerChildPhase } : {}), + ...(session.child !== undefined ? { child: session.child } : {}), params: { location: { executionHostId: 'local' as const, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts index 65c7f54fb30..e5da2eece97 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { agentSessionRecordFixture } from '../../../shared/agent-session-record.test-fixture' import type { AgentSessionBackgroundTask, AgentSessionStatusEvent, @@ -33,6 +34,8 @@ const USER_IDENTITY = { ordinal: 1 } as const +type Indexed = Parameters[0] + let root: string const journals = createTrackedJournalOpener() @@ -82,7 +85,8 @@ function feedFor( } } } as unknown as ReadonlyMap>, - getRecord: () => record as AgentSessionRecord | null, + // A partial record still has a lease: the feed reads the conversation's fence off it. + getRecord: () => (record ? { ...agentSessionRecordFixture(), ...record } : null), now: () => (now += 1) }) const events: AgentSessionStatusEvent[] = [] @@ -93,17 +97,17 @@ function feedFor( describe('StructuredAgentSessionStatusFeed', () => { it('projects whether the owned child has proven its start, and nothing once it is not owned', async () => { const journal = await openJournal() - const session = { journal, hasProviderChild: true, providerChildPhase: 'starting' as const } + const session = { journal, child: { phase: 'starting' as const } } const sessions = new Map[0]>([[SESSION, session]]) const { feed, events, dispose } = feedFor(sessions) expect(events.at(-1)).toMatchObject({ type: 'snapshot', sessions: [{ hostExecutionOwned: true, hostExecutionPhase: 'starting' }] }) - sessions.set(SESSION, { ...session, providerChildPhase: 'ready' }) + sessions.set(SESSION, { ...session, child: { phase: 'ready' } }) feed.publish(SESSION, journal) expect(events.at(-1)).toMatchObject({ session: { hostExecutionPhase: 'ready' } }) - sessions.set(SESSION, { ...session, hasProviderChild: false }) + sessions.set(SESSION, { ...session, child: null }) feed.publish(SESSION, journal) expect(events.at(-1)).not.toMatchObject({ session: { hostExecutionPhase: expect.any(String) } }) dispose() @@ -111,7 +115,7 @@ describe('StructuredAgentSessionStatusFeed', () => { it('publishes provider ownership transitions without changing journal time', async () => { const journal = await openJournal() - const sessions = new Map([[SESSION, { journal, hasProviderChild: true }]]) + const sessions = new Map([[SESSION, { journal, child: { phase: 'ready' } }]]) const { feed, events, dispose } = feedFor(sessions) events.length = 0 await journal.appendItem( @@ -130,7 +134,7 @@ describe('StructuredAgentSessionStatusFeed', () => { throw new Error('status publication missing') } const journalTime = firstStatus.session.updatedAt - sessions.get(SESSION)!.hasProviderChild = false + sessions.get(SESSION)!.child = null feed.publish(SESSION, journal) expect(events.at(-1)).toEqual({ type: 'status', @@ -167,8 +171,10 @@ describe('StructuredAgentSessionStatusFeed', () => { it('stops projecting an old-host unknown submission after the owner fence advances', async () => { const journal = await openJournal() - const session = { journal, fence: 1 } - const { feed, events } = feedFor(new Map([[SESSION, session]])) + // The conversation's fence is the record's: a child's end moves it. + const lease = agentSessionRecordFixture().lease + const record = agentSessionRecordFixture({ ...lease, runtimeFence: 1 }) + const { feed, events } = feedFor(new Map([[SESSION, { journal }]]), record) await journal.appendSubmission({ clientMessageId: 'old-host', payloadFingerprint: 'fp', @@ -183,7 +189,7 @@ describe('StructuredAgentSessionStatusFeed', () => { }) feed.publish(SESSION) expect(events.at(-1)).toMatchObject({ session: { status: 'working' } }) - session.fence = 2 + record.lease.runtimeFence = 2 feed.publish(SESSION) expect(events.at(-1)).toMatchObject({ session: { status: 'idle' } }) }) @@ -793,7 +799,7 @@ describe('the status sink sees the roster the broadcast cache deliberately lacks it('receives every change once, ownership revocation, and the forget edge', async () => { const journal = await openJournal() - const sessions = new Map([[SESSION, { journal, hasProviderChild: true }]]) + const sessions = new Map([[SESSION, { journal, child: { phase: 'ready' } }]]) const { sink, published, forgotten } = sinkFor() const { feed } = feedFor(sessions, null, undefined, undefined, sink) await journal.appendItem( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts index eef1c0e06f9..6e572f0366d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts @@ -42,9 +42,7 @@ export type StructuredAgentSessionStatusSubscriber = { type StatusFeedSession = { journal: AgentSessionJournal params: { location: AgentSessionRecord['location']; provider: AgentSessionRecord['provider'] } - hasProviderChild?: boolean - providerChildPhase?: StructuredAgentSessionProviderChildPhase - fence?: number + child?: { phase: StructuredAgentSessionProviderChildPhase } | null } export type StructuredAgentSessionStatusFeedDeps = { @@ -239,7 +237,9 @@ export class StructuredAgentSessionStatusFeed { // An unreadable journal projects as "no turn": the chat itself shows the reset. const cursor = journal.cursor() const readOnly = journal.isReadOnly - const fence = session.fence + const record = this.deps.getRecord(sessionId) + // The conversation's fence, which a child's end moves: its unanswered sends stop counting. + const fence = record?.lease.runtimeFence let projection = this.journalProjections.get(journal) if ( !projection || @@ -263,7 +263,6 @@ export class StructuredAgentSessionStatusFeed { } this.journalProjections.set(journal, projection) } - const record = this.deps.getRecord(sessionId) const providerSession = structuredAgentSessionProviderSessionMetadata(record) // The journal has no model: the record's acknowledged options are where a mid-session // switch lands, so the row follows whichever is in force. @@ -278,13 +277,8 @@ export class StructuredAgentSessionStatusFeed { sessionId, workspaceId: session.params.location.workspaceId, agent: session.params.provider, - ...(session.hasProviderChild - ? { - hostExecutionOwned: true as const, - ...(session.providerChildPhase - ? { hostExecutionPhase: session.providerChildPhase } - : {}) - } + ...(session.child + ? { hostExecutionOwned: true as const, hostExecutionPhase: session.child.phase } : {}), ...projection.summary, ...(record?.rewind?.phase === 'prepared' || record?.rewind?.phase === 'provider-succeeded' diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-subagent-recency.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-subagent-recency.test.ts index cf908932667..4053988233f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-subagent-recency.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-subagent-recency.test.ts @@ -63,7 +63,9 @@ async function openSession() { const roster: { tasks: AgentSessionBackgroundTask[] } = { tasks: [] } const server = new AgentHookServer() const feed = new StructuredAgentSessionStatusFeed({ - sessions: new Map([[SESSION, indexedStatusFeedSession({ journal, hasProviderChild: true })]]), + sessions: new Map([ + [SESSION, indexedStatusFeedSession({ journal, child: { phase: 'ready' } })] + ]), getRecord: () => null, now: () => 1, readBackgroundTasks: () => ({ state: 'monitoring', tasks: roster.tasks }), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts index 25c95b48dbe..cc145ad7995 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-surface-lifetime.test.ts @@ -295,6 +295,8 @@ describe('a chat that closes', () => { if (!result.ok) { throw new Error('send was refused') } + // Handed over first: a message still queued at close is rejected as never sent instead. + await vi.waitFor(() => expect(dispatch).toHaveBeenCalled()) const settlement = host.waitForSendSettlement(SESSION, result.value.clientMessageId) await host.close(SESSION) @@ -319,12 +321,10 @@ describe('a chat that closes', () => { }) .mockImplementation(closeJournal) - await expect(host.close(SESSION)).rejects.toMatchObject({ - step: 'forget-session', - cause: expect.objectContaining({ message: 'journal close result lost' }) - }) + // The child stopped and its lease went back; only the conversation's close is left to retry. + await expect(host.close(SESSION)).rejects.toThrow('journal close result lost') expect(host.hasSession(SESSION)).toBe(true) - expect(host['sessions'].get(SESSION)?.hasProviderChild).toBe(false) + expect(host['sessions'].get(SESSION)?.child).toBeNull() expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'released', ownerProcess: null @@ -362,7 +362,7 @@ describe('a chat that closes', () => { await expect(host.close(SESSION)).rejects.toMatchObject({ step: 'drain-published' }) // The child is proven gone, but the wind-down it owes is not done: nothing settled, no release. - expect(session!.hasProviderChild).toBe(false) + expect(session!.child).toBeNull() expect(store.getRecord(SESSION)?.lease.claimStatus).not.toBe('released') await expect(host.close(SESSION)).resolves.toBeUndefined() @@ -492,6 +492,12 @@ describe('a session evicted and opened again', () => { }) }) +function submissionState(clientMessageId: string): string | undefined { + return host + .journalSnapshot(SESSION) + .submissions.find((entry) => entry.clientMessageId === clientMessageId)?.dispatchState +} + describe('an unexpected provider exit', () => { it('publishes terminal settlement to a waiting older client', async () => { await attach() @@ -508,6 +514,8 @@ describe('an unexpected provider exit', () => { if (!result.ok) { throw new Error('send was refused') } + // Accepted first; the exit must meet a message the provider was handed. + await vi.waitFor(() => expect(dispatch).toHaveBeenCalledOnce()) const settlement = host.waitForSendSettlement(SESSION, result.value.clientMessageId) const exitedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 @@ -558,12 +566,14 @@ describe('an unexpected provider exit', () => { await host.hold(SESSION, SURFACE) dispatch.mockRejectedValueOnce(new Error('provider delivery became unknown')) const unknownBody = hostTestMessage('message with unknown delivery') + const unknownEnvelope = envelope('agentSession.send', { body: unknownBody }) await expect( - host.send(CALLER, { - envelope: envelope('agentSession.send', { body: unknownBody }), - body: unknownBody - }) - ).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'unknown' } } }) + host.send(CALLER, { envelope: unknownEnvelope, body: unknownBody }) + ).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) + // Accepted, then handed over by the delivery loop, where the thrown dispatch becomes doubt. + await vi.waitFor(() => + expect(submissionState(unknownEnvelope.clientOperationId)).toBe('unknown') + ) const exitedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 await host.handleAdapterEvent({ @@ -592,9 +602,12 @@ describe('an unexpected provider exit', () => { providerIdentity: { provider: 'codex', threadId: THREAD, turnId: 'turn-next', ordinal: 1 } }) const body = hostTestMessage('a distinct next message') - await expect( - host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) - ).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'accepted' } } }) + const nextEnvelope = envelope('agentSession.send', { body }) + await expect(host.send(CALLER, { envelope: nextEnvelope, body })).resolves.toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'pending' } } + }) + await vi.waitFor(() => expect(submissionState(nextEnvelope.clientOperationId)).toBe('accepted')) expect(dispatch).toHaveBeenCalledTimes(2) }) @@ -659,8 +672,11 @@ describe('an unexpected provider exit', () => { } await expect(host.send(CALLER, unknownParams)).resolves.toMatchObject({ ok: true, - value: { submission: { dispatchState: 'unknown' } } + value: { submission: { dispatchState: 'pending' } } }) + await vi.waitFor(() => + expect(submissionState(unknownParams.envelope.clientOperationId)).toBe('unknown') + ) const runtimeState = ( host as unknown as { runtimeState: { lifecycleBarrier: () => Promise<{ ok: false; error: Error }> } @@ -704,9 +720,12 @@ describe('an unexpected provider exit', () => { providerIdentity: { provider: 'codex', threadId: THREAD, turnId: 'turn-next', ordinal: 1 } }) const body = hostTestMessage('a distinct next message after failed-barrier recovery') - await expect( - host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) - ).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'accepted' } } }) + const nextEnvelope = envelope('agentSession.send', { body }) + await expect(host.send(CALLER, { envelope: nextEnvelope, body })).resolves.toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'pending' } } + }) + await vi.waitFor(() => expect(submissionState(nextEnvelope.clientOperationId)).toBe('accepted')) expect(dispatch).toHaveBeenCalledTimes(2) }) @@ -733,10 +752,10 @@ describe('an unexpected provider exit', () => { } ).sessions.get(SESSION) expect(session).toBeDefined() - // The dead generation's handle never accepts its settlement. - vi.spyOn(session!.journal, 'appendLifecycleBatch').mockRejectedValue( - new Error('settlement still unavailable') - ) + // The conversation's one handle refuses every write of the exit's settlement. + const refusing = vi + .spyOn(session!.journal, 'appendLifecycleBatch') + .mockRejectedValue(new Error('settlement still unavailable')) const exitedFence = store.getRecord(SESSION)?.lease.runtimeFence ?? 0 await host.handleAdapterEvent({ @@ -756,15 +775,20 @@ describe('an unexpected provider exit', () => { runtimeFence: exitedFence + 1, deathEvidence: { kind: 'exit-observed', detail: 'provider exited', observedAt: NOW } }) + // Nothing retries the settlement; the journal writes again, and the next acquire re-derives it. + refusing.mockRestore() dispatch.mockResolvedValueOnce({ state: 'accepted', providerIdentity: { provider: 'codex', threadId: THREAD, turnId: 'turn-next', ordinal: 1 } }) const body = hostTestMessage('sent after a settlement that never landed') - await expect( - host.send(CALLER, { envelope: envelope('agentSession.send', { body }), body }) - ).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'accepted' } } }) + const sentEnvelope = envelope('agentSession.send', { body }) + await expect(host.send(CALLER, { envelope: sentEnvelope, body })).resolves.toMatchObject({ + ok: true, + value: { submission: { dispatchState: 'pending' } } + }) + await vi.waitFor(() => expect(submissionState(sentEnvelope.clientOperationId)).toBe('accepted')) expect(acquire).toHaveBeenCalledTimes(2) // The new child's acquire settled the turn from the release's evidence: ended at the exit's // receipt, with the exit's own reason in the row. @@ -790,7 +814,7 @@ describe('a quit over an eviction that never got its retry', () => { failNextDrain() await expect(host.close(SESSION)).rejects.toMatchObject({ step: 'drain-published' }) - expect(host['sessions'].get(SESSION)?.hasProviderChild).toBe(false) + expect(host['sessions'].get(SESSION)?.child).toBeNull() expect(store.getRecord(SESSION)?.lease.claimStatus).not.toBe('released') await host.flushAllStreamedEvents() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts index 794b4fe8df9..5eac140868f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts @@ -25,11 +25,7 @@ import type { } from './structured-agent-session-adapter' import { providerStartupFailureRejection } from './structured-agent-session-dead-generation-settlement' import { validatePendingPrompt } from './structured-agent-session-prompt-state' -import { withTimeout } from '../../../shared/promise-timeout-fallback' -import { - AgentSessionPreDispatchError, - AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS -} from './structured-agent-session-operation-settlement' +import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key' export { performSetOption } from './structured-agent-session-turns-options' export { performPrompt } from './structured-agent-session-turns-prompt' @@ -47,8 +43,6 @@ export type AgentSessionTurnContext = { publish: () => void /** Drains provider lifecycle already accepted by the execution host. */ flushStreamedEvents: () => Promise - /** Re-derives authorization after submission persistence, immediately before provider dispatch. */ - beforeDispatch?: () => void /** What the host holds about the child this dispatch is for, read at the moment it is needed. */ providerChildPhase?: () => StructuredAgentSessionProviderChildPhase | undefined now: () => number @@ -67,10 +61,10 @@ function invalid(message: string): { ok: false; refusal: AgentSessionWireRefusal * accepted nothing (input is written only after it initializes), so a dispatch it could not * take is provably unwritten and is rejected with the cause the adapter gave. */ async function dispatchSafely( - ctx: AgentSessionTurnContext, + ctx: AgentSessionHandoverContext, clientMessageId: string, body: AgentJournalMessageItem, - requestedAt: number | undefined + requestedAt: number ): Promise { try { return await ctx.adapter.dispatch({ @@ -78,13 +72,9 @@ async function dispatchSafely( clientMessageId, body, fence: ctx.fence, - ...(ctx.beforeDispatch ? { beforeDispatch: async () => ctx.beforeDispatch?.() } : {}), - ...(requestedAt === undefined ? {} : { requestedAt }) + requestedAt }) } catch (error) { - if (error instanceof AgentSessionPreDispatchError) { - throw error - } if (ctx.providerChildPhase?.() === 'starting') { return { state: 'rejected', reason: providerStartupFailureRejection(error) } } @@ -111,6 +101,8 @@ async function appendStatus( * the whole content of the word — and one message reached the model five times * when this was a judgement call instead of an invariant. A distinct send after * a terminal rejection uses a fresh id, which is a first delivery. + * + * Accepting only records the message; the session's delivery loop hands it over. */ export async function performSend( ctx: AgentSessionTurnContext, @@ -133,79 +125,10 @@ export async function performSend( } } try { - await ctx.journal.appendSubmission({ ...input, fence: ctx.fence }) + await ctx.journal.appendSubmission({ ...input, fence: ctx.fence, handoverRecorded: true }) } catch { return invalid('The message could not be recorded and was not sent.') } - - // The row just written is the send's instant on the host clock; the turn this - // dispatch opens records it so the live counter never re-anchors at turn-open. - const requestedAt = ctx.journal - .submissions() - .find((entry) => entry.clientMessageId === input.clientMessageId)?.submittedAt - const outcome = await dispatchSafely(ctx, input.clientMessageId, input.body, requestedAt).catch( - async (error: unknown) => { - if (error instanceof AgentSessionPreDispatchError) { - const recorded = await withTimeout( - ctx.journal - .resolveDispatch({ - clientMessageId: input.clientMessageId, - state: 'rejected', - reason: error.message, - fence: ctx.fence - }) - .then(() => true), - AGENT_SESSION_ADMISSION_BARRIER_TIMEOUT_MS, - false - ) - if (!recorded) { - console.warn('[structured-agent-session] pre-dispatch refusal persistence failed') - } - } - throw error - } - ) - // An admission needs no dispatch row: the submission is already pending. - if (outcome.state === 'admitted') { - return { - ok: true, - value: { - clientMessageId: input.clientMessageId, - submission: requireSubmission(ctx, input.clientMessageId) - } - } - } - try { - await ctx.journal.resolveDispatch( - outcome.state === 'accepted' - ? { - clientMessageId: input.clientMessageId, - state: 'accepted', - providerIdentity: outcome.providerIdentity, - fence: ctx.fence - } - : { - clientMessageId: input.clientMessageId, - state: outcome.state, - reason: outcome.reason, - fence: ctx.fence - } - ) - } catch (error) { - // A failed resolution must not strand a pending row; an unknown result is - // explicitly replayable. - try { - await ctx.journal.resolveDispatch({ - clientMessageId: input.clientMessageId, - state: 'unknown', - reason: DISPATCH_DOUBT_PERSISTENCE_FAILED, - fence: ctx.fence - }) - } catch { - // Nothing further to record; the pending row is settled on the next attach. - } - throw error - } return { ok: true, value: { @@ -215,6 +138,66 @@ export async function performSend( } } +export type AgentSessionHandoverContext = Pick< + AgentSessionTurnContext, + 'sessionId' | 'journal' | 'fence' | 'adapter' | 'providerChildPhase' +> + +/** + * Hands one queued submission to the provider. The `dispatch{pending}` row goes first: a crash + * after it leaves a message in doubt, never one that reads as queued and so provably unwritten. + */ +export async function handOverSubmission( + ctx: AgentSessionHandoverContext, + submission: AgentJournalSubmission +): Promise { + const { clientMessageId } = submission + const body = ctx.journal.itemBody(agentJournalSubmissionKey(clientMessageId)) + if (body?.kind !== 'message') { + await ctx.journal.resolveDispatch({ + clientMessageId, + state: 'rejected', + reason: 'The message could not be read back and was not sent.', + fence: ctx.fence + }) + return + } + await ctx.journal.resolveDispatch({ clientMessageId, state: 'pending', fence: ctx.fence }) + // The row written at acceptance is the send's instant on the host clock; the turn this + // dispatch opens records it so the live counter never re-anchors at turn-open. + const outcome = await dispatchSafely(ctx, clientMessageId, body, submission.submittedAt) + // An admission needs no dispatch row: the submission is already pending. + if (outcome.state === 'admitted') { + return + } + try { + await ctx.journal.resolveDispatch( + outcome.state === 'accepted' + ? { + clientMessageId, + state: 'accepted', + providerIdentity: outcome.providerIdentity, + fence: ctx.fence + } + : { clientMessageId, state: outcome.state, reason: outcome.reason, fence: ctx.fence } + ) + } catch (error) { + // A failed resolution must not strand a pending row; an unknown result is + // explicitly replayable. + try { + await ctx.journal.resolveDispatch({ + clientMessageId, + state: 'unknown', + reason: DISPATCH_DOUBT_PERSISTENCE_FAILED, + fence: ctx.fence + }) + } catch { + // Nothing further to record; the pending row is settled on the next open. + } + throw error + } +} + function requireSubmission( ctx: AgentSessionTurnContext, clientMessageId: string diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unanswered-dispatch-release.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unanswered-dispatch-release.test.ts index 79f21a349b4..952fa70657d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unanswered-dispatch-release.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unanswered-dispatch-release.test.ts @@ -32,10 +32,12 @@ function contextWith(submissions: AgentJournalSubmission[]) { return { epoch: 'e', sequence: 1 } }) } - // The mutation reads only `journal.submissions`, `journal.resolveDispatch` and `fence`. - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: all three are supplied here; the rest of the host session is unreachable from this mutation. - const session = { journal, fence: FENCE } as unknown as ReleaseSession - const context: ReleaseContext = { sessions: new Map([['s-1', session]]) } + // The mutation reads only `journal.submissions`, `journal.resolveDispatch` and the record fence. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: all three are supplied here; the rest of the session and the record is unreachable from this mutation. + const context = { + sessions: new Map([['s-1', { journal }]]), + deps: { store: { getRecord: () => ({ lease: { runtimeFence: FENCE } }) } } + } as unknown as ReleaseContext return { context, resolved, journal } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts index 98901380aa3..6419e1cfb1a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts @@ -34,11 +34,18 @@ function recoveryContext(input: { handoffStage?: AgentSessionRecord['lease']['handoffStage'] resumeCapable?: boolean }) { - const session = { - hasProviderChild: false, - fence: 8, - acquisitionGeneration: input.generation ?? GENERATION - } as StructuredAgentSessionHostSession + const session: Pick = { + child: null, + lastEndedChild: { + generation: input.generation ?? GENERATION, + fence: 7, + cause: 'exit', + reason: null, + duringStartup: false, + rootGone: true, + endedAt: { epoch: 'epoch-1', sequence: 0 } + } + } const record = { lease: { runtimeFence: 8, @@ -121,11 +128,11 @@ describe('provider-exit recovery tickets', () => { .fn() .mockRejectedValueOnce(new Error('journal unavailable')) .mockResolvedValue({ epoch: 'epoch-1', sequence: 2 }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the exit reads only the child record and these journal methods; the rest of the session is unreachable from it. const session = { - hasProviderChild: true, - fence: 7, - acquisitionGeneration: GENERATION, + child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { + cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), snapshot: () => ({ items: [lifecycleItem('turn-1', 1, { state: 'running', startedAt: 1_000 })] }), @@ -193,11 +200,11 @@ describe('provider-exit recovery tickets', () => { lifecycleItem('turn-1', 1, { state: 'completed', startedAt: 10, completedAt: 20 }), lifecycleItem('turn-2', 2, { state: 'running', startedAt: 30 }) ] + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the exit reads only the child record and these journal methods; the rest of the session is unreachable from it. const session = { - hasProviderChild: true, - fence: 7, - acquisitionGeneration: GENERATION, + child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { + cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), snapshot: () => ({ items }), appendLifecycleBatch, markPendingSubmissionsUnknown: vi.fn(async () => []) @@ -242,7 +249,7 @@ describe('provider-exit recovery tickets', () => { 7, 'provider_exited_before_acknowledgement' ) - expect(session.hasProviderChild).toBe(false) + expect(session.child).toBeNull() // The running row is revised to interrupted at exit receipt, never tombstoned. expect(appendLifecycleBatch).toHaveBeenCalledExactlyOnceWith({ settlementId: `dead-generation:provider-exit:${SESSION}:7:${GENERATION}`, @@ -299,10 +306,9 @@ describe('provider-exit recovery tickets', () => { sequence: 3 })) const session: StructuredAgentSessionUnexpectedExitSession = { - hasProviderChild: true, - fence: 7, - acquisitionGeneration: GENERATION, + child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { + cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), snapshot: () => ({ items }), appendLifecycleBatch, markPendingSubmissionsUnknown: vi.fn(async () => []), @@ -356,10 +362,9 @@ describe('provider-exit recovery tickets', () => { it('settles a submission the dead child never acknowledged', async () => { const markPendingSubmissionsUnknown = vi.fn(async () => ['client-1']) const session: StructuredAgentSessionUnexpectedExitSession = { - hasProviderChild: true, - fence: 7, - acquisitionGeneration: GENERATION, + child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { + cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), snapshot: () => ({ items: [] }), appendLifecycleBatch: vi.fn(async () => ({ epoch: 'epoch-1', sequence: 1 })), markPendingSubmissionsUnknown, @@ -404,10 +409,9 @@ describe('provider-exit recovery tickets', () => { it('releases without offering a restart while terminal settlement is failing', async () => { const session: StructuredAgentSessionUnexpectedExitSession = { - hasProviderChild: true, - fence: 7, - acquisitionGeneration: GENERATION, + child: { generation: GENERATION, fence: 7, phase: 'ready' }, journal: { + cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), markPendingSubmissionsUnknown: vi.fn(async () => []), rejectPendingSubmissions: vi.fn(async () => []), snapshot: () => ({ @@ -442,8 +446,7 @@ describe('provider-exit recovery tickets', () => { const result = await settleUnexpectedStructuredAgentSessionExit(context, event) expect(result).toBeNull() - expect(session.hasProviderChild).toBe(false) - expect(session.fence).toBe(8) + expect(session.child).toBeNull() expect(publishFence).toHaveBeenCalledTimes(1) expect(release).toHaveBeenCalledTimes(2) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts index f21de56f2ba..541daec0153 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.ts @@ -1,8 +1,10 @@ -import type { - StructuredAgentSessionEndedEvent, - StructuredAgentSessionProviderChildPhase -} from './structured-agent-session-adapter' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { StructuredAgentSessionEndedEvent } from './structured-agent-session-adapter' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { + endProviderChild, + failedProviderChildStart +} from './structured-agent-session-provider-child' import { releaseStoredStructuredAgentSessionOwnerAfterUnexpectedExit, type StructuredAgentSessionLeaseStore @@ -28,13 +30,10 @@ export type StructuredAgentSessionRecoveryTicket = { stableSettlementId: string } -export type StructuredAgentSessionUnexpectedExitSession = { - journal: DeadGenerationJournal - hasProviderChild: boolean - fence: number - acquisitionGeneration: string | null - providerChildPhase?: StructuredAgentSessionProviderChildPhase -} +export type StructuredAgentSessionUnexpectedExitSession = Pick< + StructuredAgentSessionHostSession, + 'child' | 'lastEndedChild' +> & { journal: DeadGenerationJournal & Pick } export type StructuredAgentSessionUnexpectedExitContext< TSession extends StructuredAgentSessionUnexpectedExitSession = StructuredAgentSessionHostSession @@ -64,24 +63,37 @@ export async function settleUnexpectedStructuredAgentSessionExit< const observedAt = event.observedAt ?? context.now() return context.serialize(unexpectedEvent.sessionId, async () => { const session = context.sessions.get(unexpectedEvent.sessionId) + const child = session?.child if ( - !session?.hasProviderChild || - session.fence !== unexpectedEvent.fence || - session.acquisitionGeneration !== unexpectedEvent.acquisitionGeneration + !session || + !child || + child.fence !== unexpectedEvent.fence || + child.generation !== unexpectedEvent.acquisitionGeneration ) { return null } - const record = context.store.getRecord(unexpectedEvent.sessionId) - if (!record || record.lease.handoffStage !== null) { - // An acquisition or recovery already owns this lease's transition. - session.hasProviderChild = false - context.publishStatus?.(unexpectedEvent.sessionId) - return null - } // The host's own phase decides, so a provider that omits the flag still gets a start that // failed told as one: the row says so, and nothing resumes into the same failure. const exitedDuringStartup = - unexpectedEvent.startupUnproven === true || session.providerChildPhase === 'starting' + unexpectedEvent.startupUnproven === true || child.phase === 'starting' + const endChild = (): void => { + endProviderChild(session, { + generation: child.generation, + fence: child.fence, + cause: 'exit', + reason: unexpectedEvent.reason, + duringStartup: exitedDuringStartup, + // The adapter publishes an exit only once it saw the root go, first-hand or proven. + rootGone: true + }) + context.publishStatus?.(unexpectedEvent.sessionId) + } + const record = context.store.getRecord(unexpectedEvent.sessionId) + if (!record || record.lease.handoffStage !== null) { + // An acquisition or recovery already owns this lease's transition. + endChild() + return null + } let settlementFailed = false const stableSettlementId = providerExitSettlementId(unexpectedEvent) @@ -101,7 +113,8 @@ export async function settleUnexpectedStructuredAgentSessionExit< settlementFailed = !(await retryUnexpectedExitSettlement({ context, event: unexpectedEvent, - session, + journal: session.journal, + fence: child.fence, stableSettlementId, verdict: { state: 'interrupted', completedAt: observedAt }, exitedDuringStartup, @@ -123,7 +136,7 @@ export async function settleUnexpectedStructuredAgentSessionExit< sessionId: unexpectedEvent.sessionId, expectedFence: unexpectedEvent.fence, expectedAcquisitionGeneration: unexpectedEvent.acquisitionGeneration, - acquisitionGeneration: session.acquisitionGeneration, + acquisitionGeneration: child.generation, now: context.now(), exitObservedAt: observedAt, // Bare cause: whatever this settlement could not write is settled from it later, by the @@ -133,10 +146,8 @@ export async function settleUnexpectedStructuredAgentSessionExit< } catch (error) { context.onBarrierError?.(unexpectedEvent.sessionId, error) } finally { - session.hasProviderChild = false - context.publishStatus?.(unexpectedEvent.sessionId) + endChild() if (released) { - session.fence = released.lease.runtimeFence context.publishFence(unexpectedEvent.sessionId, session) } } @@ -144,8 +155,10 @@ export async function settleUnexpectedStructuredAgentSessionExit< if (settlementFailed || !released) { return null } - // Resuming a start that failed would respawn into the same failure; the next send retries. - if (exitedDuringStartup || !context.hasResumeCapableHolder(unexpectedEvent.sessionId)) { + if ( + failedProviderChildStart(session) || + !context.hasResumeCapableHolder(unexpectedEvent.sessionId) + ) { return null } return { @@ -162,10 +175,7 @@ export function isStructuredAgentSessionRecoveryTicketCurrent( store: Pick sessions: Map< string, - Pick< - StructuredAgentSessionUnexpectedExitSession, - 'hasProviderChild' | 'fence' | 'acquisitionGeneration' - > + Pick > hasResumeCapableHolder: (sessionId: string) => boolean }, @@ -174,9 +184,9 @@ export function isStructuredAgentSessionRecoveryTicketCurrent( const session = context.sessions.get(ticket.sessionId) const record = context.store.getRecord(ticket.sessionId) return ( - session?.hasProviderChild === false && - session.fence === ticket.releasedFence && - session.acquisitionGeneration === ticket.deadAcquisitionGeneration && + session !== undefined && + session.child === null && + session.lastEndedChild?.generation === ticket.deadAcquisitionGeneration && record?.lease.runtimeFence === ticket.releasedFence && record.lease.claimStatus === 'released' && record.lease.handoffStage === null && @@ -187,22 +197,25 @@ export function isStructuredAgentSessionRecoveryTicketCurrent( async function retryUnexpectedExitSettlement(input: { context: Pick event: UnexpectedExitLifecycleEvent - session: Pick + journal: DeadGenerationJournal + fence: number stableSettlementId: string verdict: StructuredAgentSessionTurnVerdict exitedDuringStartup: boolean showUnexpectedExitOutcome?: boolean }): Promise { return settleStructuredAgentSessionDeadGeneration({ - journal: input.session.journal, + journal: input.journal, sessionId: input.event.sessionId, - fence: input.session.fence, + fence: input.fence, settlementId: input.stableSettlementId, verdict: input.verdict, pendingSubmissionReason: 'provider_exited_before_acknowledgement', showUnexpectedExitOutcome: input.showUnexpectedExitOutcome, unexpectedExitReason: input.event.reason, - exitedDuringStartup: input.exitedDuringStartup, + ...(input.exitedDuringStartup + ? { exitedDuringStartup: { generation: input.event.acquisitionGeneration } } + : {}), onError: input.context.onBarrierError }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-view-start-after-failed-start.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-view-start-after-failed-start.test.ts new file mode 100644 index 00000000000..b8e2d73c27f --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-view-start-after-failed-start.test.ts @@ -0,0 +1,207 @@ +// A Claude chat whose CLI exits before it finishes starting leaves one red row per start. A view +// opening, or coming back to, a chat whose last start failed used to start the CLI again, so every +// look at the chat added an identical row. Only a send retries a failed start: it is the user asking. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import { ClaudeStructuredSessionAdapter } from '../../claude/claude-structured-session-adapter' +import { + fakeClaude, + PROVIDER_SESSION_ID +} from '../../claude/claude-structured-session-test-support' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { structuredClaudeLifecycleEvent } from '../../runtime/structured-claude-runtime-adapter' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */ +function eventually(assertion: () => unknown): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} + +const CALLER = { callerKey: 'client-1' } +const SURFACE = 'desktop-chat:1' +const LAUNCH_FAILURE = + 'claude stream-json exited (code 1): qa-shim: simulated claude launch failure' + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost +let adapter: ClaudeStructuredSessionAdapter +let claude: ReturnType +let lifecycle: Promise[] +/** Every initialize waits on it: a start that must outlast a step does not race a timer. */ +let initGate: Promise + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-view-start-after-failed-start-')) + resetHostTestOperationIds() + lifecycle = [] + initGate = Promise.resolve() + // Every start spawns, is published, and exits before it answers initialize. + claude = fakeClaude({ initDelayMs: 20, exitBeforeInit: LAUNCH_FAILURE }) + adapter = new ClaudeStructuredSessionAdapter({ + resolveLaunch: async () => ({ + pathToClaudeCodeExecutable: 'claude', + options: {}, + cwd: root, + claudeConfigDir: join(root, 'claude-home'), + providerSessionId: PROVIDER_SESSION_ID, + resumeLeafUuid: null, + resumesTranscript: (store.getRecord(SESSION)?.providerHandleChain.length ?? 0) > 0, + continuesChain: (store.getRecord(SESSION)?.providerHandleChain.length ?? 0) > 0 + }), + onEvent: (event) => { + const mapped = structuredClaudeLifecycleEvent(event) + if (mapped) { + lifecycle.push(host.handleAdapterEvent(mapped)) + } + }, + openConnection: async (launch, handlers) => { + const connection = await claude.openConnection(launch, handlers) + const initialize = connection.initializationResult + return Object.assign(connection, { + initializationResult: async () => { + await initGate + return initialize() + } + }) + }, + readProcessStartTime: async () => 1_700_000_000_000, + now: () => NOW + }) + store = await AgentSessionRecordStore.open({ directory: join(root, 'store'), hostId: 'local' }) + host = new StructuredAgentSessionHost({ + store, + adapter: Object.assign(adapter, { supportsCreate: () => true }), + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => `spawn-${claude.connections.length + 1}`, + now: () => NOW + }) +}) + +afterEach(async () => { + await adapter.closeAll() + await host.flushAllStreamedEvents() + await rm(root, { recursive: true, force: true }) +}) + +/** Waits until the adapter has published every exit it saw and the host settled each one. */ +async function settleExits(): Promise { + await eventually(async () => { + await adapter.drainObservedExits() + await Promise.all(lifecycle) + expect(host.journalSnapshot(SESSION).items.length).toBeGreaterThan(0) + }) + await Promise.all(lifecycle) +} + +/** The chat as it renders: the user's messages and the error rows, in journal order. */ +function timeline(): string[] { + return host + .journalSnapshot(SESSION) + .items.flatMap((item) => + item.body.kind === 'message' + ? ['message'] + : item.body.kind === 'status' && item.body.tone === 'error' + ? [item.body.text] + : [] + ) +} + +async function send(text: string): Promise { + const body = hostTestMessage(text) + const sent = await host.send(CALLER, { + envelope: { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: store.getRecord(SESSION)?.lease.runtimeFence ?? 1, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields: { body } + }) + }, + body + }) + expect(sent).toMatchObject({ ok: true }) + return sent.ok ? sent.value.clientMessageId : '' +} + +describe('a fresh chat whose Claude start fails', () => { + // QA saw three failed starts from opening the chat alone: the create's, and the view's. + it.each([ + ['after the create already died', false], + ['while the create is still starting', true] + ] as const)( + 'starts once for the open and once for a send, one row each, when the view binds %s', + async (_when, createStillStarting) => { + // Released only once the views bound, so no runner is slow enough to let the create die first. + let releaseCreate = (): void => {} + const createGate = new Promise((resolve) => { + releaseCreate = resolve + }) + if (!createStillStarting) { + releaseCreate() + } + initGate = createGate + claude = fakeClaude({ exitBeforeInit: LAUNCH_FAILURE }) + await expect( + host.attach( + CALLER, + hostTestAttachParams(null, { + provider: 'claude', + agent: 'claude', + accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: join(root, 'claude-home') }, + providerHandle: { kind: 'claude', sessionId: PROVIDER_SESSION_ID, leafUuid: null } + }) + ) + ).resolves.toMatchObject({ ok: true }) + if (!createStillStarting) { + await settleExits() + } + // Two surfaces bind, as a pane and a second window do; exit recovery sees a holder. + await host.hold(SESSION, SURFACE) + await host.hold(SESSION, 'desktop-chat:2') + if (createStillStarting) { + // The views bound to the create's child itself, before it exited. + expect(timeline()).toEqual([]) + releaseCreate() + } + await settleExits() + const startFailure = `The provider stopped before it finished starting: ${LAUNCH_FAILURE}.` + // Opening the chat: the create's start, once, and its row. + expect(claude.connections).toHaveLength(1) + expect(timeline()).toEqual([startFailure]) + + const sent = await send('reply with exactly: alpha') + await eventually(() => + expect( + host.journalSnapshot(SESSION).submissions.find((s) => s.clientMessageId === sent) + ).toMatchObject({ dispatchState: 'rejected', reason: startFailure }) + ) + await settleExits() + // The send's own start, once, and one row for it below the message. + expect(claude.connections).toHaveLength(2) + expect(timeline()).toEqual([startFailure, 'message', startFailure]) + + // Switching away and back re-takes the view's hold; it starts nothing and adds no row. + host.release(SESSION, SURFACE) + await host.hold(SESSION, SURFACE) + await settleExits() + expect(claude.connections).toHaveLength(2) + expect(timeline()).toEqual([startFailure, 'message', startFailure]) + } + ) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts index 0778d271129..66e3cd2cca2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts @@ -398,7 +398,7 @@ describe('already-wedged profiles become usable on load', () => { ['a restart eviction', false], ['a proven eviction by recovery', true] ] as const)( - 'settles the turn %s left even when the handle it was restored with never writes', + 'settles the turn %s left at the next acquire when the read restore could not write it', async (_origin, ownerOutlivedRestart) => { await seedStore( wedgedRecord({ claimStatus: 'live', handoffStage: null, ownerProcess: DEAD_OWNER }) @@ -415,15 +415,13 @@ describe('already-wedged profiles become usable on load', () => { : { outcome: 'pid-absent' }, stopOwnerProcess }) - // The read restore's settlement fails, and the handle it restored with never writes again. + // The read restore's settlement fails, and nothing retries it. const failing = vi .spyOn(AgentSessionJournal.prototype, 'appendLifecycleBatch') .mockRejectedValue(new Error('journal unavailable')) await host.restoreReadableSessions() failing.mockRestore() - vi.spyOn(restoredJournal(), 'appendLifecycleBatch').mockRejectedValue( - new Error('journal unavailable') - ) + expect(activeStructuredAgentSessionTurnId(restoredJournal().snapshot().items)).toBe('turn-1') expect(stopOwnerProcess).toHaveBeenCalledTimes(ownerOutlivedRestart ? 1 : 0) expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'released', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts index 8eac02dc17b..e68df2c09a6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts @@ -17,6 +17,7 @@ import { agentSessionProviderHandleRoot } from '../../../shared/agent-session-provider-handle' import { latestStructuredAgentSessionUserItem } from '../../../shared/structured-agent-session-projection' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionResumeMarker, @@ -135,8 +136,7 @@ function liveTasks( type WorkingCandidateSession = { journal: AgentSessionJournal /** Only this host generation's own child counts. A restored-for-reading journal has none. */ - hasProviderChild: boolean - fence?: number + child: { fence: number } | null } /** The offer one session is owed, taken right before teardown stops its provider child; null when @@ -154,16 +154,24 @@ export function structuredAgentSessionWorkingAtStop(input: { }): AgentSessionResumeMarker | null { const { sessionId, session } = input // A journal this host cannot read tells us nothing about what the turn was doing. - if (!session?.hasProviderChild || session.journal.isReadOnly) { + if (!session?.child || session.journal.isReadOnly) { return null } const snapshot = session.journal.snapshot() + // A queued message reached no agent, so it is no work to resume: quit rejects it as never sent. + const handedOver = snapshot.submissions.filter( + (submission) => !isQueuedAgentJournalSubmission(submission) + ) const roster = input.backgroundTasks(sessionId) - const status = structuredAgentSessionShownStatus(snapshot, roster, session.fence) + const status = structuredAgentSessionShownStatus( + { items: snapshot.items, submissions: handedOver }, + roster, + session.child.fence + ) if (status.state === 'done') { return null } - const work = structuredAgentSessionResumeWork(snapshot.items, snapshot.submissions) + const work = structuredAgentSessionResumeWork(snapshot.items, handedOver) const head = agentSessionProviderHandleChainHead( input.getRecord(sessionId)?.providerHandleChain ?? [] ) diff --git a/src/main/native-chat/agent-session-wire/structured-compaction-recovery.ts b/src/main/native-chat/agent-session-wire/structured-compaction-recovery.ts index 2199da05392..64f1ee652e7 100644 --- a/src/main/native-chat/agent-session-wire/structured-compaction-recovery.ts +++ b/src/main/native-chat/agent-session-wire/structured-compaction-recovery.ts @@ -9,12 +9,21 @@ export async function recoverInterruptedCompaction( fence: number ): Promise { const command = store.getRecord(sessionId)?.conversationCommand - if ( - command?.command !== 'compact' || - command.phase !== 'prepared' || - command.runtimeFence === undefined || - command.runtimeFence === fence - ) { + if (command?.runtimeFence === undefined || command.runtimeFence === fence) { + return + } + await settleInterruptedCompaction(store, sessionId, journal, fence) +} + +/** A compaction still prepared whose child can no longer finish it: its outcome is unknown. */ +export async function settleInterruptedCompaction( + store: AgentSessionRecordStore, + sessionId: string, + journal: AgentSessionJournal, + fence: number +): Promise { + const command = store.getRecord(sessionId)?.conversationCommand + if (command?.command !== 'compact' || command.phase !== 'prepared') { return } const error = 'Previous compaction completion could not be confirmed after session recovery.' diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts b/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts index 78b9f4a59d0..fe6e5501975 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts @@ -48,9 +48,12 @@ export function conversationCommandBlocked( : 'Wait for background tasks to finish before using this command.' } if ( - ctx.journal - .submissions() - .some((entry) => entry.dispatchState === 'pending' || entry.dispatchState === 'unknown') + ctx.journal.submissions().some( + (entry) => + entry.dispatchState === 'pending' || + // Doubt left by an earlier child is not this one's work in flight. + (entry.dispatchState === 'unknown' && entry.recovered !== true && entry.fence === ctx.fence) + ) ) { return 'Resolve pending or unconfirmed messages before using this command.' } diff --git a/src/main/runtime/agent-session-operation-admission.ts b/src/main/runtime/agent-session-operation-admission.ts index e7a4165120e..f16d48efb9d 100644 --- a/src/main/runtime/agent-session-operation-admission.ts +++ b/src/main/runtime/agent-session-operation-admission.ts @@ -36,6 +36,7 @@ export type AgentSessionMutationOperationAdmission = { hostFingerprint: string now: number operationIdScope?: 'global' + conversationWrite?: true } export type AgentSessionMutationOperationDecision = { @@ -149,7 +150,8 @@ export function admitAgentSessionMutationOperation( envelope: args.envelope, hostFingerprint: args.hostFingerprint, ledger: ledger.decision, - lease: record.lease + lease: record.lease, + ...(args.conversationWrite ? { conversationWrite: true } : {}) }) if (ledger.decision.decision === 'admit' && admission.decision === 'refused') { ledger.rows.delete(agentSessionOperationKey(operation.callerKey, operation.operationId)) diff --git a/src/main/runtime/claude-structured-send-held-for-startup.test.ts b/src/main/runtime/claude-structured-send-held-for-startup.test.ts index 19e05811055..da1135f588f 100644 --- a/src/main/runtime/claude-structured-send-held-for-startup.test.ts +++ b/src/main/runtime/claude-structured-send-held-for-startup.test.ts @@ -1,8 +1,8 @@ // A Claude chat is published the moment its child spawns, before the CLI has answered initialize. -// A send in that window — into a fresh start, or into the restart a send itself asked for after -// a start that failed — is admitted and held until the child proves its start. When the CLI dies -// first, the held message is rejected with the CLI's own diagnostic, the chat shows the cause -// once, and nothing is left as a delivery nobody can confirm. Against the production runtime, +// A send in that window — into a fresh start, or into the restart the delivery loop makes for a +// send after a start that failed — is accepted and stays queued until the child proves its start. +// When the CLI dies first, the queued message is rejected with the CLI's own diagnostic, the chat +// shows the cause once, and nothing is left as a delivery nobody can confirm. Against the production runtime, // adapter, record store and host, with only the CLI process scripted. import { afterEach, describe, expect, it, vi } from 'vitest' @@ -39,7 +39,7 @@ async function send(host: StructuredAgentSessionHost, text: string): Promise { expect(statusRows(host)).toEqual([expect.stringContaining('not signed in')]) const releasedFence = fence(host) - // The send asks for the child back and is held for its start; the CLI dies again first. + // The delivery loop asks for the child back and the message waits for its start; the CLI + // dies again first. const held = await send(host, 'hello?') - expect(claude.children(SESSION)).toHaveLength(2) - expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + await vi.waitFor(() => expect(claude.children(SESSION)).toHaveLength(2)) + await vi.waitFor(() => + expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + ) await failLatestStart(host, 2) // Rejected with the cause, not left in doubt; one row for this attempt names it. @@ -111,8 +114,8 @@ describe('a send into a Claude chat whose CLI keeps failing at startup', () => { // The user signs in and retries: one restart, proven, written to the CLI. claude.behave(SESSION, {}) await send(host, 'hello again') - expect(claude.children(SESSION)).toHaveLength(3) - expect(fence(host)).toBe(releasedFence + 3) + await vi.waitFor(() => expect(claude.children(SESSION)).toHaveLength(3)) + await vi.waitFor(() => expect(fence(host)).toBe(releasedFence + 3)) await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send')) await vi.waitFor(() => expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('live') @@ -122,7 +125,7 @@ describe('a send into a Claude chat whose CLI keeps failing at startup', () => { }) describe('a send while the first Claude start is still answering initialize', () => { - it('is held, and written once the CLI proves its start', async () => { + it('is queued, and written once the CLI proves its start', async () => { claude.behave(SESSION, { initHangs: true }) const host = await claude.install() await host.attach(CALLER, claude.attachParams(SESSION, null)) @@ -130,7 +133,7 @@ describe('a send while the first Claude start is still answering initialize', () await send(host, 'hello') expect(claude.child(SESSION).calls).not.toContain('send') - // The CLI answers: startup lands and the held message is written to the proven child. + // The CLI answers: startup lands and the queued message is written to the proven child. claude.child(SESSION).answerInit() await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send')) diff --git a/src/main/runtime/claude-structured-send-restart-dies-before-dispatch.test.ts b/src/main/runtime/claude-structured-send-restart-dies-before-dispatch.test.ts index 0d4b4909637..817b00144bc 100644 --- a/src/main/runtime/claude-structured-send-restart-dies-before-dispatch.test.ts +++ b/src/main/runtime/claude-structured-send-restart-dies-before-dispatch.test.ts @@ -1,10 +1,9 @@ -// A send restarts a chat's Claude child and is admitted against it while it is still starting. -// When that child dies before the send's dispatch reaches the adapter, the adapter has no session -// to hold the message for, so the dispatch throws. A child that never proved its start accepted -// nothing — input is only written after initialize — so the send settles `rejected` with the -// child's own diagnostic, never as a delivery nobody can confirm, and a client that was -// subscribed the whole time receives the failure row and the rejected submission over the wire. -// Against the production runtime, adapter, record store and host, with only the CLI scripted. +// A send is accepted into a chat whose Claude child is gone, and its delivery restarts the child. +// When that child dies before it proves its start, the message was never handed to it — delivery +// waits for the start — so the send settles `rejected` with the child's own diagnostic, never as a +// delivery nobody can confirm, and a client that was subscribed the whole time receives the +// failure row and the rejected submission over the wire. Against the production runtime, adapter, +// record store and host, with only the CLI scripted. import { afterEach, describe, expect, it, vi } from 'vitest' import { computeAgentSessionPayloadFingerprint } from '../../shared/agent-session-mutation-envelope' @@ -18,6 +17,11 @@ const SESSION = 'claude-send-restart-dies-first' const CALLER = { callerKey: 'client-1' } const DIAGNOSTIC = 'claude stream-json exited (code 1): claude: not signed in (rig)' +/** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */ +function eventually(assertion: () => void): Promise { + return vi.waitFor(assertion, { timeout: 10_000 }) +} + let claude = createScriptedClaudeRuntime([SESSION]) let operations = 0 /** The dispatch state each send was answered with, before any exit settled it. */ @@ -86,24 +90,14 @@ function submission(host: StructuredAgentSessionHost, clientMessageId: string) { } async function failLatestStart(host: StructuredAgentSessionHost, count: number): Promise { - await vi.waitFor(() => expect(claude.children(SESSION)).toHaveLength(count)) + await eventually(() => expect(claude.children(SESSION)).toHaveLength(count)) claude.child(SESSION).exit(new Error(DIAGNOSTIC)) await waitForStructuredAgentSessionRecovery() - await vi.waitFor(() => + await eventually(() => expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('released') ) } -/** The restarted child dies the instant the send's dispatch reaches the adapter. */ -function killChildAtDispatch(host: StructuredAgentSessionHost): void { - const adapter = host.deps.adapter - const dispatch = adapter.dispatch.bind(adapter) - vi.spyOn(adapter, 'dispatch').mockImplementationOnce((input) => { - claude.child(SESSION).exit(new Error(DIAGNOSTIC)) - return dispatch(input) - }) -} - /** Everything a subscriber received, flattened to the rows and submissions it was shown. */ function received(events: AgentSessionSubscribeEvent[]) { const statusTexts: string[] = [] @@ -129,7 +123,7 @@ function received(events: AgentSessionSubscribeEvent[]) { return { statusTexts, submissions, fences } } -describe('a send whose restarted Claude child dies before the dispatch reaches the adapter', () => { +describe('a send whose restarted Claude child dies before it proves its start', () => { it('settles rejected with the diagnostic, keeps one failure row, and a Retry is one new attempt', async () => { claude.behave(SESSION, { initHangs: true }) const host = await claude.install() @@ -139,18 +133,13 @@ describe('a send whose restarted Claude child dies before the dispatch reaches t await failLatestStart(host, 1) const releasedFence = fence(host) - killChildAtDispatch(host) const sent = await send(host, 'hello?') - // The send's own answer already says it was not delivered; it does not wait for the exit. - expect(answered.get(sent)).toBe('rejected') - expect(claude.children(SESSION)).toHaveLength(2) - await waitForStructuredAgentSessionRecovery() - await vi.waitFor(() => - expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('released') - ) + // Accepted: the answer comes before the restart it needs. + expect(answered.get(sent)).toBe('pending') + await failLatestStart(host, 2) // Provably not delivered, with the cause; not "unconfirmed". - await vi.waitFor(() => + await eventually(() => expect(submission(host, sent)).toMatchObject({ dispatchState: 'rejected', // Worded for the user: the red line under the composer shows it as it stands. @@ -168,40 +157,14 @@ describe('a send whose restarted Claude child dies before the dispatch reaches t // Retry under a new id: one restart, and once the CLI is healthy the message is written. claude.behave(SESSION, {}) await send(host, 'hello again') - expect(claude.children(SESSION)).toHaveLength(3) - await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send')) + await eventually(() => expect(claude.children(SESSION)).toHaveLength(3)) + await eventually(() => expect(claude.child(SESSION).calls).toContain('send')) expect(claude.child(SESSION).calls.filter((call) => call === 'send')).toHaveLength(1) expect(statusRows(host)).toHaveLength(2) }) - it('names the diagnostic even when the exit was fully processed before the dispatch arrived', async () => { - claude.behave(SESSION, { initHangs: true }) - const host = await claude.install() - await expect(host.attach(CALLER, claude.attachParams(SESSION, null))).resolves.toMatchObject({ - ok: true - }) - await failLatestStart(host, 1) - const adapter = host.deps.adapter - const dispatch = adapter.dispatch.bind(adapter) - vi.spyOn(adapter, 'dispatch').mockImplementationOnce(async (input) => { - claude.child(SESSION).exit(new Error(DIAGNOSTIC)) - // The adapter settles and publishes the exit; the host's own settlement waits behind this send. - await new Promise((resolve) => setTimeout(resolve, 300)) - return dispatch(input) - }) - - const sent = await send(host, 'hello?') - expect(answered.get(sent)).toBe('rejected') - await waitForStructuredAgentSessionRecovery() - expect(submission(host, sent)).toMatchObject({ - dispatchState: 'rejected', - reason: `The provider stopped before it finished starting: ${DIAGNOSTIC}.` - }) - expect(statusRows(host)).toHaveLength(2) - }) - // A restart refused because its child died before it was handed over leaves one row, from the - // send, in the words any failed start uses. + // delivery, in the words any failed start uses, and rejects the message with them. it.each(['spawn', 'start-time-read'] as const)( 'leaves one row for a restart whose child exits at %s', async (at) => { @@ -213,10 +176,13 @@ describe('a send whose restarted Claude child dies before the dispatch reaches t await failLatestStart(host, 1) claude.behave(SESSION, { exitsDuringSpawn: { diagnostic: DIAGNOSTIC, at } }) - await expect(attempt(host, 'hello?')).resolves.toMatchObject({ - ok: false, - refusal: { code: 'agent_session_owner_restart_failed' } - }) + const sent = await send(host, 'hello?') + await eventually(() => + expect(submission(host, sent)).toMatchObject({ + dispatchState: 'rejected', + reason: `The provider stopped before it finished starting: ${DIAGNOSTIC}.` + }) + ) await waitForStructuredAgentSessionRecovery() expect(statusRows(host)).toEqual([ @@ -242,14 +208,10 @@ describe('a send whose restarted Claude child dies before the dispatch reaches t }) try { await failLatestStart(host, 1) - killChildAtDispatch(host) const sent = await send(host, 'hello?') - await waitForStructuredAgentSessionRecovery() - await vi.waitFor(() => - expect(host.deps.store.getRecord(SESSION)?.lease.claimStatus).toBe('released') - ) + await failLatestStart(host, 2) - await vi.waitFor(() => { + await eventually(() => { const seen = received(events) expect(seen.submissions.get(sent)).toBe('rejected') expect(seen.statusTexts).toContainEqual( @@ -285,7 +247,7 @@ describe('a chat whose Claude CLI keeps failing to start, seen by a subscriber o return rows } - it('shows one row naming the cause per failed attempt, admitted or refused, and none once the CLI is fixed', async () => { + it('shows one row naming the cause per failed attempt, however the start died, and none once the CLI is fixed', async () => { claude.behave(SESSION, { initHangs: true }) const host = await claude.install() const events: AgentSessionSubscribeEvent[] = [] @@ -301,37 +263,38 @@ describe('a chat whose Claude CLI keeps failing to start, seen by a subscriber o }) try { await failLatestStart(host, 1) - await vi.waitFor(() => expect([...shownRows(events).values()]).toEqual([STARTUP_FAILURE])) + await eventually(() => expect([...shownRows(events).values()]).toEqual([STARTUP_FAILURE])) - // Send: admitted against the restarted child, which dies before starting. - killChildAtDispatch(host) - const sent = await attempt(host, 'hello?') - await waitForStructuredAgentSessionRecovery() - expect(sent).toMatchObject({ - ok: true, - value: { submission: { dispatchState: 'rejected', reason: STARTUP_FAILURE } } - }) - await vi.waitFor(() => + // Send: accepted, and its delivery restarts the child, which dies before starting. + const sent = await send(host, 'hello?') + await failLatestStart(host, 2) + await eventually(() => + expect(submission(host, sent)).toMatchObject({ + dispatchState: 'rejected', + reason: STARTUP_FAILURE + }) + ) + await eventually(() => expect([...shownRows(events).values()]).toEqual([STARTUP_FAILURE, STARTUP_FAILURE]) ) - // Retry while still broken: this restart dies before its child is handed over, so the send - // is refused before admission. Still one row, saying the same thing. + // Retry while still broken: this restart dies before its child is handed over, so the + // delivery's start is refused. Still one row, saying the same thing, on the rejected message. claude.behave(SESSION, { exitsDuringSpawn: { diagnostic: 'claude stream-json exited (code 1): claude: not signed in (rig)', at: 'start-time-read' } }) - await expect(attempt(host, 'hello?')).resolves.toMatchObject({ - ok: false, - refusal: { - code: 'agent_session_owner_restart_failed', - message: expect.stringMatching(/couldn't restart: .*not signed in \(rig\)/) - } - }) + const retried = await send(host, 'hello?') + await eventually(() => + expect(submission(host, retried)).toMatchObject({ + dispatchState: 'rejected', + reason: STARTUP_FAILURE + }) + ) await waitForStructuredAgentSessionRecovery() - await vi.waitFor(() => + await eventually(() => expect([...shownRows(events).values()]).toEqual([ STARTUP_FAILURE, STARTUP_FAILURE, @@ -342,7 +305,7 @@ describe('a chat whose Claude CLI keeps failing to start, seen by a subscriber o // The CLI is fixed: Retry delivers and adds no row. claude.behave(SESSION, {}) await expect(attempt(host, 'hello?')).resolves.toMatchObject({ ok: true }) - await vi.waitFor(() => expect(claude.child(SESSION).calls).toContain('send')) + await eventually(() => expect(claude.child(SESSION).calls).toContain('send')) expect(shownRows(events).size).toBe(3) } finally { unsubscribe() diff --git a/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts b/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts index 91bf1158e07..c530a1e84ab 100644 --- a/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts +++ b/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts @@ -91,7 +91,7 @@ describe('structured mailbox pointer host', () => { value: { submission: { dispatchState } } }) ) - hostRef.current = { send } + hostRef.current = { send, waitForSendSettlement: async () => undefined } await expect( createStructuredMailboxPointerHost().send({ sessionId: 's1', @@ -107,6 +107,28 @@ describe('structured mailbox pointer host', () => { expect(send.mock.calls[0]![1]!.retryUnknown).toBeUndefined() }) + it('consumes mail once an accepted nudge is delivered while the worker starts (W10)', async () => { + hostRef.current = { + send: async () => ({ + ok: true, + value: { clientMessageId: 'op1', submission: { dispatchState: 'pending' } } + }), + waitForSendSettlement: async () => ({ + value: { clientMessageId: 'op1', submission: { dispatchState: 'accepted' } } + }) + } + await expect( + createStructuredMailboxPointerHost().send({ + sessionId: 's1', + dispatchId: 'd1', + operationId: 'op1', + expectedRuntimeFence: 1, + payloadFingerprint: 'fp', + body: { kind: 'message', role: 'user', blocks: [] } + } as never) + ).resolves.toEqual({ kind: 'sent', state: 'accepted' }) + }) + it('scopes direct peer mail to the session when there is no dispatch to scope to', async () => { // Direct mail is addressed to the worker's own handle, so there may be no dispatch at all. // The ledger is keyed on (callerKey, operationId): a key derived from the session keeps that diff --git a/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts b/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts index b722952df92..907a7bcc097 100644 --- a/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts +++ b/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts @@ -6,6 +6,7 @@ * the send and reports what the host said. */ +import { ORCHESTRATION_READINESS_TIMEOUT_MS } from '../../../shared/orchestration-timing-budgets' import { AGENT_SESSION_NOT_ATTACHED } from '../../native-chat/agent-session-wire/structured-agent-session-mutation-admission' import { getStructuredAgentSessionHost } from '../../native-chat/agent-session-wire/structured-agent-session-registry' import type { StructuredMailboxPointerHost } from './structured-mailbox-pointer-delivery' @@ -94,8 +95,19 @@ export function createStructuredMailboxPointerHost(): StructuredMailboxPointerHo ? { kind: 'unattached' } : { kind: 'sent', state: 'rejected' } } - // `pending` is not yet an acknowledgement; only `accepted` may consume mail. - const state = result.value.submission.dispatchState + // `pending` is not yet an acknowledgement; only `accepted` may consume mail. Accepted is not + // delivered, so wait out a start; a wait that runs out parks for the next journal edge. + const submission = + result.value.submission.dispatchState === 'pending' + ? (( + await host + .waitForSendSettlement(input.sessionId, result.value.clientMessageId, { + budgetMs: ORCHESTRATION_READINESS_TIMEOUT_MS + }) + .catch(() => undefined) + )?.value.submission ?? result.value.submission) + : result.value.submission + const state = submission.dispatchState return { kind: 'sent', state: state === 'accepted' ? 'accepted' : state === 'rejected' ? 'rejected' : 'unknown' diff --git a/src/main/runtime/rpc/methods/orchestration-structured-worker-session.test.ts b/src/main/runtime/rpc/methods/orchestration-structured-worker-session.test.ts index 493f9d092c3..86568a0435f 100644 --- a/src/main/runtime/rpc/methods/orchestration-structured-worker-session.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-structured-worker-session.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentJournalSubmission } from '../../../../shared/agent-session-journal-types' import { dispatchWriteFailureReason } from '../../../../shared/structured-agent-session-dispatch-rejection' const hostRef: { current: unknown } = { current: null } @@ -224,20 +225,69 @@ describe('structured worker session hold', () => { }) describe('structured worker dispatch preamble', () => { - function hostWithSubmission(submission: Record) { + type PreambleHost = Parameters[0]['host'] + type Settled = Pick + + function submissionOf(settled: Settled): AgentJournalSubmission { return { - deps: { store: { getRecord: () => ({ lease: { runtimeFence: 7 } }) } }, - send: async () => ({ ok: true, value: { clientMessageId: 'c1', submission } }) - } as never + clientMessageId: 'c1', + fence: 7, + payloadFingerprint: 'fingerprint', + providerItemId: null, + submittedAt: 1, + resolvedAt: null, + ...settled + } } - const send = (host: never) => + function hostWithSubmission(submission: Settled, delivered?: Settled): PreambleHost { + return { + deps: { store: { getRecord: () => ({ lease: { runtimeFence: 7 } }) } }, + send: async () => ({ + ok: true, + replayed: false, + fence: 7, + cursor: { epoch: 'epoch-1', sequence: 1 }, + value: { clientMessageId: 'c1', submission: submissionOf(submission) } + }), + // What the submission settled as while the worker's agent started; undefined when the + // start outlasted the wait. + waitForSendSettlement: async () => + delivered + ? { + cursor: { epoch: 'epoch-1', sequence: 2 }, + value: { clientMessageId: 'c1', submission: submissionOf(delivered) } + } + : undefined + } + } + + const send = (host: PreambleHost) => sendStructuredWorkerPreamble({ host, sessionId: 's1', dispatchId: 'd1', preamble: 'spec' }) it('reports the preamble delivered only on an accepted submission', async () => { await expect( send(hostWithSubmission({ dispatchState: 'accepted', reason: null })) - ).resolves.toBeUndefined() + ).resolves.toBe('accepted') + }) + + it('waits for an accepted preamble to be delivered, and reports that delivery (W10)', async () => { + await expect( + send( + hostWithSubmission( + { dispatchState: 'pending', reason: null }, + { dispatchState: 'accepted', reason: null } + ) + ) + ).resolves.toBe('accepted') + }) + + it('reports a preamble still held for an agent that outlasted the wait, without failing the start (W10)', async () => { + // Held, not lost: the host delivers it when the agent starts. Throwing here tore the worker + // down, which rejected the preamble the start was about to deliver. + await expect( + send(hostWithSubmission({ dispatchState: 'pending', reason: null })) + ).resolves.toBe('pending') }) it('never claims delivery for a submission the provider never acknowledged', async () => { @@ -245,15 +295,13 @@ describe('structured worker dispatch preamble', () => { // into `unknown`, and `performSend` still returns ok. Reporting that as `dispatch_input: // accepted` marks the worker ready with no task, and the coordinator blocks in // `check --wait --types worker_done` until it times out. - for (const dispatchState of ['unknown', 'pending'] as const) { - const error = await send( - hostWithSubmission({ dispatchState, reason: 'provider child exited' }) - ).catch((thrown: unknown) => thrown) - expect((error as { code?: string }).code).toBe('operation_unknown') - // The wiring, not just the throw: this is the code that makes the start receipt - // `outcome_unknown` with the worker-show / worker-abandon recovery commands. - expect(isUnknownWorkerStartOutcome(error, 'dispatch_input')).toBe(true) - } + const error = await send( + hostWithSubmission({ dispatchState: 'unknown', reason: 'provider child exited' }) + ).catch((thrown: unknown) => thrown) + expect((error as { code?: string }).code).toBe('operation_unknown') + // The wiring, not just the throw: this is the code that makes the start receipt + // `outcome_unknown` with the worker-show / worker-abandon recovery commands. + expect(isUnknownWorkerStartOutcome(error, 'dispatch_input')).toBe(true) }) it('keeps a rejected preamble a proven failure under a code of its own', async () => { diff --git a/src/main/runtime/rpc/methods/orchestration-structured-worker-session.ts b/src/main/runtime/rpc/methods/orchestration-structured-worker-session.ts index 5bde461a059..06f657924b1 100644 --- a/src/main/runtime/rpc/methods/orchestration-structured-worker-session.ts +++ b/src/main/runtime/rpc/methods/orchestration-structured-worker-session.ts @@ -14,6 +14,7 @@ import { randomUUID } from 'node:crypto' import { isDefinitiveAgentSessionCreateRefusal } from '../../../../shared/agent-session-definitive-refusal' import type { AgentJournalMessageItem } from '../../../../shared/agent-session-journal-types' +import { ORCHESTRATION_READINESS_TIMEOUT_MS } from '../../../../shared/orchestration-timing-budgets' import type { StructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-host' import { getStructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-registry' import type { OrcaRuntimeService } from '../../orca-runtime' @@ -213,13 +214,22 @@ export async function discardStructuredWorkerSession( retireSettledStructuredWorkerTab(sessionId, runtime) } -/** Delivers the dispatch preamble as the worker's first turn. */ +/** What a preamble send reads of the host. */ +type StructuredWorkerPreambleHost = Pick< + StructuredAgentSessionHost, + 'send' | 'waitForSendSettlement' +> & { + deps: { store: { getRecord: (sessionId: string) => { lease: { runtimeFence: number } } | null } } +} + +/** Delivers the dispatch preamble as the worker's first turn. `pending`: the worker's agent had + * not taken it within the wait; the host still holds it for that agent, and never re-sends it. */ export async function sendStructuredWorkerPreamble(args: { - host: StructuredAgentSessionHost + host: StructuredWorkerPreambleHost sessionId: string dispatchId: string preamble: string -}): Promise { +}): Promise<'accepted' | 'pending'> { const body: AgentJournalMessageItem = { kind: 'message', role: 'user', @@ -244,9 +254,19 @@ export async function sendStructuredWorkerPreamble(args: { if (!result.ok) { throw new Error(`The dispatch preamble was refused: ${result.refusal.message}`) } - const submission = result.value.submission - if (submission.dispatchState === 'accepted') { - return + // Accepted is not delivered: the worker's agent may still be starting. + const submission = + result.value.submission.dispatchState === 'pending' + ? (( + await args.host + .waitForSendSettlement(args.sessionId, result.value.clientMessageId, { + budgetMs: ORCHESTRATION_READINESS_TIMEOUT_MS + }) + .catch(() => undefined) + )?.value.submission ?? result.value.submission) + : result.value.submission + if (submission.dispatchState === 'accepted' || submission.dispatchState === 'pending') { + return submission.dispatchState } if (submission.dispatchState === 'rejected') { // A rejection is a verdict, not a mystery: the preamble provably did not happen. diff --git a/src/main/runtime/rpc/methods/orchestration-worker-mode-opacity.test.ts b/src/main/runtime/rpc/methods/orchestration-worker-mode-opacity.test.ts index aa8c7c58574..48ffe07d5d0 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-mode-opacity.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-mode-opacity.test.ts @@ -42,6 +42,7 @@ vi.mock('./orchestration-structured-worker-session', async (importOriginal) => ( ...(await importOriginal>()), sendStructuredWorkerPreamble: async (args: { preamble: string }) => { structuredPreambles.push(args.preamble) + return 'accepted' }, releaseStructuredWorkerSession: () => {}, discardStructuredWorkerSession: async () => {} diff --git a/src/main/runtime/rpc/methods/orchestration-worker-start-mode-selection.test.ts b/src/main/runtime/rpc/methods/orchestration-worker-start-mode-selection.test.ts index 52cf3579016..a2beea9ea05 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-start-mode-selection.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-start-mode-selection.test.ts @@ -30,7 +30,7 @@ vi.mock('./orchestration/federation/federated-worker-start', () => ({ })) vi.mock('./orchestration-structured-worker-session', async (importOriginal) => ({ ...(await importOriginal>()), - sendStructuredWorkerPreamble: async () => {}, + sendStructuredWorkerPreamble: async () => 'accepted', releaseStructuredWorkerSession: () => {}, discardStructuredWorkerSession: async () => {} })) diff --git a/src/main/runtime/rpc/methods/orchestration/worker/deliver-worker-dispatch-preamble.ts b/src/main/runtime/rpc/methods/orchestration/worker/deliver-worker-dispatch-preamble.ts index ecb3270874e..bc2b9601ceb 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/deliver-worker-dispatch-preamble.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/deliver-worker-dispatch-preamble.ts @@ -5,6 +5,7 @@ import { dispatchPreambleSendOptions } from '../../../../orchestration/preamble' import { sendStructuredWorkerPreamble } from '../../orchestration-structured-worker-session' +import type { WorkerTurnStartObservation } from './worker-start-turn-observation' import type { createStructuredWorkerSessionForWorktree } from './worker-topology' type StructuredSession = Awaited> | null @@ -14,7 +15,8 @@ type StructuredSession = Awaited { +}): Promise<{ + prompt?: RuntimeTerminalSend['prompt'] + structuredTurnStart?: WorkerTurnStartObservation +}> { const { runtime, structuredSession, terminalHandle } = args const preamble = buildDispatchPreamble({ // Depth only. A worker is taught the same verbs whichever mode it runs in, so this must not @@ -45,19 +50,32 @@ export async function deliverWorkerDispatchPreamble(args: { cliCommand: runtime.getTerminalOrchestrationCliCommand(terminalHandle) }) if (structuredSession) { - await sendStructuredWorkerPreamble({ + const delivery = await sendStructuredWorkerPreamble({ host: structuredSession.host, sessionId: structuredSession.identity.sessionId, dispatchId: args.dispatchId, preamble }) - return undefined + return { + structuredTurnStart: + delivery === 'accepted' + ? { verdict: 'observed' } + : { + verdict: 'unobserved', + reason: + 'The dispatch preamble was accepted, but the agent had not started to take it. It ' + + 'is delivered when the agent starts; if the worker then reports, this Dispatch ' + + 'settles normally.' + } + } + } + return { + prompt: ( + await runtime.sendTerminalAgentPrompt( + terminalHandle, + preamble, + dispatchPreambleSendOptions(args.requestId) + ) + ).prompt } - return ( - await runtime.sendTerminalAgentPrompt( - terminalHandle, - preamble, - dispatchPreambleSendOptions(args.requestId) - ) - ).prompt } diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.test.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.test.ts new file mode 100644 index 00000000000..e39a06110dc --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('../../../../orchestration/preamble', () => ({ buildDispatchPreamble: () => 'preamble' })) +vi.mock('./worker-topology', async (importOriginal) => ({ + ...(await importOriginal>()), + monitorWorkerSetup: () => {} +})) + +const { deliverAndSettleWorkerStartReadiness } = await import('./worker-start-readiness-settlement') + +function settle(delivered: 'accepted' | undefined) { + const db = { + getWorkerDispatch: () => ({ state: 'starting' }), + markWorkerStartUnknown: vi.fn(() => ({ + stage: 'turn_start_unobserved', + residual_resources: '[]' + })), + markWorkerDispatchReady: vi.fn(() => ({ state: 'ready', stage: 'ready' })) + } + const host = { + deps: { store: { getRecord: () => ({ lease: { runtimeFence: 1 } }) } }, + send: async () => ({ + ok: true, + value: { clientMessageId: 'c1', submission: { dispatchState: 'pending', reason: null } } + }), + // undefined: the worker's agent was still starting when the wait ran out. + waitForSendSettlement: async () => + delivered + ? { value: { clientMessageId: 'c1', submission: { dispatchState: delivered } } } + : undefined + } + const args = { + runtime: { + getNestedWorkerMaxDepth: () => 3, + getTerminalOrchestrationCliCommand: () => 'orca' + }, + db, + run: { id: 'run_1' }, + task: { id: 't1', spec: 'do the thing' }, + dispatchId: 'd1', + dispatchDepth: 0, + structuredSession: { host, identity: { sessionId: 's1' } }, + terminalHandle: 'structured_worker_1', + coordinatorHandle: 'term_c', + dispatchCapability: 'capability', + devMode: undefined, + requestId: 'r1', + agent: 'claude', + setupReceipt: {}, + launchReceipt: {}, + mode: {}, + timeoutMs: 60_000, + effects: [], + terminalRevealWarning: undefined, + onStage: () => {} + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the fakes implement exactly the runtime, db and host members this settlement reaches. + const receipt = deliverAndSettleWorkerStartReadiness(args as never) + return { db, receipt } +} + +describe('a structured worker whose agent outlasts the preamble wait', () => { + it('parks as start-unknown instead of failing the start, and never names a screen to read', async () => { + const { db, receipt } = settle(undefined) + + // Resolving, not throwing, is what keeps the worker's session: a throw tears it down. + await expect(receipt).resolves.toMatchObject({ + state: 'outcome_unknown', + turnStart: 'unobserved', + nextCommands: [ + 'orca orchestration worker-show --dispatch d1 --json', + 'orca orchestration worker-abandon --dispatch d1 --json' + ] + }) + expect(db.markWorkerStartUnknown).toHaveBeenCalledWith( + 'd1', + 'turn_start_unobserved', + expect.stringContaining('delivered when the agent starts'), + expect.anything() + ) + expect(db.markWorkerDispatchReady).not.toHaveBeenCalled() + }) + + it('is ready once the agent took the preamble within the wait', async () => { + const { db, receipt } = settle('accepted') + + await expect(receipt).resolves.toMatchObject({ state: 'ready', turnStart: 'observed' }) + expect(db.markWorkerStartUnknown).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.ts index a3c57a357d2..7e9118daffb 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-readiness-settlement.ts @@ -47,7 +47,7 @@ export async function deliverAndSettleWorkerStartReadiness(args: { const { runtime, db, run, task, structuredSession, terminalHandle, effects } = args args.onStage('dispatch_input') - const promptDelivery = await deliverWorkerDispatchPreamble({ + const delivery = await deliverWorkerDispatchPreamble({ runtime, structuredSession, terminalHandle, @@ -71,11 +71,11 @@ export async function deliverAndSettleWorkerStartReadiness(args: { // The write above was accepted without waiting on provider hooks; now demand the positive // evidence the receipt claims is observable. A worker whose turn never starts must not be // reported ready — a wedged agent and a working one looked identical before this gate. - // A structured preamble send is acknowledged by the provider or throws, so it is already - // positive evidence. - const turnStart: WorkerTurnStartObservation = structuredSession - ? { verdict: 'observed' } - : await observeWorkerTurnStart({ runtime, terminalHandle, prompt: promptDelivery }) + // A structured preamble send is its own evidence: acknowledged, or still held for its agent. + const promptDelivery = delivery.prompt + const turnStart: WorkerTurnStartObservation = + delivery.structuredTurnStart ?? + (await observeWorkerTurnStart({ runtime, terminalHandle, prompt: promptDelivery })) const deliveredPrompt = turnStart.prompt ?? promptDelivery monitorWorkerSetup({ runtime, @@ -98,7 +98,7 @@ export async function deliverAndSettleWorkerStartReadiness(args: { id: terminalHandle, state: 'turn_unobserved' }) - const reason = describeUnobservedWorkerTurnStart(args.agent) + const reason = turnStart.reason ?? describeUnobservedWorkerTurnStart(args.agent) const worker = db.markWorkerStartUnknown( args.dispatchId, 'turn_start_unobserved', @@ -122,7 +122,8 @@ export async function deliverAndSettleWorkerStartReadiness(args: { residualResources: JSON.parse(worker.residual_resources) as unknown[], nextCommands: [ `orca orchestration worker-show --dispatch ${args.dispatchId} --json`, - `orca terminal read --terminal ${terminalHandle} --screen`, + // A structured worker has no screen to read. + ...(structuredSession ? [] : [`orca terminal read --terminal ${terminalHandle} --screen`]), `orca orchestration worker-abandon --dispatch ${args.dispatchId} --json` ], ...(args.terminalRevealWarning ? { warning: args.terminalRevealWarning } : {}) diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-start-turn-observation.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-turn-observation.ts index d1c9e59adfa..cc648ee7dde 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-start-turn-observation.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-turn-observation.ts @@ -19,6 +19,8 @@ export type WorkerTurnStartVerdict = 'observed' | 'permission' | 'unsupported' | export type WorkerTurnStartObservation = { verdict: WorkerTurnStartVerdict prompt?: RuntimeTerminalPromptDelivery + /** Why an `unobserved` start is unknown, when the default PTY wording does not fit. */ + reason?: string } function classifyPromptDelivery(prompt: RuntimeTerminalPromptDelivery): WorkerTurnStartVerdict { diff --git a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts new file mode 100644 index 00000000000..9ae846df6fc --- /dev/null +++ b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.test.ts @@ -0,0 +1,84 @@ +// Which clients get a send answered at acceptance, and which have their reply held until the +// message is handed over: a client that cannot show a rejection after `pending` must not see one. + +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { + ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES, + STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY +} from '../../../../shared/protocol-version' +import { DESKTOP_RENDERER_RUNTIME_CLIENT_CAPABILITIES } from '../../../ipc/desktop-renderer-runtime-capabilities' +import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from '../../../native-chat/agent-session-wire/structured-agent-session-send-settlement' +import { + call, + clearStructuredHostStub, + hostCalls, + installStructuredHostStub, + SESSION, + sendParams, + STRUCTURED_CLIENT +} from './structured-agent-session-rpc.test-fixture' + +beforeEach(() => { + installStructuredHostStub() +}) + +afterEach(() => { + clearStructuredHostStub() +}) + +describe('agentSession.send reply timing', () => { + it('holds a pending reply until handover for a client that cannot show a later rejection', async () => { + hostCalls.send.mockResolvedValueOnce(pendingSendResult()) + hostCalls.waitForSendSettlement.mockResolvedValueOnce(undefined) + + await call('agentSession.send', sendParams(), STRUCTURED_CLIENT) + + expect(hostCalls.waitForSendSettlement).toHaveBeenCalledWith(SESSION, 'client-1', { + until: 'handed-over', + budgetMs: STRUCTURED_AGENT_SESSION_START_WAIT_MS + }) + }) + + it('answers at acceptance for the local desktop and paired desktop clients (W2)', async () => { + for (const clientCapabilities of [ + DESKTOP_RENDERER_RUNTIME_CLIENT_CAPABILITIES, + // A paired desktop gains the structured surface as its own capability; the reply rule rides + // on the list it already sends. + [...ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES, STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY] + ]) { + hostCalls.send.mockResolvedValueOnce(pendingSendResult()) + const response = await call('agentSession.send', sendParams(), { + clientKind: 'runtime', + clientCapabilities: [...clientCapabilities] + }) + expect(response).toMatchObject({ + ok: true, + result: { value: { submission: { dispatchState: 'pending' } } } + }) + } + expect(hostCalls.waitForSendSettlement).not.toHaveBeenCalled() + }) +}) + +function pendingSendResult() { + return { + ok: true, + replayed: false, + fence: 1, + cursor: { epoch: 'epoch-a', sequence: 1 }, + value: { + clientMessageId: 'client-1', + submission: { + clientMessageId: 'client-1', + fence: 1, + payloadFingerprint: 'fingerprint', + dispatchState: 'pending' as const, + providerItemId: null, + reason: null, + submittedAt: 1, + resolvedAt: null, + handoverRecorded: true as const + } + } + } +} diff --git a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts index 8b948869990..5fde5e0f6e4 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-send-compatibility.ts @@ -1,26 +1,43 @@ -import { AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' +import { + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, + AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY +} from '../../../../shared/protocol-version' import type { StructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-host' +import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from '../../../native-chat/agent-session-wire/structured-agent-session-send-settlement' import type { RpcContext } from '../core' import { requireStructuredHost, structuredCallerFor } from './structured-agent-session-gate' +/** + * A send answers once the host accepts it. A client that predates that answer cannot show a + * message rejected after it, so its reply is held until the message is handed over or rejected; + * one that predates pending replies at all waits, as before, for the provider's answer. + */ export async function sendStructuredAgentSessionForClient( params: Parameters[1], context: RpcContext ) { const host = requireStructuredHost(context) const result = await host.send(structuredCallerFor(context), params) + const capabilities = context.clientCapabilities ?? [] if ( !result.ok || result.value.submission.dispatchState !== 'pending' || context.clientKind === undefined || - context.clientCapabilities?.includes(AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY) + capabilities.includes(AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY) ) { return result } + // The start that used to run before the reply now runs after acceptance, so both waits cover it. const settled = await host.waitForSendSettlement( params.envelope.sessionId, result.value.clientMessageId, - context.signal + { + until: capabilities.includes(AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY) + ? 'handed-over' + : 'answered', + budgetMs: STRUCTURED_AGENT_SESSION_START_WAIT_MS, + ...(context.signal ? { signal: context.signal } : {}) + } ) return settled ? { ...result, ...settled } : result } diff --git a/src/main/runtime/rpc/methods/structured-agent-session.test.ts b/src/main/runtime/rpc/methods/structured-agent-session.test.ts index 38b55fd70eb..f97a458226a 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session.test.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session.test.ts @@ -4,6 +4,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { setStructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-registry' import { + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY, RUNTIME_CAPABILITIES, RUNTIME_PROTOCOL_VERSION, @@ -12,6 +13,7 @@ import { STRUCTURED_AGENT_SESSION_REVEAL_RUNTIME_CAPABILITY, STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' +import { STRUCTURED_AGENT_SESSION_START_WAIT_MS } from '../../../native-chat/agent-session-wire/structured-agent-session-send-settlement' import { computeAgentSessionPayloadFingerprint } from '../../../../shared/agent-session-mutation-envelope' import { ALL_RPC_METHODS } from './index' import { STRUCTURED_AGENT_SESSION_METHODS } from './structured-agent-session' @@ -149,6 +151,8 @@ describe('capability gating', () => { it('advertises the capability without bumping the protocol version', () => { expect(RUNTIME_CAPABILITIES).toContain(STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY) expect(RUNTIME_CAPABILITIES).toContain(AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY) + // A client tells a host that accepts first, and admits a writer-free Stop before a turn, by it. + expect(RUNTIME_CAPABILITIES).toContain(AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY) expect(RUNTIME_CAPABILITIES).toContain(STRUCTURED_AGENT_SESSION_HOLD_RUNTIME_CAPABILITY) expect(RUNTIME_CAPABILITIES).toContain(STRUCTURED_AGENT_SESSION_REVEAL_RUNTIME_CAPABILITY) // Separate from the structured capability on purpose: a host can serve the rest of the @@ -252,11 +256,11 @@ describe('capability gating', () => { signal: controller.signal }) - expect(hostCalls.waitForSendSettlement).toHaveBeenCalledWith( - SESSION, - 'client-1', - controller.signal - ) + expect(hostCalls.waitForSendSettlement).toHaveBeenCalledWith(SESSION, 'client-1', { + until: 'answered', + budgetMs: STRUCTURED_AGENT_SESSION_START_WAIT_MS, + signal: controller.signal + }) expect(response).toMatchObject({ ok: true, result: { @@ -302,36 +306,6 @@ describe('capability gating', () => { }) }) - it('returns durable pending immediately to clients that understand admission', async () => { - hostCalls.send.mockResolvedValueOnce({ - ok: true, - replayed: false, - fence: 1, - cursor: { epoch: 'epoch-a', sequence: 1 }, - value: { - clientMessageId: 'client-1', - submission: { - clientMessageId: 'client-1', - fence: 1, - payloadFingerprint: 'fingerprint', - dispatchState: 'pending', - providerItemId: null, - reason: null, - submittedAt: 1, - resolvedAt: null - } - } - }) - - const response = await call('agentSession.send', sendParams(), STRUCTURED_CLIENT) - - expect(hostCalls.waitForSendSettlement).not.toHaveBeenCalled() - expect(response).toMatchObject({ - ok: true, - result: { value: { submission: { dispatchState: 'pending' } } } - }) - }) - it('requires the host structured-chat setting for mobile clients', async () => { const response = await call('agentSession.send', sendParams(), STRUCTURED_MOBILE_CLIENT, { getClientSettings: () => ({ experimentalStructuredNativeChat: false }) diff --git a/src/main/runtime/runtime-worktree-structured-agent-rows-liveness.test.ts b/src/main/runtime/runtime-worktree-structured-agent-rows-liveness.test.ts index b1e43a9025d..27d5ba48b29 100644 --- a/src/main/runtime/runtime-worktree-structured-agent-rows-liveness.test.ts +++ b/src/main/runtime/runtime-worktree-structured-agent-rows-liveness.test.ts @@ -89,7 +89,7 @@ async function awaitingApproval() { SESSION, { journal, - hasProviderChild: true, + child: { phase: 'ready' as const }, params: { location: { executionHostId: 'local' as const, diff --git a/src/main/runtime/structured-agent-session-integration-replay.test.ts b/src/main/runtime/structured-agent-session-integration-replay.test.ts index 4fa390be0fb..a4e2500ffc4 100644 --- a/src/main/runtime/structured-agent-session-integration-replay.test.ts +++ b/src/main/runtime/structured-agent-session-integration-replay.test.ts @@ -317,11 +317,14 @@ describe('a structured codex session over agentSession.*', () => { body } + const turnStarts = () => codex.live().calls.filter((entry) => entry.method === 'turn/start') await ok('agentSession.send', params) + // Accepted first; the delivery loop hands it over once. + await vi.waitFor(() => expect(turnStarts()).toHaveLength(1)) const replay = await call('agentSession.send', params) expect(replay).toMatchObject({ ok: true, result: { ok: true, replayed: true } }) - expect(codex.live().calls.filter((entry) => entry.method === 'turn/start')).toHaveLength(1) + expect(turnStarts()).toHaveLength(1) }) it('joins an acquired attach through journal bind before draining final rows', async () => { diff --git a/src/main/runtime/structured-agent-session-integration.test.ts b/src/main/runtime/structured-agent-session-integration.test.ts index 3f8adae4906..d81170ee075 100644 --- a/src/main/runtime/structured-agent-session-integration.test.ts +++ b/src/main/runtime/structured-agent-session-integration.test.ts @@ -218,6 +218,11 @@ function createIntentParams() { } let codex: CodexScript +/** Accepted first; the delivery loop hands the send over as `turn/start` after the reply. */ +const handedOverAs = (params: Record) => + vi.waitFor(() => + expect(codex.live().calls.at(-1)).toMatchObject({ method: 'turn/start', params }) + ) let root: string let dispatcher: RpcDispatcher let bootEnvironmentReads: number @@ -466,10 +471,7 @@ describe('a structured codex session over agentSession.*', () => { // send coalesced into a running turn is answered with that turn's id, so // which message landed where is knowable only from the echo. expect(sent.submission).toMatchObject({ dispatchState: 'pending', providerItemId: null }) - expect(codex.live().calls.at(-1)).toMatchObject({ - method: 'turn/start', - params: { threadId: THREAD, clientUserMessageId: sent.clientMessageId } - }) + await handedOverAs({ threadId: THREAD, clientUserMessageId: sent.clientMessageId }) codex.notify('turn/started', { turn: { id: TURN } }) // Codex echoes the message back carrying the `clientId` it was sent under, @@ -557,14 +559,11 @@ describe('a structured codex session over agentSession.*', () => { // "delivery unconfirmed" — it carries no identity yet, because the response // to a coalesced send names the running turn rather than this message. expect(sent.submission).toMatchObject({ dispatchState: 'pending', providerItemId: null }) - expect(codex.live().calls.at(-1)).toMatchObject({ - method: 'turn/start', - params: { - threadId: THREAD, - clientUserMessageId: sent.clientMessageId, - model: 'gpt-live', - effort: 'high' - } + await handedOverAs({ + threadId: THREAD, + clientUserMessageId: sent.clientMessageId, + model: 'gpt-live', + effort: 'high' }) // ── stream ────────────────────────────────────────────────────────────── diff --git a/src/main/runtime/structured-agent-session-runtime-exit.test.ts b/src/main/runtime/structured-agent-session-runtime-exit.test.ts index 7a3736b9009..0b2c8bd4f4b 100644 --- a/src/main/runtime/structured-agent-session-runtime-exit.test.ts +++ b/src/main/runtime/structured-agent-session-runtime-exit.test.ts @@ -128,7 +128,8 @@ describe('structured session runtime provider-exit wiring', () => { await expect( host.send({ callerKey: 'runtime-test' }, { envelope, body }) ).resolves.toMatchObject({ ok: true, value: { submission: { dispatchState: 'pending' } } }) - expect(turn).toBe(1) + // The send answers at acceptance; the delivery loop hands it to the reacquired child after. + await vi.waitFor(() => expect(turn).toBe(1)) }) it('does not reacquire when the production exit callback comes from a requested close', async () => { diff --git a/src/renderer/src/components/native-chat/NativeChatDeliveryRetry.tsx b/src/renderer/src/components/native-chat/NativeChatDeliveryRetry.tsx index 5a93657f943..e37016d646d 100644 --- a/src/renderer/src/components/native-chat/NativeChatDeliveryRetry.tsx +++ b/src/renderer/src/components/native-chat/NativeChatDeliveryRetry.tsx @@ -18,9 +18,13 @@ export function NativeChatDeliveryRetry({ retry: (clientMessageId: string) => void }): React.JSX.Element | null { // Why: read through the drain's own rule, so Retry can never name an entry other than the one - // the queue actually stopped on -- which is no longer always the head. + // the queue actually stopped on -- which is no longer always the head. A rejected entry holds + // nothing up, so it is offered only when the queue itself is not stopped. const admission = admitStructuredAgentSessionOutboxEntry(outbox, blockedClientMessageId) - const retryable = admission.state === 'blocked' ? admission.entry : null + const retryable = + admission.state === 'blocked' + ? admission.entry + : (outbox.find((entry) => entry.state === 'rejected') ?? null) if (!retryable) { return null } diff --git a/src/renderer/src/components/native-chat/structured-agent-session-outbox-dispatch.ts b/src/renderer/src/components/native-chat/structured-agent-session-outbox-dispatch.ts index 91e6149cb09..9f91b9f9b3b 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-outbox-dispatch.ts +++ b/src/renderer/src/components/native-chat/structured-agent-session-outbox-dispatch.ts @@ -57,6 +57,18 @@ export function readMountedStructuredAgentSessionOutbox( ) } +/** A send left dispatching when its owner changed goes out again, under the same id. */ +export function requeueInterruptedStructuredAgentSessionDispatches( + entries: StructuredAgentSessionOutboxEntry[], + fence: number | null +): StructuredAgentSessionOutboxEntry[] { + return entries.map((entry) => + entry.state === 'dispatching' && !hasInFlightLaunchDispatch(entry, fence) + ? { ...entry, state: 'queued' as const } + : entry + ) +} + export function dispatchStructuredAgentSessionOutboxEntry(args: { next: StructuredAgentSessionOutboxEntry persisted: readonly StructuredAgentSessionOutboxEntry[] diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-fence.test.tsx b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-fence.test.tsx new file mode 100644 index 00000000000..61b5ea95e72 --- /dev/null +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-fence.test.tsx @@ -0,0 +1,139 @@ +// @vitest-environment happy-dom + +// A moved fence is not a reason to send anything again on a host that records every send before it +// starts an agent. There, only a Retry or a new send goes out. An older host, which restarts the +// agent inside the send and refuses it unrecorded when that fails, keeps the resend on a new fence. + +import { act, renderHook, waitFor } from '@testing-library/react' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' + +const mocks = vi.hoisted(() => ({ call: vi.fn() })) + +vi.mock('@/runtime/structured-agent-session-client', () => ({ + callStructuredAgentSession: mocks.call +})) + +import { setLocalRuntimeCapabilitiesForTests } from '@/runtime/local-runtime-capabilities' +import { useStructuredAgentSessionOutbox } from './use-structured-agent-session-outbox' + +const LOCAL_TARGET = { kind: 'local' } as const + +function deferred() { + let resolve!: (value: T) => void + const promise = new Promise((next) => { + resolve = next + }) + return { promise, resolve } +} + +function pendingResult(clientMessageId: string) { + return { + ok: true, + replayed: false, + fence: 1, + cursor: { epoch: 'epoch-1', sequence: 1 }, + value: { + clientMessageId, + submission: { + clientMessageId, + fence: 1, + payloadFingerprint: 'fingerprint', + dispatchState: 'pending' as const, + handoverRecorded: true, + providerItemId: null, + reason: null, + submittedAt: 1, + resolvedAt: null + } + } + } +} + +function sentId(call: number): string { + const id: unknown = mocks.call.mock.calls[call]?.[2].envelope.clientOperationId + return String(id) +} + +function render() { + return renderHook( + ({ fence }) => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target: LOCAL_TARGET, + fence, + submissions: [] + }), + { initialProps: { fence: 1 } } + ) +} + +/** Long enough for any effect a fence change schedules to have sent. */ +async function settle(): Promise { + await act(() => new Promise((resolve) => setTimeout(resolve, 50))) +} + +describe('an outbox on a host that accepts a send before any agent has it', () => { + beforeEach(() => { + vi.clearAllMocks() + localStorage.clear() + setLocalRuntimeCapabilitiesForTests([AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY]) + }) + + it('neither resends nor drops the answer of a send in flight when the fence moves', async () => { + const answer = deferred>() + mocks.call.mockReturnValueOnce(answer.promise) + const { result, rerender } = render() + + act(() => expect(result.current.send('hello')).toBe(true)) + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(1)) + // A start or restart on the host moves the fence while the send is out. + rerender({ fence: 2 }) + rerender({ fence: 3 }) + await settle() + expect(mocks.call).toHaveBeenCalledTimes(1) + + await act(async () => answer.resolve(pendingResult(sentId(0)))) + // The answer lands: the entry is the host's now, not re-queued behind a moved fence. + expect(result.current.outbox).toMatchObject([{ state: 'dispatching' }]) + await settle() + expect(mocks.call).toHaveBeenCalledTimes(1) + }) + + it('keeps a blocked head blocked across a fence change; only Retry sends it', async () => { + mocks.call.mockRejectedValueOnce(new Error('send failed')).mockResolvedValue({ ok: true }) + const { result, rerender } = render() + + act(() => expect(result.current.send('hello')).toBe(true)) + await waitFor(() => expect(result.current.blockedClientMessageId).not.toBeNull()) + rerender({ fence: 2 }) + await settle() + expect(mocks.call).toHaveBeenCalledTimes(1) + // The failure stays on the message; the fence change neither clears nor resends it. + expect(result.current.outbox[0]?.lastFailure).toEqual({ kind: 'failed' }) + + act(() => result.current.retry(result.current.outbox[0]!.clientMessageId)) + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(2)) + }) +}) + +describe('an outbox on an older host', () => { + beforeEach(() => { + vi.clearAllMocks() + localStorage.clear() + setLocalRuntimeCapabilitiesForTests([]) + }) + + it('still resends a send in flight when the fence moves, as the new owner may take it', async () => { + mocks.call.mockReturnValueOnce(new Promise(() => {})).mockReturnValue(new Promise(() => {})) + const { result, rerender } = render() + + act(() => expect(result.current.send('hello')).toBe(true)) + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(1)) + rerender({ fence: 2 }) + + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(2)) + expect(sentId(1)).toBe(sentId(0)) + }) +}) diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-rejection-cause.test.tsx b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-rejection-cause.test.tsx index b6774f46416..a8b9f01b20b 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-rejection-cause.test.tsx +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox-rejection-cause.test.tsx @@ -1,7 +1,9 @@ // @vitest-environment happy-dom import { act, renderHook, waitFor } from '@testing-library/react' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentJournalSubmission } from '../../../../shared/agent-session-journal-types' +import { DISPATCH_REJECTED_CANCELLED } from '../../../../shared/structured-agent-session-dispatch-rejection' const mocks = vi.hoisted(() => ({ call: vi.fn() @@ -11,7 +13,7 @@ vi.mock('@/runtime/structured-agent-session-client', () => ({ callStructuredAgentSession: mocks.call })) -import type { AgentJournalSubmission } from '../../../../shared/agent-session-journal-types' +import { setLocalRuntimeCapabilitiesForTests } from '@/runtime/local-runtime-capabilities' import { useStructuredAgentSessionOutbox } from './use-structured-agent-session-outbox' import { agentSessionWriteNoticeEnglish } from '../../../../shared/agent-session-refusal-notice' import { structuredAgentSessionAttemptFailureParts } from '../../../../shared/structured-agent-session-send-disposition' @@ -28,6 +30,28 @@ function shownFailure(entry: StructuredAgentSessionOutboxEntry | undefined): str const REASON = 'The provider stopped before it finished starting: claude stream-json exited (code 1): claude: not signed in.' +function acceptedResultFor(clientMessageId: string) { + return { + ok: true, + replayed: false, + fence: 1, + cursor: { epoch: 'epoch-1', sequence: 2 }, + value: { + clientMessageId, + submission: { + clientMessageId, + fence: 1, + payloadFingerprint: 'fingerprint', + dispatchState: 'accepted', + providerItemId: `provider-${clientMessageId}`, + reason: null, + submittedAt: 1, + resolvedAt: 1 + } + } + } +} + function rejectedResultFor(clientMessageId: string) { return { ok: true, @@ -76,14 +100,241 @@ describe('a send the host rejected because the agent never started', () => { act(() => expect(result.current.send('hello')).toBe(true)) await waitFor(() => expect(shownFailure(result.current.outbox[0])).toBe(REASON)) - expect(result.current.outbox[0]?.state).toBe('queued') - expect(result.current.blockedClientMessageId).toBe(result.current.outbox[0]?.clientMessageId) + // Settled as not delivered: it waits for Retry and holds no later message up. + expect(result.current.error).toBeNull() + expect(result.current.outbox[0]?.state).toBe('rejected') + expect(result.current.blockedClientMessageId).toBeNull() + }) + + it('sends a new message past one the host could not start the agent for, without resending it', async () => { + const message = "Claude couldn't restart: Not logged in. Please run /login." + mocks.call.mockImplementation(async (_target, _method, params) => { + const request = params as { + envelope: { clientOperationId: string } + body: { blocks: { text?: string }[] } + } + return request.body.blocks[0]?.text === 'first' + ? { ok: false, refusal: { code: 'agent_session_owner_restart_failed', message } } + : acceptedResultFor(request.envelope.clientOperationId) + }) + const { result } = renderHook(() => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target: { kind: 'local' }, + fence: 1, + submissions: [] + }) + ) + + act(() => expect(result.current.send('first')).toBe(true)) + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('rejected')) + const rejectedId = result.current.outbox[0]!.clientMessageId + + // The user's next message is the retry of the start: it goes out on its own. + act(() => expect(result.current.send('second')).toBe(true)) + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(2)) + await act(() => new Promise((resolve) => setTimeout(resolve, 50))) + + const sent = mocks.call.mock.calls.map( + (call) => (call[2] as { body?: { blocks?: { text?: string }[] } })?.body?.blocks?.[0]?.text + ) + expect(sent).toEqual(['first', 'second']) + expect(result.current.outbox.map((entry) => [entry.clientMessageId, entry.state])).toEqual([ + [rejectedId, 'rejected'] + ]) + }) + + it('keeps a message the host accepted and then could not deliver, with its reason and Retry', async () => { + const reason = "Codex couldn't restart: spawn codex ENOENT." + mocks.call.mockImplementation( + async ( + _target: unknown, + _method: unknown, + params: { envelope: { clientOperationId: string } } + ) => pendingResultFor(params.envelope.clientOperationId) + ) + const target = { kind: 'local' } as const + const { result, rerender } = renderHook( + (props: { submissions: AgentJournalSubmission[] }) => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target, + fence: 1, + submissions: props.submissions + }), + { initialProps: { submissions: NO_SUBMISSIONS } } + ) + + act(() => expect(result.current.send('hello')).toBe(true)) + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('dispatching')) + const id = result.current.outbox[0]!.clientMessageId + + rerender({ + submissions: [{ ...pendingResultFor(id).value.submission, dispatchState: 'rejected', reason }] + }) + + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('rejected')) + expect(shownFailure(result.current.outbox[0])).toBe(reason) + expect(result.current.error).toBeNull() + expect(result.current.blockedClientMessageId).toBeNull() + + // Retry is a new message with the same text: a fresh id, sent once. + act(() => result.current.retry(id)) + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(2)) + const retried: { + envelope: { clientOperationId: string } + body: { blocks: { text?: string }[] } + } = mocks.call.mock.calls[1]![2] + expect(retried.envelope.clientOperationId).not.toBe(id) + expect(retried.body.blocks[0]?.text).toBe('hello') + }) + + it('says nothing when a Stop withdrew the message', async () => { + mocks.call.mockImplementation( + async ( + _target: unknown, + _method: unknown, + params: { envelope: { clientOperationId: string } } + ) => pendingResultFor(params.envelope.clientOperationId) + ) + const target = { kind: 'local' } as const + const { result, rerender } = renderHook( + (props: { submissions: AgentJournalSubmission[] }) => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target, + fence: 1, + submissions: props.submissions + }), + { initialProps: { submissions: NO_SUBMISSIONS } } + ) + + act(() => expect(result.current.send('hello')).toBe(true)) + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('dispatching')) + const id = result.current.outbox[0]!.clientMessageId + + rerender({ + submissions: [ + { + ...pendingResultFor(id).value.submission, + dispatchState: 'rejected', + reason: DISPATCH_REJECTED_CANCELLED + } + ] + }) + + await waitFor(() => expect(result.current.outbox).toEqual([])) + expect(result.current.error).toBeNull() + }) + + it('reads a message rejected while the chat was closed as not sent, and sends past it', async () => { + const reason = "Codex couldn't restart: spawn codex ENOENT." + mocks.call.mockImplementation( + async ( + _target: unknown, + _method: unknown, + params: { envelope: { clientOperationId: string } } + ) => pendingResultFor(params.envelope.clientOperationId) + ) + const target = { kind: 'local' } as const + const first = renderHook(() => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target, + fence: 1, + submissions: NO_SUBMISSIONS + }) + ) + act(() => expect(first.result.current.send('hello')).toBe(true)) + await waitFor(() => expect(first.result.current.outbox[0]?.state).toBe('dispatching')) + const id = first.result.current.outbox[0]!.clientMessageId + first.unmount() + + // Reopened after the start failed, or after a quit settled the message as not sent. + const rejected = [ + { ...pendingResultFor(id).value.submission, dispatchState: 'rejected' as const, reason } + ] + const { result } = renderHook(() => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target, + fence: 1, + submissions: rejected + }) + ) + + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('rejected')) + expect(shownFailure(result.current.outbox[0])).toBe(reason) + act(() => expect(result.current.send('second')).toBe(true)) + await waitFor(() => expect(mocks.call).toHaveBeenCalledTimes(2)) + }) + + it('keeps the rejection when the journal settles the message before the send answers', async () => { + const reason = "Codex couldn't restart: spawn codex ENOENT." + let answer: (value: unknown) => void = () => undefined + mocks.call.mockImplementationOnce( + () => + new Promise((resolve) => { + answer = resolve + }) + ) + const target = { kind: 'local' } as const + const { result, rerender } = renderHook( + (props: { submissions: AgentJournalSubmission[] }) => + useStructuredAgentSessionOutbox({ + sessionId: 'session-1', + target, + fence: 1, + submissions: props.submissions + }), + { initialProps: { submissions: NO_SUBMISSIONS } } + ) + + act(() => expect(result.current.send('hello')).toBe(true)) + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('dispatching')) + const id = result.current.outbox[0]!.clientMessageId + + // A start refused at once: the rejection frame lands before the send's own `pending` answer. + rerender({ + submissions: [{ ...pendingResultFor(id).value.submission, dispatchState: 'rejected', reason }] + }) + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('rejected')) + await act(async () => answer(pendingResultFor(id))) + + expect(result.current.outbox[0]?.state).toBe('rejected') + expect(shownFailure(result.current.outbox[0])).toBe(reason) + expect(result.current.error).toBeNull() }) }) +const NO_SUBMISSIONS: AgentJournalSubmission[] = [] + +function pendingResultFor(clientMessageId: string) { + const submission: AgentJournalSubmission = { + clientMessageId, + fence: 1, + payloadFingerprint: 'fingerprint', + dispatchState: 'pending', + providerItemId: null, + reason: null, + submittedAt: 1, + resolvedAt: null, + handoverRecorded: true + } + return { + ok: true, + replayed: false, + fence: 1, + cursor: { epoch: 'epoch-1', sequence: 2 }, + value: { + clientMessageId, + submission + } + } +} + // Stable across renders, as a mounted pane's target is. const LOCAL_TARGET = { kind: 'local' } as const -const NO_SUBMISSIONS: AgentJournalSubmission[] = [] function submission( clientMessageId: string, @@ -101,10 +352,16 @@ function submission( } } +// An older host: it restarts the agent inside the send, so a new fence is its word to send again. describe('a send refused while its agent restarted', () => { beforeEach(() => { vi.clearAllMocks() localStorage.clear() + setLocalRuntimeCapabilitiesForTests([]) + }) + + afterEach(() => { + setLocalRuntimeCapabilitiesForTests(null) }) // The live order: the restart takes seconds, so the journal settles the resend before its reply. diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.test.tsx b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.test.tsx index 12c190291b9..04b25f93e8a 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.test.tsx +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.test.tsx @@ -248,7 +248,7 @@ describe('useStructuredAgentSessionOutbox', () => { ) act(() => expect(result.current.send('hello')).toBe(true)) - await waitFor(() => expect(result.current.outbox[0]?.state).toBe('queued')) + await waitFor(() => expect(result.current.outbox[0]?.state).toBe('rejected')) const firstId = (mocks.call.mock.calls[0]![2] as { envelope: { clientOperationId: string } }) .envelope.clientOperationId const retryId = result.current.outbox[0]!.clientMessageId @@ -558,7 +558,9 @@ describe('useStructuredAgentSessionOutbox', () => { expect(mocks.call).toHaveBeenCalledOnce() expect(result.current.outbox).toHaveLength(1) - expect(result.current.blockedClientMessageId).toBe(result.current.outbox[0]?.clientMessageId) + // Never sent, and never re-sent on its own: it waits for Retry and holds nothing up. + expect(result.current.outbox[0]?.state).toBe('rejected') + expect(result.current.blockedClientMessageId).toBeNull() // Settled, not pending: the refused id never ran, so a Retry is a new operation. const sentId: unknown = mocks.call.mock.calls[0]![2].envelope.clientOperationId const retryId = result.current.outbox[0]!.clientMessageId @@ -775,8 +777,8 @@ describe('useStructuredAgentSessionOutbox', () => { // A refused write is answered, not doubted: the entry parks with its rejection rather than // under the "delivery is unconfirmed" banner. The disposition tests pin its words. await waitFor(() => expect(result.current.outbox[0]?.lastFailure?.kind).toBe('rejected')) - expect(result.current.outbox[0]?.state).toBe('queued') - expect(result.current.blockedClientMessageId).toBe(firstId) + expect(result.current.outbox[0]?.state).toBe('rejected') + expect(result.current.blockedClientMessageId).toBeNull() // Retry immediately, before the journal subscription can publish the rejected row. act(() => result.current.retry(firstId)) diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.ts b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.ts index 17a9e2769f3..254c936cf63 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-outbox.ts @@ -7,15 +7,19 @@ import { reconcileStructuredAgentSessionOutbox, type StructuredAgentSessionOutboxEntry } from '../../../../shared/structured-agent-session-outbox' -import type { StructuredAgentSessionSendDisposition } from '../../../../shared/structured-agent-session-send-disposition' +import { + journalAnswersInFlightSend, + type StructuredAgentSessionSendDisposition +} from '../../../../shared/structured-agent-session-send-disposition' import type { RuntimeClientTarget } from '@/runtime/runtime-rpc-client' import { readOutbox, writeOutbox } from './structured-agent-session-outbox-storage' import { dispatchStructuredAgentSessionOutboxEntry, - hasInFlightLaunchDispatch, - readMountedStructuredAgentSessionOutbox + readMountedStructuredAgentSessionOutbox, + requeueInterruptedStructuredAgentSessionDispatches } from './structured-agent-session-outbox-dispatch' import { getStructuredAgentLaunchPromptDispatch } from '@/lib/structured-agent-session-launch-prompt' +import { useStructuredAgentSessionOutboxOwnerChange } from '@/runtime/structured-agent-session-accepted-send-capability' import { createBrowserUuid } from '@/lib/browser-uuid' export function structuredSessionOperationId(): string { @@ -25,8 +29,8 @@ export function structuredSessionOperationId(): string { const UNCONFIRMED_PROBE_BASE_DELAY_MS = 1_000 /** No attempt ceiling: a transport outage outlives any fixed budget, and giving up * restores the wedge this fixes. Growth caps the rate at one status query per 16s. - * A refusal that blocks the head still ends probing until a fence change or a manual - * Retry, because the entry leaves `unconfirmed` -- pre-existing, not closed here. */ + * A refusal that blocks the head still ends probing until a manual Retry (or, on an older + * host, a fence change), because the entry leaves `unconfirmed`. */ const UNCONFIRMED_PROBE_MAX_DELAY_MS = 16_000 export function useStructuredAgentSessionOutbox(args: { @@ -36,7 +40,8 @@ export function useStructuredAgentSessionOutbox(args: { submissions: readonly AgentJournalSubmission[] }) { const { fence, sessionId, submissions, target } = args - const targetKey = target.kind === 'local' ? 'local' : `environment:${target.environmentId}` + // What resends, unblocks and drops a send in flight besides a Retry or a new send; see the hook. + const owner = useStructuredAgentSessionOutboxOwnerChange(target, fence) const [outbox, setOutbox] = useState(() => readMountedStructuredAgentSessionOutbox(sessionId, fence, readOutbox) ) @@ -68,19 +73,15 @@ export function useStructuredAgentSessionOutbox(args: { blockedIdRef.current = null retryWithFreshClientMessageIdRef.current = null probeAttemptsRef.current = { id: null, attempts: 0 } - }, [fence, sessionId, targetKey]) + }, [owner.ownerChange, owner.targetKey, sessionId]) useEffect(() => { const sessionChanged = outboxSessionRef.current !== sessionId outboxSessionRef.current = sessionId const current = sessionChanged - ? readMountedStructuredAgentSessionOutbox(sessionId, fence, readOutbox) + ? readMountedStructuredAgentSessionOutbox(sessionId, owner.fenceRef.current, readOutbox) : outboxRef.current - const next = current.map((entry) => - entry.state === 'dispatching' && !hasInFlightLaunchDispatch(entry, fence) - ? { ...entry, state: 'queued' as const } - : entry - ) + const next = requeueInterruptedStructuredAgentSessionDispatches(current, owner.fenceRef.current) if ( sessionChanged || next.some((entry, index) => entry !== current[index]) || @@ -90,7 +91,7 @@ export function useStructuredAgentSessionOutbox(args: { setOutbox(next) writeOutbox(sessionId, next) } - }, [fence, sessionId, target]) + }, [owner.fenceRef, owner.ownerChange, sessionId, target]) useEffect(() => { const current = outboxRef.current @@ -103,7 +104,7 @@ export function useStructuredAgentSessionOutbox(args: { .map((submission) => submission.clientMessageId) ) const next = reconcileStructuredAgentSessionOutbox(current, submissions) - const admittedInFlight = inFlightIdRef.current !== null && hostOwns.has(inFlightIdRef.current) + const admittedInFlight = journalAnswersInFlightSend(submissions, inFlightIdRef.current) if ( admittedInFlight || next.some((entry, index) => entry !== current[index]) || @@ -240,7 +241,7 @@ export function useStructuredAgentSessionOutbox(args: { const probeSettled = probeId !== null && submissions.some((submission) => submission.clientMessageId === probeId) useEffect(() => { - if (probeId === null || probeSettled || fence === null) { + if (probeId === null || probeSettled || !owner.attached) { return } const attempts = probeAttemptsRef.current.id === probeId ? probeAttemptsRef.current.attempts : 0 @@ -257,7 +258,7 @@ export function useStructuredAgentSessionOutbox(args: { Math.min(UNCONFIRMED_PROBE_BASE_DELAY_MS * 2 ** attempts, UNCONFIRMED_PROBE_MAX_DELAY_MS) ) return () => clearTimeout(timer) - }, [fence, probeId, probeSettled, sessionId, targetKey]) + }, [owner.attached, owner.ownerChange, owner.targetKey, probeId, probeSettled, sessionId]) const send = useCallback( (text: string, attachments: readonly { path: string; previewUri: string }[] = []): boolean => { diff --git a/src/renderer/src/runtime/structured-agent-session-accepted-send-capability.test.tsx b/src/renderer/src/runtime/structured-agent-session-accepted-send-capability.test.tsx new file mode 100644 index 00000000000..acdfdbb0323 --- /dev/null +++ b/src/renderer/src/runtime/structured-agent-session-accepted-send-capability.test.tsx @@ -0,0 +1,51 @@ +// @vitest-environment happy-dom + +import { renderHook, waitFor } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY } from '../../../shared/protocol-version' + +const mocks = vi.hoisted(() => ({ supports: vi.fn() })) + +vi.mock('./runtime-rpc-client', () => ({ + runtimeEnvironmentSupportsCapability: mocks.supports +})) + +import { setLocalRuntimeCapabilitiesForTests } from './local-runtime-capabilities' +import { useStructuredAgentSessionHostAcceptsSend } from './structured-agent-session-accepted-send-capability' + +afterEach(() => { + setLocalRuntimeCapabilitiesForTests(null) + vi.clearAllMocks() +}) + +describe('whether a host accepts a send before any agent has it', () => { + it('reads the local host from its advertised capabilities', () => { + setLocalRuntimeCapabilitiesForTests([AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY]) + const { result } = renderHook(() => useStructuredAgentSessionHostAcceptsSend({ kind: 'local' })) + expect(result.current).toBe(true) + }) + + it('treats a local host that does not advertise it as an older one', () => { + setLocalRuntimeCapabilitiesForTests([]) + const { result } = renderHook(() => useStructuredAgentSessionHostAcceptsSend({ kind: 'local' })) + expect(result.current).toBe(false) + }) + + it('asks a remote host, and reads a failed probe as an older host', async () => { + mocks.supports.mockResolvedValueOnce(true).mockRejectedValueOnce(new Error('offline')) + const accepts = renderHook(() => + useStructuredAgentSessionHostAcceptsSend({ kind: 'environment', environmentId: 'env-1' }) + ) + await waitFor(() => expect(accepts.result.current).toBe(true)) + expect(mocks.supports).toHaveBeenCalledWith( + 'env-1', + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY + ) + + const offline = renderHook(() => + useStructuredAgentSessionHostAcceptsSend({ kind: 'environment', environmentId: 'env-2' }) + ) + await waitFor(() => expect(mocks.supports).toHaveBeenCalledTimes(2)) + expect(offline.result.current).toBe(false) + }) +}) diff --git a/src/renderer/src/runtime/structured-agent-session-accepted-send-capability.ts b/src/renderer/src/runtime/structured-agent-session-accepted-send-capability.ts new file mode 100644 index 00000000000..c6e130b8bd5 --- /dev/null +++ b/src/renderer/src/runtime/structured-agent-session-accepted-send-capability.ts @@ -0,0 +1,87 @@ +import { useEffect, useLayoutEffect, useRef, useState, type RefObject } from 'react' +import { AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY } from '../../../shared/protocol-version' +import type { RuntimeClientTarget } from './runtime-client-target' +import { + ensureLocalRuntimeCapabilities, + readLocalRuntimeCapabilitiesOrUnknown +} from './local-runtime-capabilities' +import { runtimeEnvironmentSupportsCapability } from './runtime-rpc-client' + +function targetKey(target: RuntimeClientTarget): string { + return target.kind === 'local' ? 'local' : `environment:${target.environmentId}` +} + +/** + * Whether the host answers a send at acceptance and never refuses one because its agent could not + * start. Such a host records every send before it starts anything, so nothing about a moved fence + * calls for a resend. False until the host has said so: an older host is handled as it always was. + */ +export function useStructuredAgentSessionHostAcceptsSend(target: RuntimeClientTarget): boolean { + const key = targetKey(target) + const [answer, setAnswer] = useState<{ key: string; accepts: boolean }>(() => ({ + key, + accepts: + target.kind === 'local' && + (readLocalRuntimeCapabilitiesOrUnknown()?.includes( + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY + ) ?? + false) + })) + const environmentId = target.kind === 'environment' ? target.environmentId : null + useEffect(() => { + let cancelled = false + const probe = + environmentId === null + ? ensureLocalRuntimeCapabilities().then( + (capabilities) => + capabilities?.includes(AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY) ?? false + ) + : runtimeEnvironmentSupportsCapability( + environmentId, + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY + ) + void probe + .catch(() => false) + .then((accepts) => { + if (!cancelled) { + setAnswer((current) => + current.key === key && current.accepts === accepts ? current : { key, accepts } + ) + } + }) + return () => { + cancelled = true + } + }, [environmentId, key]) + return answer.key === key && answer.accepts +} + +/** + * When an outbox treats its owner as changed: resending a send in flight under the same id, + * dropping that send's answer, and unblocking a refused head. An older host restarts the agent + * inside the send and refuses it, unrecorded, when that fails, so a new fence is its only word that + * another try may land. A host that accepts first records every send before it starts anything, + * so a moved fence means nothing there, and only a Retry or a new send goes out. + */ +export function useStructuredAgentSessionOutboxOwnerChange( + target: RuntimeClientTarget, + fence: number | null +): { + ownerChange: number | null + attached: boolean + fenceRef: RefObject + targetKey: string +} { + const acceptsSend = useStructuredAgentSessionHostAcceptsSend(target) + // Read by effects that run on an owner change, not on every fence move. + const fenceRef = useRef(fence) + useLayoutEffect(() => { + fenceRef.current = fence + }, [fence]) + return { + ownerChange: acceptsSend ? null : fence, + attached: fence !== null, + fenceRef, + targetKey: targetKey(target) + } +} diff --git a/src/shared/agent-session-journal-schemas.ts b/src/shared/agent-session-journal-schemas.ts index b7a80634bf8..04f7183eb3f 100644 --- a/src/shared/agent-session-journal-schemas.ts +++ b/src/shared/agent-session-journal-schemas.ts @@ -304,7 +304,9 @@ export const AgentJournalSubmissionSchema = z.object({ reason: z.string().nullable(), submittedAt: z.number(), resolvedAt: z.number().nullable(), - recovered: z.literal(true).optional() + recovered: z.literal(true).optional(), + handoverRecorded: z.literal(true).optional(), + handedOverAt: z.number().optional() }) export function isAdmissibleAgentJournalItemBody(value: unknown): value is AgentJournalItemBody { diff --git a/src/shared/agent-session-journal-types.ts b/src/shared/agent-session-journal-types.ts index f5bd250c0c9..0a6cb655cd8 100644 --- a/src/shared/agent-session-journal-types.ts +++ b/src/shared/agent-session-journal-types.ts @@ -364,6 +364,13 @@ export type AgentJournalSubmission = { /** Set when crash reconciliation resolved the dispatch, not the provider. A live * `unknown` is a send still outstanding; a recovered one outlived its writer. */ recovered?: true + /** The host accepted this send to hand over later; absent on sends dispatched as they were + * recorded (older hosts). With no `handedOverAt` yet, a pending one is still queued. */ + handoverRecorded?: true + /** When the host handed it to the provider (its `dispatch{pending}` row). */ + handedOverAt?: number + /** Host-only: the submission row's sequence, which tells which host process accepted it. */ + acceptedSequence?: number } /** Durable answer to "did my send land?", keyed by client message id. Only an diff --git a/src/shared/agent-session-mutation-envelope.ts b/src/shared/agent-session-mutation-envelope.ts index b591d82af2e..dd409d406d1 100644 --- a/src/shared/agent-session-mutation-envelope.ts +++ b/src/shared/agent-session-mutation-envelope.ts @@ -92,6 +92,9 @@ export function admitAgentSessionMutation(input: { /** Decision from the durable ledger, evaluated under `hostFingerprint`. */ ledger: AgentSessionOperationDecision lease: AgentSessionLease + /** A write to the conversation, not to the provider child: a send is accepted and a Stop + * withdraws queued messages whoever owns the child, so the lease does not admit them. */ + conversationWrite?: true }): AgentSessionMutationAdmission { const { envelope, lease, ledger } = input const mismatch = agentSessionFingerprintConflict(envelope, input.hostFingerprint) @@ -110,6 +113,9 @@ export function admitAgentSessionMutation(input: { if (ledger.decision === 'replay') { return { decision: 'replay', row: ledger.row } } + if (input.conversationWrite) { + return { decision: 'admit', row: ledger.row } + } const leaseRefusal = refuseUnlessWriterAdmitted(lease) if (leaseRefusal) { return { decision: 'refused', refusal: leaseRefusal } diff --git a/src/shared/agent-session-queued-submission.ts b/src/shared/agent-session-queued-submission.ts new file mode 100644 index 00000000000..6295897fd54 --- /dev/null +++ b/src/shared/agent-session-queued-submission.ts @@ -0,0 +1,13 @@ +import type { AgentJournalSubmission } from './agent-session-journal-types' + +/** Accepted by the host and not yet handed to the provider: provably unwritten, so every way out + * of this state is delivery or a rejection, never doubt. */ +export function isQueuedAgentJournalSubmission( + submission: Pick +): boolean { + return ( + submission.handoverRecorded === true && + submission.dispatchState === 'pending' && + submission.handedOverAt === undefined + ) +} diff --git a/src/shared/agent-session-wire.ts b/src/shared/agent-session-wire.ts index 7118a2237eb..f5ceb625770 100644 --- a/src/shared/agent-session-wire.ts +++ b/src/shared/agent-session-wire.ts @@ -295,7 +295,7 @@ export type AgentSessionAttachResult = { sessionId: string fence: number page: AgentSessionHistoryPage - /** Submissions the crash boundary settled as `unknown` while attaching. */ + /** Submissions a crash boundary left `unknown` that provider history could not decide. */ unconfirmedClientMessageIds: string[] /** The host-owned id of the tab showing this chat, when it has one. Absent from older hosts. */ tabId?: string diff --git a/src/shared/protocol-version.ts b/src/shared/protocol-version.ts index db9d7f1b544..dd992e32ea5 100644 --- a/src/shared/protocol-version.ts +++ b/src/shared/protocol-version.ts @@ -165,6 +165,11 @@ export const STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY = 'agent-session.struct // clients skip the host's bounded best-effort settlement observation. export const AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY = 'agent-session.pending-send-result.v1' as const +// Why: a send is now answered once the host accepts it, before any agent has it. A client without +// this cannot show a message rejected after that answer, so the host holds its reply until the +// message is handed over or rejected. +export const AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY = + 'agent-session.accepted-send.v1' as const // Why: paired clients advertise Claude-structured support so the host can gate its agent-specific // journal and lifecycle surfaces independently from Codex support. export const CLAUDE_STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY = @@ -302,6 +307,7 @@ export const NATIVE_REMOTE_RUNTIME_CLIENT_CAPABILITIES = [ export const ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES = [ ...NATIVE_REMOTE_RUNTIME_CLIENT_CAPABILITIES, AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY, + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, BROWSER_CLIENT_HOST_RUNTIME_CAPABILITY, BROWSER_CLIENT_PAGE_METADATA_RUNTIME_CAPABILITY, // Why: only the renderer runs the retirement-proof ledger; CLI and mobile must keep full lists. @@ -372,6 +378,9 @@ export const RUNTIME_CAPABILITIES = [ AGENT_SESSION_KEYBOARD_RUNTIME_CAPABILITY, STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY, AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY, + // The host side: it accepts a send before any agent has it, and a Stop with no writer before a + // turn starts, so a client may gate on either. + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, STRUCTURED_AGENT_SESSION_HOLD_RUNTIME_CAPABILITY, STRUCTURED_AGENT_SESSION_REVEAL_RUNTIME_CAPABILITY, STRUCTURED_AGENT_SESSION_RESUME_HISTORY_RUNTIME_CAPABILITY, diff --git a/src/shared/structured-agent-session-dispatch-rejection.ts b/src/shared/structured-agent-session-dispatch-rejection.ts index 14e197325d8..a88dad5b16e 100644 --- a/src/shared/structured-agent-session-dispatch-rejection.ts +++ b/src/shared/structured-agent-session-dispatch-rejection.ts @@ -29,6 +29,12 @@ export const DISPATCH_REJECTED_CODEX_QUEUE_FULL = 'codex structured dispatch que /** The provider confirmed a queued frame was withdrawn before execution. */ export const DISPATCH_REJECTED_CANCELLED = 'provider_cancelled_before_start' +/** Accepted by a host process that ended before handing it to any provider. */ +export const DISPATCH_REJECTED_HOST_RESTARTED = 'host_restarted_before_delivery' + +/** Accepted, then the provider was closed before the message was handed to it. */ +export const DISPATCH_REJECTED_PROVIDER_CLOSED = 'provider_closed_before_delivery' + export function dispatchWriteFailureReason(error: unknown): string { const detail = error instanceof Error ? error.message : String(error) return `${DISPATCH_REJECTED_WRITE_FAILED}: ${detail}` @@ -54,6 +60,8 @@ export function dispatchRejectionReasonIsInternal(reason: string | null | undefi dispatchRejectionWasTransportWriteFailure(reason) || reason === DISPATCH_REJECTED_QUEUE_FULL || reason === DISPATCH_REJECTED_CODEX_QUEUE_FULL || - reason === DISPATCH_REJECTED_CANCELLED + reason === DISPATCH_REJECTED_CANCELLED || + reason === DISPATCH_REJECTED_HOST_RESTARTED || + reason === DISPATCH_REJECTED_PROVIDER_CLOSED ) } diff --git a/src/shared/structured-agent-session-outbox.ts b/src/shared/structured-agent-session-outbox.ts index ef5d4cfd7d7..6c820921162 100644 --- a/src/shared/structured-agent-session-outbox.ts +++ b/src/shared/structured-agent-session-outbox.ts @@ -11,7 +11,13 @@ import type { import { structuredAgentSessionPayloadFingerprint } from './structured-agent-session-mutation' import { DISPATCH_REJECTED_CANCELLED } from './structured-agent-session-dispatch-rejection' -export type StructuredAgentSessionOutboxState = 'queued' | 'dispatching' | 'unconfirmed' +/** `rejected`: the host settled the send as not delivered. The drain never sends it again on its + * own and nothing queues behind it; only the user's Retry does. */ +export type StructuredAgentSessionOutboxState = + | 'queued' + | 'dispatching' + | 'unconfirmed' + | 'rejected' export type StructuredAgentSessionOutboxEntry = { clientMessageId: string @@ -126,10 +132,12 @@ export function requeueStructuredAgentSessionSendRefusal( ) { return { ...entry, state: 'queued' } } + // Only here is the refusal proof the message never landed: an earlier attempt under this id, or + // one whose delivery was in doubt, may have, so those stay queued behind the block. return { ...entry, clientMessageId: createOperationId(), - state: 'queued', + state: 'rejected', lastAttemptAt: null, retryAfterUnknownSubmittedAt: null } @@ -154,6 +162,21 @@ export function reconcileStructuredAgentSessionOutbox( if (submission?.dispatchState === 'pending') { return entry.state === 'dispatching' ? [entry] : [{ ...entry, state: 'dispatching' as const }] } + // Accepted, then not delivered — the agent never started, or its start was refused. The text + // and why stay here for the user's Retry, and nothing queues behind it. `unconfirmed` is how a + // remount reads an entry it left dispatching; the journal has since answered it. + if ( + submission?.dispatchState === 'rejected' && + (entry.state === 'dispatching' || entry.state === 'unconfirmed') + ) { + return [ + { + ...entry, + state: 'rejected' as const, + lastFailure: { kind: 'rejected' as const, reason: submission.reason } + } + ] + } if ( submission?.dispatchState === 'unknown' && entry.retryAfterUnknownSubmittedAt !== -1 && @@ -185,6 +208,10 @@ export function admitStructuredAgentSessionOutboxEntry( blockedClientMessageId: string | null ): StructuredAgentSessionOutboxAdmission { for (const entry of entries) { + // It can no longer land, so nothing it could be reordered around; it waits for Retry. + if (entry.state === 'rejected') { + continue + } if (entry.state === 'unconfirmed' || entry.clientMessageId === blockedClientMessageId) { return { state: 'blocked', entry } } @@ -214,7 +241,7 @@ export function parseStructuredAgentSessionOutboxEntry( !Array.isArray(body.blocks) || !Array.isArray(entry.previewUris) || !entry.previewUris.every((uri) => typeof uri === 'string') || - !['queued', 'dispatching', 'unconfirmed'].includes(entry.state ?? '') + !['queued', 'dispatching', 'unconfirmed', 'rejected'].includes(entry.state ?? '') ) { return null } diff --git a/src/shared/structured-agent-session-send-disposition.ts b/src/shared/structured-agent-session-send-disposition.ts index 72409b8e928..11ba9703b83 100644 --- a/src/shared/structured-agent-session-send-disposition.ts +++ b/src/shared/structured-agent-session-send-disposition.ts @@ -7,6 +7,7 @@ // the refs, the React state and the storage write, and nothing else decides an // entry's state. +import type { AgentJournalSubmission } from './agent-session-journal-types' import type { AgentSessionMutationResult, AgentSessionSendResult } from './agent-session-wire' import { agentSessionRefusalFailure, @@ -92,6 +93,18 @@ function refusedRedelivery( ) } +/** Whether the journal already answers a send still in flight, so its own reply adds nothing: the + * host holds the message, or rejected it — a later `pending` reply must not undo that. */ +export function journalAnswersInFlightSend( + submissions: readonly AgentJournalSubmission[], + clientMessageId: string | null +): boolean { + return submissions.some( + (submission) => + submission.clientMessageId === clientMessageId && submission.dispatchState !== 'unknown' + ) +} + /** * What to put on screen for a rejection. * @@ -166,12 +179,17 @@ export function disposeStructuredAgentSessionSendResult( ) : candidate ) + const refused = entries[refusedIndex] return { entries, error: null, // Read back by index rather than from the input: a refusal can rotate the id, and the // refused entry is not always the head now that an admitted one no longer holds the queue. - blockedClientMessageId: entries[refusedIndex]?.clientMessageId ?? null, + // A rejected one holds nothing: it can no longer land, and it keeps its own Retry. + blockedClientMessageId: + !refused || refused.state === 'rejected' + ? input.blockedClientMessageId + : refused.clientMessageId, retryWithFreshClientMessageId: null } } @@ -194,12 +212,12 @@ export function disposeStructuredAgentSessionSendResult( } if (submission.dispatchState === 'rejected') { return { - entries: replaceEntryState(input, 'queued', { + entries: replaceEntryState(input, 'rejected', { kind: 'rejected', reason: submission.reason }), error: null, - blockedClientMessageId: input.entry.clientMessageId, + blockedClientMessageId: input.blockedClientMessageId, retryWithFreshClientMessageId: input.entry.clientMessageId } } diff --git a/src/shared/structured-agent-session-unanswered-dispatch.ts b/src/shared/structured-agent-session-unanswered-dispatch.ts index 36c70a357cb..002db36cb57 100644 --- a/src/shared/structured-agent-session-unanswered-dispatch.ts +++ b/src/shared/structured-agent-session-unanswered-dispatch.ts @@ -1,4 +1,5 @@ import type { AgentJournalSubmission } from './agent-session-journal-types' +import { isQueuedAgentJournalSubmission } from './agent-session-queued-submission' /** One send the provider has neither opened a turn for nor refused; the rule is explained on * `hasUnansweredStructuredAgentSessionDispatch`, which asks it of every send. */ @@ -6,6 +7,10 @@ export function isUnansweredStructuredAgentSessionDispatch( submission: AgentJournalSubmission, currentFence?: number | null ): boolean { + if (isQueuedAgentJournalSubmission(submission)) { + // Accepted and still owed to whichever child the host starts next, whatever the fence. + return true + } return ( (currentFence == null || submission.fence >= currentFence) && (submission.dispatchState === 'pending' || diff --git a/tests/e2e/codex-child-approval-activity-row.unit.test.ts b/tests/e2e/codex-child-approval-activity-row.unit.test.ts index b9210bf79e8..71bb0fddce1 100644 --- a/tests/e2e/codex-child-approval-activity-row.unit.test.ts +++ b/tests/e2e/codex-child-approval-activity-row.unit.test.ts @@ -143,7 +143,9 @@ async function openHost() { journalDir: join(root, SESSION) }) const feed = new StructuredAgentSessionStatusFeed({ - sessions: new Map([[SESSION, indexedStatusFeedSession({ journal, hasProviderChild: true })]]), + sessions: new Map([ + [SESSION, indexedStatusFeedSession({ journal, child: { phase: 'ready' } })] + ]), getRecord: () => null, now: () => 1, readBackgroundTasks: () => ({ state: 'monitoring', tasks: [] }) diff --git a/tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts b/tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts index b038a558517..193216e3c44 100644 --- a/tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts +++ b/tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts @@ -22,6 +22,7 @@ import { AgentSessionRecordStore } from '../../../src/main/runtime/agent-session import { RuntimeSubscriptionRegistry } from '../../../src/main/runtime/runtime-subscription-registry' import type { AgentSessionSubscribeEvent } from '../../../src/shared/agent-session-wire' import { + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY, AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY, AGENT_SESSION_QUESTION_ANSWERS_RUNTIME_CAPABILITY, AGENT_SESSION_REWIND_RUNTIME_CAPABILITY, @@ -619,6 +620,10 @@ describe('cross-version structured agent sessions', () => { let store: AgentSessionRecordStore let runtime: unknown + /** Holds a provider start open, so a reply's timing can be read against it. */ + let startGate: Promise = Promise.resolve() + let starts = 0 + /** Phase 2 owns provider processes; the adapter is the only stub here. */ function adapter(): StructuredAgentSessionAdapter { return { @@ -626,23 +631,27 @@ describe('cross-version structured agent sessions', () => { // `supportsLocation`, which this fake also lacks, so the client-supplied-location gate // refused for the fake's silence rather than for the location. supportsCreate: () => true, - acquire: async ({ fence }) => ({ - process: { - hostId: 'local', - pid: 4242, - processStartTimeMs: 1_700_000_000_000, - spawnToken: store.getRecord(SESSION)?.lease.reservedSpawnToken ?? 'spawn-a' - }, - link: { - linkId: `link-${fence}`, - handle: { provider: 'codex', threadId: THREAD }, - // A restarted host re-proves the thread it inherited; only the first - // owner of a session may claim to have created it. - origin: store.getRecord(SESSION)?.providerHandleChain.length ? 'resumed' : 'created', - mintedAtFence: fence, - observedAt: NOW + acquire: async ({ fence }) => { + starts += 1 + await startGate + return { + process: { + hostId: 'local', + pid: 4242, + processStartTimeMs: 1_700_000_000_000, + spawnToken: store.getRecord(SESSION)?.lease.reservedSpawnToken ?? 'spawn-a' + }, + link: { + linkId: `link-${fence}`, + handle: { provider: 'codex', threadId: THREAD }, + // A restarted host re-proves the thread it inherited; only the first + // owner of a session may claim to have created it. + origin: store.getRecord(SESSION)?.providerHandleChain.length ? 'resumed' : 'created', + mintedAtFence: fence, + observedAt: NOW + } } - }), + }, dispatch: async () => ({ state: 'accepted', providerIdentity: { provider: 'codex', threadId: THREAD, turnId: 'turn-1', ordinal: 1 } @@ -697,14 +706,18 @@ describe('cross-version structured agent sessions', () => { return reattached } - async function call(method: string, params: unknown): Promise { + async function call( + method: string, + params: unknown, + clientCapabilities: readonly string[] = [STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY] + ): Promise { return callBuild( current, method, params, { clientKind: 'runtime', - clientCapabilities: [STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY], + clientCapabilities, clientId: 'paired-device-1', connectionId: 'connection-1' }, @@ -723,6 +736,7 @@ describe('cross-version structured agent sessions', () => { beforeEach(async () => { resetOperationIds() + startGate = Promise.resolve() root = await mkdtemp(join(tmpdir(), 'orca-cross-version-agent-session-')) runtime = runtimeStub() await bootHost('a') @@ -776,5 +790,58 @@ describe('cross-version structured agent sessions', () => { fence: reattached.fence }) }) + + // A released client answers a send's `pending` as delivered-or-refused; it has no way to show + // a rejection that arrives after it. So it is answered once the message is handed over, while + // a client that advertises accepted sends is answered at acceptance, start or no start (W9). + it('holds the send reply of a released client until the handover, and answers a current one at once', async () => { + const released = [ + STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY, + AGENT_SESSION_PENDING_SEND_RESULT_RUNTIME_CAPABILITY + ] + expect(baseline.capabilities).not.toContain(AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY) + const created = await answer('agentSession.create', createIntentParams()) + await bootHost('b') + let open = (): void => undefined + startGate = new Promise((resolve) => (open = resolve)) + + let answered = false + const releasedReply = call( + 'agentSession.send', + sendParams('released', created.fence), + released + ).finally(() => (answered = true)) + // The start that delivers it is under way, and the reply still waits for it. + const before = starts + await vi.waitFor(() => expect(starts).toBeGreaterThan(before)) + await new Promise((resolve) => setTimeout(resolve, 20)) + expect(answered).toBe(false) + open() + const [reply] = await releasedReply + // Handed over, whatever the provider has said since. + expect(reply).toMatchObject({ + ok: true, + result: { value: { submission: { handedOverAt: expect.any(Number) } } } + }) + + const restarted = await bootHost('c') + startGate = new Promise((resolve) => (open = resolve)) + const currentReply = await call('agentSession.send', sendParams('current', created.fence), [ + ...released, + AGENT_SESSION_ACCEPTED_SEND_RUNTIME_CAPABILITY + ]) + expect(currentReply[0]).toMatchObject({ + ok: true, + result: { value: { submission: { dispatchState: 'pending', handoverRecorded: true } } } + }) + open() + await vi.waitFor(() => + expect( + restarted + .journalSnapshot(SESSION) + .submissions.every((row) => row.dispatchState !== 'pending' || row.handedOverAt) + ).toBe(true) + ) + }) }) })