Merge main into native chat SQLite storage

Carry retained Codex input intent, admission safety, account attribution and
idle confirmation through the SQL journal owners. Preserve main relocations
and deletions and the SQL queue, history and custody assertions.

Suppress publication of unchanged SQL command acknowledgments so empty queue
claims cannot renew a chat's idle clock. This SQL mismatch predates this sync;
main's new idle-retirement coverage exposed it. Mail custody fixtures now hold
native admission and use real edit leases without altering automatic delivery.

Validation: direct Node through tc-gate; merge hook, basic lint, tc:node,
e2e typecheck and pinned-main changed quality pass. All 148 owning tests in
25 files pass on both Node and configured Bun. O1 diagnostics remain excluded.
This commit is contained in:
Brennan Benson
2026-11-12 18:41:39 -08:00
227 changed files with 4207 additions and 3167 deletions
+2 -3
View File
@@ -221,8 +221,8 @@ jobs:
pnpm-lock.yaml
cloud/pnpm-lock.yaml
# These two tests import the cloud relay workspace directly. Keeping them in one job
# avoids installing and building the same workspace once per unit-test shard.
# This test imports the cloud relay workspace directly. Its dedicated job avoids
# installing and building the same workspace once per unit-test shard.
- name: Install relay integration dependencies
id: relay_dependencies
if: ${{ !cancelled() && steps.root_dependencies.outcome == 'success' }}
@@ -237,5 +237,4 @@ jobs:
ORCA_BACKGROUND_LAUNCH: '1'
run: >-
pnpm exec vitest run --config config/vitest.config.ts
tests/e2e/relay-region-compatibility.unit.test.ts
tests/e2e/relay-region-correction.unit.test.ts
+1 -1
View File
@@ -5,7 +5,7 @@ import { appendFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs
import { SignJWT, exportJWK, generateKeyPair, importJWK, type JWK } from 'jose'
import nacl from 'tweetnacl'
import WebSocket from 'ws'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
@@ -2,7 +2,7 @@
// client wrapper, the echo service, phone sockets and workerd memory sampling.
import { execSync } from 'node:child_process'
import WebSocket from 'ws'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
export const PORT = 8787
@@ -39,7 +39,7 @@ export async function runLifecycleChecks(ctx: Ctx, main: Control, mainAck: Ack):
mute.on('close', (c) => (muteClose = c))
await new Promise((res) => mute.on('open', res))
// Complete the handshake by hand with the shipped proof answerer, then never pong.
const { answerRelayHostChallenge } = await import('../../../../src/main/runtime/mobile-cloud-relay/relay-host-proof')
const { answerRelayHostChallenge } = await import('../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-host-proof')
mute.send(JSON.stringify({ type: 'host-hello', v: 1, relayHostId: ctx.relayHostId, assignmentEpoch: 7,
hostPublicKeyB64: ctx.keypair.publicKeyB64, appVersion: 'spike' }))
mute.on('message', (raw) => {
@@ -5,7 +5,7 @@ import { readFileSync } from 'node:fs'
import { SignJWT, type KeyLike } from 'jose'
import nacl from 'tweetnacl'
import WebSocket from 'ws'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
export const PORT = 8789
@@ -8,7 +8,7 @@ import { join } from 'node:path'
import { SignJWT, type KeyLike } from 'jose'
import nacl from 'tweetnacl'
import WebSocket from 'ws'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
import { RelayPhoneHelloSchema } from '../../../../src/shared/remote-runtime/relay/mobile-relay-phone-protocol'
+1 -1
View File
@@ -15,7 +15,7 @@ import { SignJWT, exportJWK, generateKeyPair, type KeyLike } from 'jose'
import nacl from 'tweetnacl'
import pg from 'pg'
import WebSocket from 'ws'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/relay-control-client'
import { RelayControlClient } from '../../../../src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client'
import { CloudRelayTransport } from '../../../../src/main/runtime/rpc/transports/relay-transport'
import { RelayPhoneHelloSchema } from '../../../../src/shared/remote-runtime/relay/mobile-relay-phone-protocol'
@@ -1,8 +0,0 @@
Exact relay contract snapshots from `027acb4efa2e6b226d40df266b86367423946d62`, `cloud/packages/relay-contract/src/`. Used to exercise the pre-correction strict wire parsers. Do not format or edit these baseline sources.
```text
aba94e108a5cd0f1af8b38875429ad8636d24c43a728273e3df60d9a1a1d1b6d director-messages.ts
bd13b5a694a5d683a5b680c14e46ab33f4ef4a5bfedf040d046b09d540cb4c17 wire-scalars.ts
bc89116f884a2f20a6588f9b91219aa596bc2410d28b499a93a78350def109d5 relay-regions.ts
8fcae470a5fc72f2fcdde9d2f09cd20289c256356dd490484ac1cfa53839fbe4 control-messages.ts
```
@@ -1,146 +0,0 @@
import { z } from 'zod'
import {
Base6432ByteSchema,
Base64Raw24ByteSchema,
Base64Url32ByteSchema,
EpochMsSchema,
GenerationSchema,
OpaqueIdSchema,
PositiveDurationMsSchema,
RelayHostIdSchema
} from './wire-scalars.js'
const AppVersionSchema = z.string().min(1).max(128)
const BoundedCiphertextSchema = z
.string()
.min(1)
.max(16 * 1024)
.regex(/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/)
const ConnectionKindSchema = z.enum(['invite', 'resume'])
export const HostHelloSchema = z
.object({
v: z.literal(1),
relayHostId: RelayHostIdSchema,
assignmentEpoch: GenerationSchema,
hostPublicKeyB64: Base6432ByteSchema,
appVersion: AppVersionSchema,
previousGeneration: GenerationSchema.optional(),
controlResumeSecret: Base64Url32ByteSchema.optional()
})
.strict()
export const HostChallengeSchema = z
.object({
challengeId: OpaqueIdSchema,
relayEphemeralPublicKeyB64: Base6432ByteSchema,
nonceB64: Base64Raw24ByteSchema,
ciphertextB64: BoundedCiphertextSchema,
expiresAt: EpochMsSchema
})
.strict()
export const HostChallengeAckSchema = z
.object({ challengeId: OpaqueIdSchema, proofB64: Base6432ByteSchema })
.strict()
// Advertised on the control upgrade rather than in host-hello: HostHelloSchema
// is strict, so a new hello key is refused by every already-deployed cell.
export const RELAY_HOST_CAPABILITIES_HEADER = 'x-orca-host-capabilities'
// The host accepts kind/relayDeviceId on a pendingConns entry. A host that does
// not advertise this parses those entries strictly and would drop the whole ack.
export const RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS = 'pending-conn-details'
export function parseRelayHostCapabilities(
header: string | string[] | undefined
): ReadonlySet<string> {
const raw = Array.isArray(header) ? header.join(',') : (header ?? '')
return new Set(
raw
.split(',')
.map((token) => token.trim())
.filter((token) => token.length > 0 && token.length <= 64)
.slice(0, 16)
)
}
// kind/relayDeviceId are optional so an entry stays readable by a host that
// predates them; the cell only emits them to a host that advertised support.
const PendingConnectionSchema = z
.object({
connId: OpaqueIdSchema,
connTicket: Base64Url32ByteSchema,
kind: ConnectionKindSchema.optional(),
relayDeviceId: OpaqueIdSchema.optional()
})
.strict()
export const HostHelloAckSchema = z
.object({
v: z.literal(1),
generation: GenerationSchema,
controlResumeSecret: Base64Url32ByteSchema,
leaseExpiresAt: EpochMsSchema,
activeConnIds: z.array(OpaqueIdSchema).max(8),
pendingConns: z.array(PendingConnectionSchema).max(8)
})
.strict()
export const ConnectionOpenSchema = z
.object({
connId: OpaqueIdSchema,
connTicket: Base64Url32ByteSchema,
kind: ConnectionKindSchema,
relayDeviceId: OpaqueIdSchema,
attachDeadlineMs: PositiveDurationMsSchema
})
.strict()
export const HostDataAuthSchema = z
.object({
v: z.literal(1),
connTicket: Base64Url32ByteSchema,
generation: GenerationSchema
})
.strict()
export const InviteCreateSchema = z
.object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema })
.strict()
export const InviteCreatedSchema = z
.object({
reqId: OpaqueIdSchema,
inviteToken: Base64Url32ByteSchema,
expiresAt: EpochMsSchema,
maxAttempts: z.number().int().positive().max(16)
})
.strict()
export const DeviceRevokeSchema = z
.object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema })
.strict()
export const AuthRefreshSchema = z.object({ relayJwt: z.string().min(1).max(8 * 1024) }).strict()
export const DrainSchema = z
.object({
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000),
recovery: z.literal('resolve-director')
})
.strict()
export const HeartbeatSchema = z.object({ t: EpochMsSchema }).strict()
export type HostHello = z.infer<typeof HostHelloSchema>
export type HostChallenge = z.infer<typeof HostChallengeSchema>
export type HostChallengeAck = z.infer<typeof HostChallengeAckSchema>
export type HostHelloAck = z.infer<typeof HostHelloAckSchema>
export type ConnectionOpen = z.infer<typeof ConnectionOpenSchema>
export type HostDataAuth = z.infer<typeof HostDataAuthSchema>
export type InviteCreate = z.infer<typeof InviteCreateSchema>
export type InviteCreated = z.infer<typeof InviteCreatedSchema>
export type DeviceRevoke = z.infer<typeof DeviceRevokeSchema>
export type AuthRefresh = z.infer<typeof AuthRefreshSchema>
export type Drain = z.infer<typeof DrainSchema>
export type Heartbeat = z.infer<typeof HeartbeatSchema>
@@ -1,75 +0,0 @@
import { z } from 'zod'
import {
Base64Url32ByteSchema,
CanonicalHttpsOriginSchema,
EpochMsSchema,
GenerationSchema,
RelayHostIdSchema
} from './wire-scalars.js'
import { RelayRegionSchema } from './relay-regions.js'
const SignedAssignmentLeaseSchema = z.string().min(1).max(8 * 1024)
export const AssignmentRequestSchema = z
.object({
v: z.literal(1),
relayHostId: RelayHostIdSchema,
// Client-declared reconnection; the director verifies it against the
// durable assignment before granting fast-lane admission.
reconnect: z.boolean().optional(),
preferredRegion: RelayRegionSchema.optional()
})
.strict()
export const AssignmentResponseSchema = z
.object({
v: z.literal(1),
cellUrl: CanonicalHttpsOriginSchema,
assignmentEpoch: GenerationSchema,
lease: SignedAssignmentLeaseSchema
})
.strict()
export const ResolveRequestSchema = z
.object({
v: z.literal(1),
relayHostId: RelayHostIdSchema,
resumeToken: Base64Url32ByteSchema
})
.strict()
export const ResolveResponseSchema = z
.object({
v: z.literal(1),
cellUrl: CanonicalHttpsOriginSchema,
assignmentEpoch: GenerationSchema,
leaseExpiresAt: EpochMsSchema
})
.strict()
export const RelayMovedSchema = z
.object({
v: z.literal(1),
cellUrl: CanonicalHttpsOriginSchema,
assignmentEpoch: GenerationSchema
})
.strict()
export function isTrustedNewerMove(input: {
sourceOrigin: string
configuredDirectorOrigin: string
currentAssignmentEpoch: number
move: z.infer<typeof RelayMovedSchema>
}): boolean {
// Why: cells and stale director responses must never redirect a credential-bearing client.
return (
input.sourceOrigin === input.configuredDirectorOrigin &&
input.move.assignmentEpoch > input.currentAssignmentEpoch
)
}
export type AssignmentRequest = z.infer<typeof AssignmentRequestSchema>
export type AssignmentResponse = z.infer<typeof AssignmentResponseSchema>
export type ResolveRequest = z.infer<typeof ResolveRequestSchema>
export type ResolveResponse = z.infer<typeof ResolveResponseSchema>
export type RelayMoved = z.infer<typeof RelayMovedSchema>
@@ -1,71 +0,0 @@
import { z } from 'zod'
export const RELAY_REGIONS = ['us-central1', 'asia-east2'] as const
export const RelayRegionSchema = z.enum(RELAY_REGIONS)
export type RelayRegion = z.infer<typeof RelayRegionSchema>
export const RELAY_DEFAULT_REGION: RelayRegion = 'us-central1'
// Field-name segment for the flat per-region runtime counters, spelled out rather than derived so
// the Terraform side can hold the same literal and a test can compare the two. `satisfies` makes a
// new region a compile error here, which is the point: a region with no segment would silently
// drop out of the region-skew alert's denominators.
export const RELAY_REGION_METRIC_SEGMENTS = {
'us-central1': 'UsCentral1',
'asia-east2': 'AsiaEast2'
} as const satisfies Record<RelayRegion, string>
const RelayProbeOriginSchema = z.string().url().max(2_048).refine(isCanonicalHttpsOrigin)
export const RelayRegionCatalogResponseSchema = z
.object({
v: z.literal(1),
regions: z
.array(
z
.object({
region: RelayRegionSchema,
probeOrigins: z.array(RelayProbeOriginSchema).min(1).max(2)
})
.strict()
)
.max(RELAY_REGIONS.length)
})
.strict()
.superRefine((catalog, context) => {
const regions = new Set<RelayRegion>()
const origins = new Set<string>()
for (const [regionIndex, entry] of catalog.regions.entries()) {
if (regions.has(entry.region)) {
context.addIssue({
code: 'custom',
message: 'duplicate relay region',
path: ['regions', regionIndex, 'region']
})
}
regions.add(entry.region)
for (const [originIndex, origin] of entry.probeOrigins.entries()) {
if (origins.has(origin)) {
context.addIssue({
code: 'custom',
message: 'duplicate relay probe origin',
path: ['regions', regionIndex, 'probeOrigins', originIndex]
})
}
origins.add(origin)
}
}
})
export type RelayRegionCatalogResponse = z.infer<typeof RelayRegionCatalogResponseSchema>
function isCanonicalHttpsOrigin(value: string): boolean {
try {
const url = new URL(value)
return url.protocol === 'https:' && url.origin === value
} catch {
return false
}
}
@@ -1,20 +0,0 @@
import { z } from 'zod'
export const Base64Url32ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{43}$/)
export const Base64Url24ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{32}$/)
export const Base6432ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){10}[A-Za-z0-9+/]{3}=$/)
export const Base64Raw24ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){8}$/)
export const RelayHostIdSchema = z.string().regex(/^[A-Za-z0-9_-]{16}$/)
export const OpaqueIdSchema = z.string().min(1).max(128)
export const EpochMsSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER)
export const GenerationSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER)
export const PositiveDurationMsSchema = z.number().int().positive().max(24 * 60 * 60 * 1000)
export const CanonicalHttpsOriginSchema = z.string().max(2048).refine((value) => {
try {
const url = new URL(value)
return url.protocol === 'https:' && url.origin === value && url.pathname === '/'
} catch {
return false
}
}, 'must be a canonical HTTPS origin')
+2 -2
View File
@@ -474,7 +474,7 @@
1 test-no-wall-clock:src/main/runtime/agent-session/lifecycle/structured-codex-child-work-runtime.test.ts
2 test-no-wall-clock:src/main/runtime/agent-session/structured-agent-session-held-signal.test.ts
1 test-no-wall-clock:src/main/runtime/agent-session/structured-child-records-switch.test.ts
1 test-no-wall-clock:src/main/runtime/mobile-cloud-relay/mobile-relay-e2ee.integration.test.ts
1 test-no-wall-clock:src/main/runtime/mobile-cloud-relay/testing/mobile-relay-e2ee.integration.test.ts
2 test-no-wall-clock:src/main/runtime/orchestration/mailbox/terminal-pointer/orchestration-codex-real-pty.integration.test.ts
1 test-no-wall-clock:src/main/runtime/push/push-device-registration-persistence.test.ts
1 test-no-wall-clock:src/main/runtime/push/push-registration-rpc.test.ts
@@ -613,7 +613,7 @@
1 test-timeout-needs-reason:src/main/providers/local-pty/local-pty-child-process-verdict.test.ts
2 test-timeout-needs-reason:src/main/providers/startup-line-typed-length.live-shell.test.ts
5 test-timeout-needs-reason:src/main/pty/process-retirement/node-pty-master-fd-retirement.test.ts
1 test-timeout-needs-reason:src/main/runtime/mobile-cloud-relay/mobile-relay-e2ee.integration.test.ts
1 test-timeout-needs-reason:src/main/runtime/mobile-cloud-relay/testing/mobile-relay-e2ee.integration.test.ts
1 test-timeout-needs-reason:src/main/runtime/rpc/server/orchestration-cli-subprocess.test.ts
2 test-timeout-needs-reason:src/main/runtime/rpc/server/remote-runtime-request-connection.integration.test.ts
1 test-timeout-needs-reason:src/main/runtime/session-construction/agent-session/remote-agent-session-host-authority.integration.test.ts
+2 -2
View File
@@ -479,8 +479,8 @@
1 src/main/runtime/client/navigation/desktop-chat-creation-history.ts
9 src/main/runtime/host-instance-identity.ts
4 src/main/runtime/hosts/platform-probe/host-descriptor.ts
5 src/main/runtime/mobile-cloud-relay/relay-region-preference.ts
2 src/main/runtime/mobile-cloud-relay/relay-revoke-outbox.ts
5 src/main/runtime/mobile-cloud-relay/region-selection/relay-region-preference.ts
2 src/main/runtime/mobile-cloud-relay/pairing/relay-revoke-outbox.ts
1 src/main/runtime/orchestration/db/runtime-orchestration-database-access.ts
2 src/main/runtime/push/push-unregister-outbox.ts
1 src/main/runtime/rpc/methods/mobile-delivery/bundled-mobile-web-bundle.ts
+1 -1
View File
@@ -49,7 +49,7 @@
1 src/main/rate-limits/service/service-polling.ts
1 src/main/runtime/agent-state-rules/agent-state-rules-live-update.ts
1 src/main/runtime/mobile-cloud-relay/desktop-relay-service.ts
3 src/main/runtime/mobile-cloud-relay/relay-control-liveness.ts
3 src/main/runtime/mobile-cloud-relay/control-channel/relay-control-liveness.ts
1 src/main/runtime/rpc/methods/orchestration/messaging/check-run.ts
1 src/main/runtime/rpc/mobile/mobile-socket-wiring.ts
1 src/main/runtime/rpc/transports/websocket/remote-runtime-server-heartbeat.ts
@@ -3000,19 +3000,17 @@
"src/main/runtime/workspaces/creation/managed-worktree-create-registry.test.ts": 4143,
"src/main/runtime/workspaces/creation/managed-worktree-create-telemetry.test.ts": 4152,
"src/main/runtime/workspaces/removal/missing-worktree-terminal-reconciliation.test.ts": 432,
"src/main/runtime/mobile-cloud-relay/desktop-relay-service-installer.test.ts": 48,
"src/main/runtime/mobile-cloud-relay/mobile-relay-e2ee.integration.test.ts": 408,
"src/main/runtime/mobile-cloud-relay/relay-auth-coordinator-readiness.test.ts": 580,
"src/main/runtime/mobile-cloud-relay/relay-auth-coordinator-recovery.test.ts": 161,
"src/main/runtime/mobile-cloud-relay/relay-auth-coordinator.test.ts": 1013,
"src/main/runtime/mobile-cloud-relay/relay-control-client.test.ts": 701,
"src/main/runtime/mobile-cloud-relay/testing/desktop-relay-service-installer.test.ts": 48,
"src/main/runtime/mobile-cloud-relay/testing/mobile-relay-e2ee.integration.test.ts": 408,
"src/main/runtime/mobile-cloud-relay/authentication/relay-auth-coordinator-readiness.test.ts": 580,
"src/main/runtime/mobile-cloud-relay/authentication/relay-auth-coordinator.test.ts": 1013,
"src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client.test.ts": 701,
"src/main/runtime/mobile-cloud-relay/relay-control-origin.test.ts": 147,
"src/main/runtime/mobile-cloud-relay/relay-host-proof.test.ts": 122,
"src/main/runtime/mobile-cloud-relay/relay-http-client.test.ts": 175,
"src/main/runtime/mobile-cloud-relay/relay-readiness-retention.test.ts": 196,
"src/main/runtime/mobile-cloud-relay/relay-region-preference.test.ts": 253,
"src/main/runtime/mobile-cloud-relay/relay-region-refresh.test.ts": 127,
"src/main/runtime/mobile-cloud-relay/relay-revoke-outbox.test.ts": 73,
"src/main/runtime/mobile-cloud-relay/control-channel/relay-host-proof.test.ts": 122,
"src/main/runtime/mobile-cloud-relay/director/relay-http-client.test.ts": 175,
"src/main/runtime/mobile-cloud-relay/authentication/relay-readiness-retention.test.ts": 196,
"src/main/runtime/mobile-cloud-relay/region-selection/relay-region-preference.test.ts": 253,
"src/main/runtime/mobile-cloud-relay/pairing/relay-revoke-outbox.test.ts": 73,
"src/main/runtime/mobile-cloud-relay/relay-session-broker.test.ts": 804,
"src/main/runtime/mobile-desktop-relay/mobile-desktop-relay.integration.test.ts": 11377,
"src/main/runtime/mobile-desktop-relay/mobile-desktop-relay.test.ts": 1185,
@@ -3509,7 +3507,6 @@
"src/main/runtime/rpc/server/runtime-rpc-browser-host-admission.test.ts": 4402,
"src/main/runtime/rpc/server/runtime-rpc-delegated-device-sync.test.ts": 4986,
"src/main/runtime/rpc/server/runtime-rpc-device-key-dispatch.test.ts": 3597,
"src/main/runtime/rpc/server/runtime-rpc-device-revocation.test.ts": 5350,
"src/main/runtime/rpc/server/runtime-rpc-local-disconnect-forwarding.test.ts": 5138,
"src/main/runtime/rpc/server/runtime-rpc-long-poll-transport.test.ts": 30838,
"src/main/runtime/rpc/server/runtime-rpc-metadata-lifecycle.test.ts": 4652,
@@ -6607,16 +6604,12 @@
"src/shared/plugins/declarative/testing/testing-editor-host.test.ts": 197,
"src/shared/plugins/frames/plugin-frame-document-session.test.ts": 404,
"src/shared/plugins/frames/plugin-frame-document.test.ts": 426,
"src/shared/plugins/frames/plugin-frame-facades.test.ts": 383,
"src/shared/plugins/frames/plugin-frame-guest.test.ts": 50,
"src/shared/plugins/frames/plugin-panel-call-admission.test.ts": 89,
"src/shared/plugins/frames/plugin-panel-message-budget.test.ts": 99,
"src/shared/plugins/frames/plugin-panel-shell.test.ts": 376,
"src/shared/plugins/images/plugin-image-format.test.ts": 104,
"src/shared/plugins/images/plugin-image-url.test.ts": 42,
"src/shared/plugins/installation/plugin-file-validation.test.ts": 363,
"src/shared/plugins/manifest/plugin-display-text.test.ts": 285,
"src/shared/plugins/manifest/plugin-manifest-source-module-floor.test.ts": 212,
"src/shared/plugins/manifest/plugin-manifest-v2-rules.test.ts": 239,
"src/shared/plugins/manifest/plugin-manifest.test.ts": 234,
"src/shared/plugins/view/plugin-editor-shot-acts.test.ts": 112,
@@ -45,7 +45,6 @@ export const UNIT_EXCLUDE = [
'src/shared/terminal/pty/ingress/pty-reply-echo-shapes.node-pty.test.ts',
'src/shared/startup-shell-portability.live-shell.test.ts',
'src/shared/posix-command-path-lookup.test.ts',
'tests/e2e/relay-region-compatibility.unit.test.ts',
'tests/e2e/relay-region-correction.unit.test.ts',
'src/main/browser/cookies/browser-google-cookie-clear.electron.test.ts',
// Electron probes with in-process deadlines; the package job runs them serially.
@@ -141,7 +141,7 @@ const NODE_RUNTIME_INCLUDE = [
'src/main/ipc/renderer-shutdown-checkpoint-sleep-records.test.ts',
'src/main/runtime/agent-session/records/structured-agent-session-startup-tab-restore.test.ts',
'src/shared/remote-runtime/control/remote-runtime-shared-control-connection.test.ts',
'src/main/runtime/mobile-cloud-relay/relay-control-client.test.ts',
'src/main/runtime/mobile-cloud-relay/control-channel/relay-control-client.test.ts',
'src/main/ipc/filesystem/watchers/filesystem-watcher-ignore-real.test.ts',
'src/main/ipc/runtime-environments/runtime-environment-preference.test.ts',
'src/renderer/src/components/emulator-pane/emulator-device-frame-visibility.test.tsx',
+3 -1
View File
@@ -37,7 +37,7 @@ orca computer click --app com.spotify.client --element-index 42 --json
Element indexes are scoped to the latest `get-app-state` result and may be **sparse**. In `--json` output, read the tree from `result.snapshot.treeText`. Do not invent indexes from `elementCount`. Refresh state after navigation, focus changes, scrolling, or any app re-render before reusing an index.
Action results return only the accessibility-tree lines that changed since the last state for that window (`~` changed, `+` added, `-` removed), marked by `snapshot.treeDiff`; pass `--full-tree` to get the whole tree. To find one control in a large window, `get-app-state --query <text>` prints only matching elements and their ancestors, and `--max-elements <n>` caps the output.
Action results return only the accessibility-tree lines that changed since the last state for that window (`~` changed, `+` added, `-` removed), marked by `snapshot.treeDiff`; pass `--full-tree` to get the whole tree. To find one control in a large window, `get-app-state --query <text>` prints only matching elements and their ancestors, and `--max-elements <n>` caps the output. A window larger than the snapshot limit shows `... N more not shown (snapshot limit)` where the tree was cut; `get-app-state --element-index <n>` reads only that element's subtree, so pass the cut-off children's parent to see them.
## Waiting for the app
@@ -87,6 +87,8 @@ orca computer perform-secondary-action --app <app> --element-index <i> --action
Prefer semantic actions (`click`, `set-value`, `perform-secondary-action`) over raw `type-text` or `press-key` — they target accessibility elements directly and survive focus changes that keyboard input doesn't.
`hotkey` and `press-key` refuse shortcuts that quit or close the app, log out, lock the screen, force-quit or end the session, such as `Cmd+Q`, `Shift+Cmd+Q`, `Ctrl+Cmd+Q` and `Option+Cmd+Esc` on macOS, `Alt+F4` and `Win+L` on Windows, and `Ctrl+Q`, `Alt+F4`, `Super+L` and `Ctrl+Alt+Delete` on Linux. The error code is `system_shortcut_blocked`. Pass `--allow-system-shortcut` to send one anyway.
On macOS, actions run in the background: the target app isn't brought forward, your frontmost app and key window stay put, and the pointer doesn't move, so you can keep working while an agent drives another window. Each action result reports `action.delivery` (`background` or `foreground`). Pass `--restore-window` only when an app ignores background input, such as drags in Chromium or Electron apps or games; it brings the app forward and uses the real pointer and keyboard. Popup menus and web `<select>` controls are set with `set-value` instead of a click. On Linux and Windows, clicks, scrolling, drags and keyboard input still bring the window forward.
When accessibility targeting fails, fall back carefully to coordinates:
@@ -43,14 +43,19 @@ export function MobileNativeChatUsageLimitStatus({
const { usageLimit, openAccounts } = useContext(MobileNativeChatNoticeContext)
const failure = readWholeAgentSessionFailureFact(notice.failure)
const resetsAt = failure?.kind === 'usageLimitReached' ? failure.usageLimit?.resetsAt : undefined
const resetPassed = useResetPassed(resetsAt)
const actions =
failure?.kind === 'usageLimitReached' &&
notice.usageLimitTurnItemId !== undefined &&
usageLimit?.turnItemId === notice.usageLimitTurnItemId
? usageLimit
: null
// The line names the reset Orca waits for; Continue still keys off the cut's own reset.
// Continue applies at the cut's own reset, or earlier once the host says the chat moved accounts.
const resetPassed = useResetPassed(
resetsAt !== undefined && actions?.continueAt !== undefined
? Math.min(resetsAt, actions.continueAt)
: resetsAt
)
// The line names the reset Orca waits for.
const shownAt = actions?.autoContinue ? (actions.continueAt ?? resetsAt) : resetsAt
const shownPassed = useResetPassed(shownAt)
if (failure?.kind !== 'usageLimitReached') {
+23 -12
View File
@@ -487,13 +487,19 @@ def render_accessibility_tree(root, root_path, compact_browser_tabs=False, geome
has_text = bool(item["name"] or item["automationId"] or item["value"])
return role in {"panel", "filler", "unknown", "section"} and not has_text and not meaningful_actions(item["actions"]) and summary is None and child_count <= 1
def walk_children(child_items, depth, path):
for position, (child_index, child) in enumerate(child_items):
if len(records) >= MAX_NODES or depth > MAX_DEPTH:
truncation["truncated"] = True
if depth > MAX_DEPTH:
truncation["maxDepthReached"] = True
# Marks where the walk stopped, so the agent can read that parent's subtree by index.
lines.append(("\t" * depth) + f"... {len(child_items) - position} more not shown (snapshot limit)")
return
walk(child, depth, path + [child_index])
def walk(node, depth, path):
nonlocal next_index
if len(records) >= MAX_NODES or depth > MAX_DEPTH:
truncation["truncated"] = True
if depth > MAX_DEPTH:
truncation["maxDepthReached"] = True
return
item = record(node, next_index, path)
role_key = (item["controlType"] or "").lower()
title = item["name"] or item["automationId"] or ""
@@ -505,8 +511,7 @@ def render_accessibility_tree(root, root_path, compact_browser_tabs=False, geome
if len(summary_values) >= 2 and is_plain_text_subtree(node):
generic_summary = " ".join(summary_values)
if should_elide(item, len(child_items), generic_summary):
for child_index, child in child_items:
walk(child, depth, path + [child_index])
walk_children(child_items, depth, path)
return
records.append(item)
next_index += 1
@@ -529,8 +534,7 @@ def render_accessibility_tree(root, root_path, compact_browser_tabs=False, geome
if generic_summary or suppressed:
return
child_line_start = len(lines)
for child_index, child in child_items:
walk(child, depth + 1, path + [child_index])
walk_children(child_items, depth + 1, path)
if compact_browser_tabs:
compact_rendered_browser_tabs(records, lines, child_line_start, depth + 1)
@@ -1203,17 +1207,23 @@ def window_json(app, index, window):
}
def make_snapshot(query, include_screenshot, window_id=None, window_index=None, restore=False):
def make_snapshot(query, include_screenshot, window_id=None, window_index=None, restore=False, root=None):
app = find_app(query)
window_index, window = choose_window(app, window_id, window_index)
if restore:
restore_window(app, window)
window_index, window = choose_window(app, window_id, window_index)
bounds = screen_rect(window)
tree_root, root_path = window, [window_index]
if root is not None:
tree_root = find_element(app, root)
if tree_root is None:
raise RuntimeError(f"stale element: element {root.get('index')} is gone; run get-app-state again and use a fresh element index")
root_path = list(root.get("runtimeId") or [])
geometry_nodes = {}
records, lines, truncation = render_accessibility_tree(
window,
[window_index],
tree_root,
root_path,
compact_browser_tabs=is_browser_app(query, app),
geometry_nodes=geometry_nodes,
)
@@ -1875,6 +1885,7 @@ def run_operation(operation):
operation.get("windowId"),
operation.get("windowIndex"),
bool(operation.get("restoreWindow")),
operation.get("root"),
),
}
@@ -699,6 +699,26 @@ class RuntimeRenderTest(unittest.TestCase):
indexes = [item["index"] for item in snapshot["elements"]]
self.assertEqual(len(indexes), len(set(indexes)))
def test_cut_tree_marks_hidden_children_and_root_reads_them(self):
items = [FakeAccessible("button", f"A{i}") for i in range(3)]
root = FakeAccessible("frame", "Main", children=[
FakeAccessible("list", "Items", children=items), FakeAccessible("button", "After"),
])
with SnapshotFixture(root), patch.object(runtime, "MAX_NODES", 4):
snapshot = runtime.make_snapshot("Fixture", False)
self.assertEqual(snapshot["treeLines"], [
"0 frame Main",
"\t1 list Items",
"\t\t2 button A0",
"\t\t3 button A1",
"\t\t... 1 more not shown (snapshot limit)",
"\t... 1 more not shown (snapshot limit)",
])
items_record = next(item for item in snapshot["elements"] if item["name"] == "Items")
subtree = runtime.run_operation({"tool": "get_app_state", "app": "Fixture", "noScreenshot": True, "root": items_record})["snapshot"]
self.assertEqual(subtree["treeLines"], ["0 list Items", "\t1 button A0", "\t2 button A1", "\t3 button A2"])
self.assertEqual(subtree["elements"][-1]["runtimeId"], [0, 0, 2])
@@ -143,6 +143,12 @@ final class ElementRecord {
}
}
/// A snapshot that read one element's subtree; its elements also keep the window's earlier ones.
struct SnapshotScope {
let rootIndex: Int
let walkedIndexes: Set<Int>
}
struct Snapshot {
let id: String
let app: AppDescriptor
@@ -161,6 +167,7 @@ struct Snapshot {
let nextFreshElementIndex: Int
let truncated: Bool
let maxDepthReached: Bool
let scope: SnapshotScope?
/// On-screen windows of the app (menus and sheets included), so a new or closed window counts as a change.
let appWindowIds: [CGWindowID]
@@ -187,6 +194,7 @@ struct Snapshot {
nextFreshElementIndex: nextFreshElementIndex,
truncated: truncated,
maxDepthReached: maxDepthReached,
scope: scope,
appWindowIds: appWindowIds
)
}
@@ -262,9 +270,10 @@ final class Provider {
var action = try runAction()
// Why: only --restore-window activates the app or posts to the HID tap; every other route leaves the user's desktop alone.
action["delivery"] = isForegroundRequest(params) ? "foreground" : "background"
// Why: the action's currentSnapshot just cached this target's tree from before the action.
// Why: the action's currentSnapshot just cached this target's tree from before the action;
// a subtree read can't be compared with the whole-window read that follows.
let before = action["path"] as? String == "accessibility" && action["verification"] == nil
? try? cachedSnapshot(params: params)
? (try? cachedSnapshot(params: params)).flatMap { $0.scope == nil ? $0 : nil }
: nil
do {
let after = try observe(params: params)
@@ -315,12 +324,14 @@ final class Provider {
}
let namespace = snapshotNamespace(params)
pruneSnapshotCache()
let subtree = try requestedSubtree(params: params, app: app)
let snapshot = try buildSnapshot(
app: app,
includeScreenshot: params["noScreenshot"]?.bool != true,
windowId: windowId,
windowIndex: windowIndex,
windowId: subtree.map(\.base.windowId) ?? windowId,
windowIndex: subtree == nil ? windowIndex : nil,
restoreWindow: params["restoreWindow"]?.bool == true,
subtree: subtree,
previousSnapshot: { [snapshots] resolvedWindowId in
// Why: indexes carry over from this caller's own last view of exactly this window.
snapshots[namespacedSnapshotKey(namespace, snapshotCanonicalWindowIdKey(resolvedWindowId))]
@@ -339,6 +350,15 @@ final class Provider {
return snapshot
}
/// Resolves `rootElementIndex` against this caller's last snapshot of the window, whose indexes it extends.
private func requestedSubtree(params: [String: JSONValue], app: AppDescriptor) throws -> (root: ElementRecord, base: Snapshot)? {
guard let rootIndex = try optionalInteger(params, "rootElementIndex") else { return nil }
guard let base = try cachedSnapshot(params: params), base.app.pid == app.pid else {
throw ProviderError.coded("element_not_found", "--element-index needs a get-app-state snapshot of this app/window first")
}
return (try element(base, rootIndex), base)
}
private func rememberSnapshot(
query: String,
app: AppDescriptor,
@@ -416,19 +436,30 @@ final class Provider {
}
}
private func currentSnapshot(params: [String: JSONValue]) throws -> Snapshot {
private func currentSnapshot(params: [String: JSONValue], needsFocus: Bool = false) throws -> Snapshot {
pruneSnapshotCache()
let cached = try cachedSnapshot(params: params)
let requestedIndexes = try requestedElementIndexes(params)
let observeParams = params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement }
// Why: elements read with --element-index, focused ones included, may lie past the whole-window node limit,
// so only their subtree can re-find them.
if let scope = cached?.scope,
requestedIndexes.isEmpty ? needsFocus : requestedIndexes.allSatisfy(scope.walkedIndexes.contains),
let scoped = try? observe(params: observeParams.merging(["rootElementIndex": .number(Double(scope.rootIndex))]) { _, replacement in replacement }),
!requestedIndexes.isEmpty || scoped.focusedElementId != nil {
try validateRequestedElements(cached: cached, current: scoped, requestedIndexes: requestedIndexes)
return scoped
}
// Why: cached AX frames can be stale after a window move or resize, and
// stale geometry can turn an intended action into a misclick.
let snapshot = try observe(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement })
try validateRequestedElements(cached: cached, current: snapshot, params: params)
let snapshot = try observe(params: observeParams)
try validateRequestedElements(cached: cached, current: snapshot, requestedIndexes: requestedIndexes)
return snapshot
}
private func currentKeyboardSnapshot(params: [String: JSONValue]) throws -> Snapshot {
// Why: keyboard actions need fresh focused-element state, never pixels.
try currentSnapshot(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement })
try currentSnapshot(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement }, needsFocus: true)
}
private func cachedSnapshot(params: [String: JSONValue]) throws -> Snapshot? {
@@ -473,17 +504,24 @@ final class Provider {
(isExplicitSnapshotNamespace(namespace) ? nil : snapshots[key])
}
private func validateRequestedElements(cached: Snapshot?, current: Snapshot, params: [String: JSONValue]) throws {
let requestedIndexes = try ["elementIndex", "fromElementIndex", "toElementIndex"].compactMap { key -> Int? in
private func requestedElementIndexes(_ params: [String: JSONValue]) throws -> [Int] {
try ["elementIndex", "fromElementIndex", "toElementIndex"].compactMap { key -> Int? in
guard params[key]?.number != nil else { return nil }
return try optionalInteger(params, key)
}
}
private func validateRequestedElements(cached: Snapshot?, current: Snapshot, requestedIndexes: [Int]) throws {
guard !requestedIndexes.isEmpty else { return }
guard let cached else {
throw ProviderError.coded("element_not_found", "element indexes require a fresh get-app-state snapshot for this app/window")
}
for index in requestedIndexes {
guard let expected = cached.elements[index], let actual = current.elements[index] else {
// Why: a subtree read also carries the window's earlier records, which prove nothing about the element now.
guard let expected = cached.elements[index],
let actual = current.elements[index],
current.scope?.walkedIndexes.contains(index) ?? true
else {
throw ProviderError.coded("element_not_found", "element \(index) is stale; run get-app-state again and use a fresh element index")
}
guard expected.signature == actual.signature else {
@@ -658,6 +696,7 @@ final class Provider {
windowId: CGWindowID?,
windowIndex: Int?,
restoreWindow: Bool,
subtree: (root: ElementRecord, base: Snapshot)?,
previousSnapshot: (CGWindowID) -> Snapshot?
) throws -> Snapshot {
guard accessibilityTrustedSettled() else {
@@ -703,11 +742,29 @@ final class Provider {
focused: focusedElement(appElement: appElement),
compactBrowserTabs: app.isKnownBrowser
)
renderer.render(window)
if let subtree {
guard let ancestors = ancestorChain(of: subtree.root.element, in: window) else {
throw ProviderError.coded("element_not_found", "element \(subtree.root.index) is no longer in this window; run get-app-state again and use a fresh element index")
}
renderer.render(subtree.root.element, ancestors: ancestors)
} else {
renderer.render(window)
}
if renderer.appStoppedResponding {
throw AccessibilityMessaging.appNotResponding(app.name)
}
let indexes = renderer.carryOverIndexes(from: previousSnapshot(capture.windowId))
let previous = subtree?.base ?? previousSnapshot(capture.windowId)
let indexes = renderer.carryOverIndexes(from: previous)
var elements = renderer.records
var elementOrder = indexes.order
if let subtree {
// Why: elements outside the subtree keep resolving, so the agent can still act on what the whole-window read showed.
// Re-reading the same subtree replaces its last walk, so rows scrolled away don't pile up.
let replaced = subtree.base.scope.flatMap { $0.rootIndex == subtree.root.index ? $0.walkedIndexes : nil } ?? []
let kept = subtree.base.elements.filter { !replaced.contains($0.key) && renderer.records[$0.key] == nil }
elements = kept.merging(renderer.records) { _, walked in walked }
elementOrder = subtree.base.elementOrder.filter { kept[$0] != nil } + indexes.order
}
let screenshot = includeScreenshot ? capture.screenshotPayload() : nil
let screenshotStatus: ScreenshotStatus = if screenshot != nil {
.captured
@@ -725,17 +782,18 @@ final class Provider {
windowBounds: capture.bounds,
windowId: capture.windowId,
windowLayer: capture.layer,
treeText: renderTreeText(app: app, title: title, bounds: capture.bounds, lines: renderer.lines, focused: renderer.focusedSummary),
treeText: renderTreeText(app: app, title: title, rootIndex: subtree?.root.index, lines: renderer.lines, focused: renderer.focusedSummary),
focusedElementId: renderer.focusedElementId,
screenshot: screenshot,
screenshotStatus: screenshotStatus,
screenshotScale: screenshotScale(screenshot: screenshot, bounds: capture.bounds),
screenshotEngine: capture.image?.engine,
elements: renderer.records,
elementOrder: indexes.order,
elements: elements,
elementOrder: elementOrder,
nextFreshElementIndex: indexes.nextFreshIndex,
truncated: renderer.truncated,
maxDepthReached: renderer.maxDepthReached,
scope: subtree.map { SnapshotScope(rootIndex: $0.root.index, walkedIndexes: Set(indexes.order)) },
appWindowIds: windowCandidates.map(\.windowId).sorted()
)
}
@@ -751,39 +809,43 @@ final class Provider {
"scale": payload.scale,
]
}
return [
"snapshot": [
"id": snapshot.id,
"app": [
"name": snapshot.app.name,
"bundleId": jsonNullable(snapshot.app.bundleId),
"pid": Int(snapshot.app.pid),
],
"window": [
"id": Int(snapshot.windowId),
"title": snapshot.windowTitle,
"x": Int(snapshot.windowBounds.origin.x.rounded()),
"y": Int(snapshot.windowBounds.origin.y.rounded()),
"width": Int(snapshot.windowBounds.width.rounded()),
"height": Int(snapshot.windowBounds.height.rounded()),
"isMinimized": false,
"isOffscreen": false,
"screenIndex": jsonNullable(screenIndex(for: snapshot.windowBounds)),
"platform": [
"layer": snapshot.windowLayer,
],
],
"coordinateSpace": "window",
"treeText": snapshot.treeText,
"elementCount": snapshot.elements.count,
"focusedElementId": snapshot.focusedElementId as Any,
"truncation": [
"truncated": snapshot.truncated,
"maxNodes": TreeRenderer.maxNodes,
"maxDepth": TreeRenderer.maxDepth,
"maxDepthReached": snapshot.maxDepthReached,
var snapshotPayload: [String: Any] = [
"id": snapshot.id,
"app": [
"name": snapshot.app.name,
"bundleId": jsonNullable(snapshot.app.bundleId),
"pid": Int(snapshot.app.pid),
],
"window": [
"id": Int(snapshot.windowId),
"title": snapshot.windowTitle,
"x": Int(snapshot.windowBounds.origin.x.rounded()),
"y": Int(snapshot.windowBounds.origin.y.rounded()),
"width": Int(snapshot.windowBounds.width.rounded()),
"height": Int(snapshot.windowBounds.height.rounded()),
"isMinimized": false,
"isOffscreen": false,
"screenIndex": jsonNullable(screenIndex(for: snapshot.windowBounds)),
"platform": [
"layer": snapshot.windowLayer,
],
],
"coordinateSpace": "window",
"treeText": snapshot.treeText,
"elementCount": snapshot.scope?.walkedIndexes.count ?? snapshot.elements.count,
"focusedElementId": snapshot.focusedElementId as Any,
"truncation": [
"truncated": snapshot.truncated,
"maxNodes": TreeRenderer.maxNodes,
"maxDepth": TreeRenderer.maxDepth,
"maxDepthReached": snapshot.maxDepthReached,
],
]
if let scope = snapshot.scope {
snapshotPayload["rootElementIndex"] = scope.rootIndex
}
return [
"snapshot": snapshotPayload,
"screenshot": screenshot,
"screenshotStatus": renderScreenshotStatus(snapshot.screenshotStatus, snapshot: snapshot),
]
@@ -985,14 +1047,16 @@ final class Provider {
private func typeText(params: [String: JSONValue]) throws -> [String: Any] {
let snapshot = try currentKeyboardSnapshot(params: params)
let text = try requiredString(params, "text")
if let focused = focusedRecord(snapshot), let verification = try TextInput.replaceSelection(focused.element, with: text) {
let focused = focusedRecord(snapshot)?.element
if let focused, let verification = try TextInput.replaceSelection(focused, with: text) {
return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification)
}
let insertion = TextInput.Insertion(into: focused, text: text)
try deliverKeys(try withTransportErrors { try BackgroundInputTransport.textEvents(text) }, snapshot: snapshot, params: params)
return actionMetadata(
path: "synthetic",
actionName: "typeText",
verification: unverifiedAction(reason: "synthetic_input")
verification: insertion?.verification() ?? unverifiedAction(reason: "synthetic_input")
)
}
@@ -1027,17 +1091,19 @@ final class Provider {
private func pasteText(params: [String: JSONValue]) throws -> [String: Any] {
let snapshot = try currentKeyboardSnapshot(params: params)
let text = try requiredString(params, "text")
if let focused = focusedRecord(snapshot), let verification = try TextInput.replaceSelection(focused.element, with: text) {
let focused = focusedRecord(snapshot)?.element
if let focused, let verification = try TextInput.replaceSelection(focused, with: text) {
return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification)
}
let pasteKeys = try Input.keyEvents("cmd+v")
let insertion = TextInput.Insertion(into: focused, text: text)
try Input.withPasteboardText(text) {
try deliverKeys(pasteKeys, snapshot: snapshot, params: params)
}
return actionMetadata(
path: "clipboard",
actionName: "paste",
verification: unverifiedAction(reason: "clipboard_paste")
verification: insertion?.verification() ?? unverifiedAction(reason: "clipboard_paste")
)
}
@@ -1462,6 +1528,18 @@ private func pointerRecipient(
return .focused(syntheticClickRecipient(pid: ownerPID, windowId: candidate.windowId))
}
/// The element's parents from `window` down, or nil when it is no longer inside `window`.
private func ancestorChain(of element: AXUIElement, in window: AXUIElement) -> [AXUIElement]? {
var chain: [AXUIElement] = []
var current = element
while !CFEqual(current, window) {
guard chain.count < 256, let parent = copyElement(current, kAXParentAttribute as String) else { return nil }
chain.append(parent)
current = parent
}
return chain.reversed()
}
private func containingWindow(_ element: AXUIElement) -> AXUIElement? {
var current = element
for _ in 0..<64 {
@@ -2179,15 +2257,18 @@ private func mouseButton(_ raw: String?) throws -> MouseButtonSelection {
}
}
private func renderTreeText(app: AppDescriptor, title: String, bounds: CGRect, lines: [String], focused: String?) -> String {
private func renderTreeText(app: AppDescriptor, title: String, rootIndex: Int?, lines: [String], focused: String?) -> String {
var output = [
"App=\(app.bundleId ?? app.name.replacingOccurrences(of: " ", with: "_")) (pid \(app.pid))",
"Window: \"\(sanitize(title))\", App: \(sanitize(app.name)).",
"",
]
if let rootIndex {
output.append("Showing element \(rootIndex) and its descendants.")
}
output.append("")
output.append(contentsOf: lines)
output.append("")
output.append(focused.map { "The focused UI element is \($0)." } ?? "No UI element is currently focused.")
output.append(focused.map { "The focused UI element is \($0)." } ?? (rootIndex == nil ? "No UI element is currently focused." : "No UI element inside this one is focused."))
return output.joined(separator: "\n")
}
@@ -2231,15 +2312,6 @@ private final class TreeRenderer {
func render(_ element: AXUIElement, depth: Int = 0, ancestors: [AXUIElement] = []) {
guard !reader.appStoppedResponding else { return }
guard nextIndex < Self.maxNodes else {
truncated = true
return
}
guard depth < Self.maxDepth else {
truncated = true
maxDepthReached = true
return
}
guard !ancestors.contains(where: { CFEqual($0, element) }) else { return }
let role = reader.stringAttribute(element, kAXRoleAttribute as String) ?? "AXUnknown"
@@ -2290,9 +2362,7 @@ private final class TreeRenderer {
webAreaDepth: webAreaDepth
)
if SnapshotRenderHeuristics.shouldElide(node) {
for child in children {
render(child, depth: depth, ancestors: ancestors + [element])
}
renderChildren(children, depth: depth, ancestors: ancestors + [element])
return
}
@@ -2315,9 +2385,8 @@ private final class TreeRenderer {
return
}
if compactBrowserTabs, let tabStripCompaction = tabStripCompaction(parent: node, children: children) {
for (childIndex, child) in children.enumerated() where tabStripCompaction.retainedIndexes.contains(childIndex) {
render(child, depth: depth + 1, ancestors: ancestors + [element])
}
let retained = children.enumerated().filter { tabStripCompaction.retainedIndexes.contains($0.offset) }.map(\.element)
renderChildren(retained, depth: depth + 1, ancestors: ancestors + [element])
lines.append(
String(repeating: "\t", count: depth + 1) +
"... \(tabStripCompaction.omittedCount) inactive browser tabs omitted"
@@ -2325,14 +2394,26 @@ private final class TreeRenderer {
return
}
let childLineStart = lines.count
for child in children {
render(child, depth: depth + 1, ancestors: ancestors + [element])
}
renderChildren(children, depth: depth + 1, ancestors: ancestors + [element])
if compactBrowserTabs {
compactRenderedBrowserTabs(parent: node, startLine: childLineStart, depth: depth + 1)
}
}
/// Stops at the node or depth limit with a note of how many children were left, so the agent can read that subtree.
private func renderChildren(_ children: [AXUIElement], depth: Int, ancestors: [AXUIElement]) {
for (position, child) in children.enumerated() {
// Why: browser-tab compaction drops records, so the limit counts what is kept, as on Linux and Windows.
guard records.count < Self.maxNodes, depth < Self.maxDepth else {
truncated = true
maxDepthReached = maxDepthReached || depth >= Self.maxDepth
lines.append(String(repeating: "\t", count: depth) + "... \(children.count - position) more not shown (snapshot limit)")
return
}
render(child, depth: depth, ancestors: ancestors)
}
}
private func tabStripCompaction(parent: SnapshotRenderNode, children: [AXUIElement]) -> SnapshotTabStripCompaction? {
let childNodes = children.map { child in
let role = reader.stringAttribute(child, kAXRoleAttribute as String) ?? "AXUnknown"
@@ -3219,6 +3300,49 @@ private enum TextInput {
return verifiedAction(property: "focusedText")
}
/// Text sent as keys or a paste: what the focused field should read once it lands.
struct Insertion {
let element: AXUIElement
let expected: String
/// Nil when the field can't prove the result: none focused, a secure field, no readable value and selection, or text a readback can't confirm.
init?(into element: AXUIElement?, text: String) {
guard let element,
!AXSnapshotReader().isSecureTextElement(element),
let current = rawStringAttribute(element, kAXValueAttribute as String),
// Why: settle re-reads the whole value every 25 ms; a terminal's scrollback or a long document is too costly.
current.utf16.count <= 64 * 1024
else {
return nil
}
let selection = TextInput.selectedTextRange(element).flatMap { range -> Range<Int>? in
range.location >= 0 && range.length >= 0 && range.length <= Int.max - range.location
? range.location..<(range.location + range.length) : nil
}
guard let expected = TextInsertionReadback.expectedValue(current: current, selection: selection, inserting: text) else {
return nil
}
self.element = element
self.expected = expected
}
/// Nil when the readback can't tell: the field stopped answering or was still changing at the cap.
func verification() -> [String: Any]? {
let outcome = TextInsertionReadback.settle(expected: expected) {
rawStringAttribute(element, kAXValueAttribute as String)
}
switch outcome {
case .landed:
return verifiedAction(property: "focusedText")
case .mismatch:
// Why: the field may hold part of the text, so the result never shows what it reads.
return unverifiedAction(reason: "text_mismatch")
case .undetermined:
return nil
}
}
}
static func selectAll(_ element: AXUIElement) -> Bool {
guard let current = rawStringAttribute(element, kAXValueAttribute as String) else {
return false
@@ -0,0 +1,81 @@
import Foundation
/// Checks that text sent as keys or a paste landed: predict the field's value, then read it back.
public enum TextInsertionReadback {
public enum Outcome: Equatable {
case landed
case mismatch
/// The field stopped answering, or was still changing when the wait ran out.
case undetermined
}
/// The value a field should hold once `text` replaces its selection. `selection` is in UTF-16 offsets, as
/// AXSelectedTextRange reports it. Nil means a readback can't prove the result: a non-empty field with no
/// selection, a range outside the value or inside a character, text with control characters (Return or Tab
/// can submit, end editing or move focus instead of being stored), or a value the field already holds.
public static func expectedValue(current: String, selection: Range<Int>?, inserting text: String) -> String? {
guard !text.unicodeScalars.contains(where: { CharacterSet.controlCharacters.contains($0) }) else {
return nil
}
let units = Array(current.utf16)
let range: Range<Int>
if let selection {
guard selection.lowerBound >= 0, selection.upperBound <= units.count else {
return nil
}
range = selection
} else if units.isEmpty {
range = 0..<0
} else {
return nil
}
let prefix = Array(units[..<range.lowerBound])
let suffix = Array(units[range.upperBound...])
let before = String(decoding: prefix, as: UTF16.self)
let after = String(decoding: suffix, as: UTF16.self)
// Why: a boundary inside a surrogate pair decodes to U+FFFD, which no real field would read back.
guard Array(before.utf16) == prefix, Array(after.utf16) == suffix else {
return nil
}
let expected = before + text + after
return sameText(expected, current) ? nil : expected
}
/// Reads the field until it holds `expected`. Keys reach it through the app's event queue, so it can still be
/// filling after the last key is posted: each change restarts the quiet window, up to `capSeconds` in all.
public static func settle(
expected: String,
quietSeconds: Double = 0.5,
capSeconds: Double = 2,
intervalSeconds: Double = 0.025,
now: () -> Double = CFAbsoluteTimeGetCurrent,
sleep: (Double) -> Void = { Thread.sleep(forTimeInterval: $0) },
read: () -> String?
) -> Outcome {
let start = now()
var readback = read()
// Why: an AX read can block for seconds; the quiet window counts from the answer, not the ask.
var quietDeadline = now() + quietSeconds
while let current = readback, !sameText(current, expected) {
let time = now()
// Why: past the cap a field may still be filling or a read may have blocked through the quiet window; calling it a mismatch invites typing the rest twice.
guard time < start + capSeconds else {
return .undetermined
}
guard time < quietDeadline else {
return .mismatch
}
sleep(intervalSeconds)
readback = read()
if readback != current {
quietDeadline = now() + quietSeconds
}
}
return readback == nil ? .undetermined : .landed
}
// Why: Chromium contenteditables store typed trailing or repeated spaces as no-break spaces.
private static func sameText(_ lhs: String, _ rhs: String) -> Bool {
lhs.replacingOccurrences(of: "\u{00A0}", with: " ") == rhs.replacingOccurrences(of: "\u{00A0}", with: " ")
}
}
@@ -0,0 +1,86 @@
import XCTest
@testable import OrcaComputerUseMacOSCore
final class TextInsertionReadbackTests: XCTestCase {
func testInsertionReplacesTheSelectionInUTF16Offsets() {
XCTAssertEqual(
TextInsertionReadback.expectedValue(current: "a😀bcd", selection: 3..<5, inserting: "XY"),
"a😀XYd"
)
XCTAssertEqual(TextInsertionReadback.expectedValue(current: "", selection: nil, inserting: "hi"), "hi")
}
func testUnpredictableInsertionsReturnNil() {
XCTAssertNil(TextInsertionReadback.expectedValue(current: "abc", selection: nil, inserting: "d"))
XCTAssertNil(TextInsertionReadback.expectedValue(current: "abc", selection: 2..<4, inserting: "d"))
XCTAssertNil(TextInsertionReadback.expectedValue(current: "a😀", selection: 2..<2, inserting: "d"))
XCTAssertNil(TextInsertionReadback.expectedValue(current: "", selection: 0..<0, inserting: "send\n"))
XCTAssertNil(TextInsertionReadback.expectedValue(current: "", selection: 0..<0, inserting: "a\tb"))
XCTAssertNil(TextInsertionReadback.expectedValue(current: "hello", selection: 0..<5, inserting: "hello"))
}
func testFieldStillFillingPastTheQuietWindowLands() {
// One character every 0.4 s: each read is past the first 0.5 s window, but each change restarts it.
let reads = ["", "h", "he", "hel", "hell", "hello"]
XCTAssertEqual(settle(expected: "hello", reads: reads, secondsPerRead: 0.4), .landed)
}
func testSlowFirstReadStillGetsAQuietWindow() {
var clock = 0.0
var reads = ["h", "hello"][...]
let outcome = TextInsertionReadback.settle(
expected: "hello",
now: { clock },
sleep: { clock += $0 },
read: {
if reads.count == 2 { clock += 0.6 }
return reads.popFirst()
}
)
XCTAssertEqual(outcome, .landed)
}
func testReadBlockedPastTheCapIsUndetermined() {
var clock = 0.0
var reads = ["h", "h"][...]
let outcome = TextInsertionReadback.settle(
expected: "hello",
now: { clock },
sleep: { clock += $0 },
read: {
if reads.count == 1 { clock = 2.2 }
return reads.popFirst()
}
)
XCTAssertEqual(outcome, .undetermined)
}
func testContentEditableNoBreakSpacesLand() {
XCTAssertEqual(settle(expected: "a b ", reads: ["a \u{00A0}b\u{00A0}"], secondsPerRead: 0.025), .landed)
}
func testFieldThatDropsTheTextIsAMismatch() {
XCTAssertEqual(settle(expected: "hello", reads: Array(repeating: "", count: 100), secondsPerRead: 0.025), .mismatch)
}
func testFieldThatStopsAnsweringIsUndetermined() {
XCTAssertEqual(settle(expected: "hello", reads: ["", "h"], secondsPerRead: 0.025), .undetermined)
}
func testFieldStillFillingAtTheCapIsUndetermined() {
let reads = (0..<100).map { String(repeating: "a", count: $0) }
XCTAssertEqual(settle(expected: String(repeating: "a", count: 200), reads: reads, secondsPerRead: 0.1), .undetermined)
}
private func settle(expected: String, reads: [String], secondsPerRead: Double) -> TextInsertionReadback.Outcome {
var clock = 0.0
var remaining = reads[...]
return TextInsertionReadback.settle(
expected: expected,
now: { clock },
sleep: { _ in clock += secondsPerRead },
read: { remaining.popFirst() }
)
}
}
+25 -14
View File
@@ -794,12 +794,20 @@ function Render-OrcaTree($RootElement, $WindowFrame, [bool]$CompactBrowserTabs =
# Tab compaction shrinks records mid-walk, so $records.Count would reuse retained indexes.
$nextIndex = [ref]0
function Visit-OrcaNode($Node, [int]$Depth) {
if ($records.Count -ge $MaxNodes -or $Depth -gt $MaxDepth) {
$truncation.truncated = $true
if ($Depth -gt $MaxDepth) { $truncation.maxDepthReached = $true }
return
function Visit-OrcaChildren($Children, [int]$Depth) {
for ($i = 0; $i -lt $Children.Count; $i++) {
if ($records.Count -ge $MaxNodes -or $Depth -gt $MaxDepth) {
$truncation.truncated = $true
if ($Depth -gt $MaxDepth) { $truncation.maxDepthReached = $true }
# Marks where the walk stopped, so the agent can read that parent's subtree by index.
$lines.Add(("`t" * $Depth) + "... $($Children.Count - $i) more not shown (snapshot limit)")
return
}
Visit-OrcaNode $Children.Item($i) $Depth
}
}
function Visit-OrcaNode($Node, [int]$Depth) {
$identity = try { (@($Node.GetRuntimeId()) -join ".") } catch { [Guid]::NewGuid().ToString() }
if (-not $seen.Add($identity)) { return }
@@ -820,9 +828,7 @@ function Render-OrcaTree($RootElement, $WindowFrame, [bool]$CompactBrowserTabs =
}
}
if (($roleKey -in @("pane", "group", "custom", "unknown")) -and [string]::IsNullOrWhiteSpace($title) -and [string]::IsNullOrWhiteSpace($record.value) -and $meaningfulActions.Count -eq 0 -and $null -eq $genericSummary -and $children.Count -le 1) {
for ($i = 0; $i -lt $children.Count; $i++) {
Visit-OrcaNode $children.Item($i) $Depth
}
Visit-OrcaChildren $children $Depth
return
}
@@ -847,9 +853,7 @@ function Render-OrcaTree($RootElement, $WindowFrame, [bool]$CompactBrowserTabs =
if (-not [string]::IsNullOrWhiteSpace($genericSummary) -or (Test-OrcaSuppressChildren $roleKey $title $record.value $genericSummary)) { return }
$childLineStart = $lines.Count
for ($i = 0; $i -lt $children.Count; $i++) {
Visit-OrcaNode $children.Item($i) ($Depth + 1)
}
Visit-OrcaChildren $children ($Depth + 1)
if ($CompactBrowserTabs) {
Compress-OrcaRenderedBrowserTabs $records $lines $childLineStart ($Depth + 1)
}
@@ -1025,14 +1029,21 @@ function Get-OrcaScreenshot([bool]$IncludeScreenshot, $WindowFrame, [IntPtr]$Win
}
}
function New-OrcaSnapshot([string]$Query, [bool]$IncludeScreenshot, $WindowId = $null, $WindowIndex = $null, [bool]$RestoreWindow = $false) {
function New-OrcaSnapshot([string]$Query, [bool]$IncludeScreenshot, $WindowId = $null, $WindowIndex = $null, [bool]$RestoreWindow = $false, $TreeRoot = $null) {
$process = Find-OrcaProcess $Query
Assert-OrcaWindowTarget $process $WindowId $WindowIndex
# Root first: its hang probe must run before ShowWindow, which blocks on a hung window.
$root = Get-OrcaRootElement $process
if ($RestoreWindow) { Restore-OrcaWindow $process }
$windowFrame = Get-OrcaWindowFrame $process $root
$tree = Render-OrcaTree $root $windowFrame (Test-OrcaBrowserProcess $process)
$walkRoot = $root
if ($null -ne $TreeRoot) {
$walkRoot = Find-OrcaElement $root $TreeRoot
if ($null -eq $walkRoot) {
throw "stale element: element $($TreeRoot.index) is gone; run get-app-state again and use a fresh element index"
}
}
$tree = Render-OrcaTree $walkRoot $windowFrame (Test-OrcaBrowserProcess $process)
$screenshot = Get-OrcaScreenshot $IncludeScreenshot $windowFrame ([IntPtr]$process.MainWindowHandle)
[pscustomobject]@{
@@ -1452,7 +1463,7 @@ function Invoke-OrcaOperation($Operation) {
return [pscustomobject]@{ ok = $true; app = $list.app; windows = @($list.windows) }
}
if ($Operation.tool -eq "get_app_state") {
return [pscustomobject]@{ ok = $true; snapshot = New-OrcaSnapshot $Operation.app $includeScreenshot $Operation.windowId $Operation.windowIndex ([bool]$Operation.restoreWindow) }
return [pscustomobject]@{ ok = $true; snapshot = New-OrcaSnapshot $Operation.app $includeScreenshot $Operation.windowId $Operation.windowIndex ([bool]$Operation.restoreWindow) $Operation.root }
}
$process = Find-OrcaProcess $Operation.app
+10 -1
View File
@@ -46,6 +46,12 @@ In a large window, look for one control with `get-app-state --query <text>`: it
ORCA computer get-app-state --app com.apple.finder --query Downloads --json
```
A snapshot reads at most `truncation.maxNodes` elements and `maxDepth` levels. When it stops early, `truncation.truncated` is true and each cut-off place shows a `... N more not shown (snapshot limit)` line, indented like the children it stands for. To read them, pass the parent element's index (the nearest line above with less indentation) as `get-app-state --element-index <n>`: it reads only that element and its descendants, with the full limit. Elements already shown keep their indexes and hidden ones get new ones, and you can act on them. Action results show the whole window again, so read the subtree again before acting on another hidden element.
```text
ORCA computer get-app-state --app com.apple.Safari --element-index 312 --json
```
## Waiting
Never `sleep` between actions. When the UI changes on its own after an action (a save finishes, a dialog opens, a button becomes clickable), wait for the change with `wait`. It returns as soon as the condition holds and prints the matching elements or windows; it fails with `wait_timeout` if the condition does not hold in time (default 10 s, `--timeout <ms>` up to 120000).
@@ -116,12 +122,14 @@ printf '%s' "$TEXT" | ORCA computer set-value --app <app> --element-index <index
- `unverified (accessibility action unasserted)` means the accessibility call succeeded but no post-state assertion was made.
- `unverified (suspected noop)` means the accessibility call succeeded but the window's tree and the app's windows were unchanged about a second later. Web and Electron controls often ignore it. Do not repeat the same element click; if the effect you expect would show in the window, retry once with `click --x --y` at the control from a fresh screenshot. If the effect lives elsewhere (clipboard, another app), check there first.
- `unverified (synthetic input)` means input was fired into the void and is unverifiable.
- `unverified (text mismatch)` means typed or pasted text was read back and the focused field does not hold what inserting it should produce: it may be missing, partly entered, reformatted or autocompleted, or taken by a control that does not store it (terminals and canvas editors leave their input field empty). Do not type it again blindly, or it may land twice; check a fresh screenshot or `get-app-state` first.
- Missing verification metadata is unverified, including responses from older runtimes.
- Never report an unverified action as success. If it could have sent, submitted, bought, or deleted something, say the effect is unproven.
- Prefer semantic actions: `set-value` for editable fields, `click` for controls, and `perform-secondary-action` only for listed action names.
- After any UI-changing action, use the returned state or rerun `get-app-state` before choosing the next element index.
- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; synthetic keyboard delivery is reported as unverified, so inspect the returned state before assuming text landed.
- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; unless the result says `verified focusedText`, inspect the returned state before assuming text landed.
- Use `press-key` for single/navigation keys such as Return, Escape, Tab, and arrows. Use `hotkey` only for one modifier chord plus one key, such as `CmdOrCtrl+A` or `CmdOrCtrl+Shift+P`; prefer `CmdOrCtrl+...` for cross-platform combos.
- Shortcuts that quit or close the app, log out, lock the screen, force-quit or end the session (such as `Cmd+Q` on macOS, `Alt+F4` on Windows and Linux, `Ctrl+Q` on Linux) are refused with `system_shortcut_blocked`. The error says what the shortcut would do; reach that goal another way, such as the window's own close button or menu item. Add `--allow-system-shortcut` only when the user asked for that exact shortcut.
- Use `click --modifiers <chord>` for modifier-clicks. Never synthesize separate modifier-down and modifier-up commands around a click; interruption can leave a modifier logically held.
- Coordinates are window-local; use coordinates from the latest screenshot/state for the same target window.
@@ -182,6 +190,7 @@ Slack: the accessibility tree may be shallow while the screenshot contains usefu
- `app_not_found`: run `list-apps` and retry with the bundle ID. If the target is a web app such as Gmail, choose the desktop browser app/window that contains it; do not retry `ORCA computer ... --app Gmail` unchanged because `orca computer` app selectors refer to desktop apps, not website names.
- `app_blocked`: stop; the target is intentionally blocked from computer-use.
- `system_shortcut_blocked`: the error message says what the shortcut would do. Reach the same goal with a semantic action, such as a button or menu item, or ask the user; retry with `--allow-system-shortcut` only when the user asked for that exact shortcut.
- `window_not_found` / `window_stale`: run `list-windows`, choose a current selector, then rerun `get-app-state`.
- `background_unavailable`: for a popup or `<select>`, use `set-value` on it; otherwise retry once with `--restore-window` after telling the user it will take over their screen.
- `window_not_focused`: retry once with `--restore-window`; if the message says restore was already requested, stop retrying restore and bring the app forward manually or check permissions. For editable fields prefer `set-value`, then inspect before assuming keyboard input worked.
+10 -2
View File
@@ -145,24 +145,32 @@ export async function getComputerTextActionFlags(flags: Map<string, string | boo
export function getComputerKeyActionFlags(flags: Map<string, string | boolean>): {
key: string
allowSystemShortcut?: boolean
} {
const key = getRequiredStringFlag(flags, 'key')
const message = computerUsePressKeyValidationMessage(key)
if (message) {
throw new RuntimeClientError('invalid_argument', message)
}
return { key }
return {
key,
...(flags.get('allow-system-shortcut') === true ? { allowSystemShortcut: true } : {})
}
}
export function getComputerHotkeyActionFlags(flags: Map<string, string | boolean>): {
key: string
allowSystemShortcut?: boolean
} {
const key = getRequiredStringFlag(flags, 'key')
const message = computerUseHotkeyValidationMessage(key)
if (message) {
throw new RuntimeClientError('invalid_argument', message)
}
return { key }
return {
key,
...(flags.get('allow-system-shortcut') === true ? { allowSystemShortcut: true } : {})
}
}
export async function getComputerSetValueActionFlags(
+1 -1
View File
@@ -31,7 +31,7 @@ export function formatGetAppState(result: ComputerSnapshotResult): string {
? ''
: ` @ ${result.snapshot.window.x},${result.snapshot.window.y}`
const truncation = result.snapshot.truncation?.truncated
? ` Truncated: yes (max nodes ${result.snapshot.truncation.maxNodes ?? 'unknown'}, max depth ${result.snapshot.truncation.maxDepth ?? 'unknown'})`
? ` Truncated: yes (max nodes ${result.snapshot.truncation.maxNodes ?? 'unknown'}, max depth ${result.snapshot.truncation.maxDepth ?? 'unknown'}); read cut-off children with --element-index <their parent's index>`
: ' Truncated: no'
return [
`${app.name} (pid ${app.pid}${bundle})`,
+14 -1
View File
@@ -7,6 +7,7 @@ import type {
} from '../../shared/computer-use-runtime-types'
import type { CommandHandler } from '../dispatch/dispatch'
import {
getOptionalNonNegativeIntegerFlag,
getOptionalPositiveIntegerFlag,
getOptionalStringFlag,
getRequiredStringFlag
@@ -85,11 +86,23 @@ export const COMPUTER_HANDLERS: Record<string, CommandHandler> = {
const observeFlags = getComputerObserveFlags(flags)
const query = getOptionalStringFlag(flags, 'query')
const maxElements = getOptionalPositiveIntegerFlag(flags, 'max-elements')
const rootElementIndex = getOptionalNonNegativeIntegerFlag(flags, 'element-index')
const target = await getComputerCommandTarget(flags, cwd, client)
const result = await client.call<ComputerSnapshotResult>('computer.getAppState', {
...target,
...observeFlags
...observeFlags,
...(rootElementIndex === undefined ? {} : { rootElementIndex })
})
// Why: older runtimes drop the param and return the whole window, which would look like the subtree.
if (
rootElementIndex !== undefined &&
result.result.snapshot.rootElementIndex !== rootElementIndex
) {
throw new RuntimeClientError(
'unsupported_capability',
'This Orca runtime cannot read one element subtree; update Orca, or run get-app-state without --element-index'
)
}
if (query !== undefined || maxElements !== undefined) {
const { snapshot } = result.result
snapshot.treeText = queryComputerTreeText(snapshot.treeText, { query, maxElements })
@@ -19,6 +19,8 @@ const COMMAND_SCOPED_FLAG_HELP: Record<string, Record<string, string>> = {
key: '--key <key> Opaque key from reference list; repeat for several'
},
'computer get-app-state': {
'element-index':
'--element-index <n> Read only this element and its descendants, including any the window snapshot cut off',
query:
'--query <text> Show only elements whose line contains this text, with their ancestors'
},
+2
View File
@@ -10,6 +10,8 @@ export const FLAG_HELP_TEXT: Record<string, string> = {
action: '--action <name> Secondary accessibility action name',
activate: '--activate Reveal the new worktree in the Orca app',
app: '--app <app> App name, bundle ID, or pid:N',
'allow-system-shortcut':
'--allow-system-shortcut Send a quit, log out or lock shortcut that is refused by default',
direction:
'--direction <dir> Direction: up|down|left|right for scroll, horizontal|vertical for split',
'display-name': '--display-name <name> Override the Orca display name',
+10 -1
View File
@@ -1065,6 +1065,12 @@ const ORCA_COMPUTER_USE_MARKDOWN = [
"ORCA computer get-app-state --app com.apple.finder --query Downloads --json",
"```",
"",
"A snapshot reads at most `truncation.maxNodes` elements and `maxDepth` levels. When it stops early, `truncation.truncated` is true and each cut-off place shows a `... N more not shown (snapshot limit)` line, indented like the children it stands for. To read them, pass the parent element's index (the nearest line above with less indentation) as `get-app-state --element-index <n>`: it reads only that element and its descendants, with the full limit. Elements already shown keep their indexes and hidden ones get new ones, and you can act on them. Action results show the whole window again, so read the subtree again before acting on another hidden element.",
"",
"```text",
"ORCA computer get-app-state --app com.apple.Safari --element-index 312 --json",
"```",
"",
"## Waiting",
"",
"Never `sleep` between actions. When the UI changes on its own after an action (a save finishes, a dialog opens, a button becomes clickable), wait for the change with `wait`. It returns as soon as the condition holds and prints the matching elements or windows; it fails with `wait_timeout` if the condition does not hold in time (default 10 s, `--timeout <ms>` up to 120000).",
@@ -1135,12 +1141,14 @@ const ORCA_COMPUTER_USE_MARKDOWN = [
" - `unverified (accessibility action unasserted)` means the accessibility call succeeded but no post-state assertion was made.",
" - `unverified (suspected noop)` means the accessibility call succeeded but the window's tree and the app's windows were unchanged about a second later. Web and Electron controls often ignore it. Do not repeat the same element click; if the effect you expect would show in the window, retry once with `click --x --y` at the control from a fresh screenshot. If the effect lives elsewhere (clipboard, another app), check there first.",
" - `unverified (synthetic input)` means input was fired into the void and is unverifiable.",
" - `unverified (text mismatch)` means typed or pasted text was read back and the focused field does not hold what inserting it should produce: it may be missing, partly entered, reformatted or autocompleted, or taken by a control that does not store it (terminals and canvas editors leave their input field empty). Do not type it again blindly, or it may land twice; check a fresh screenshot or `get-app-state` first.",
" - Missing verification metadata is unverified, including responses from older runtimes.",
" - Never report an unverified action as success. If it could have sent, submitted, bought, or deleted something, say the effect is unproven.",
"- Prefer semantic actions: `set-value` for editable fields, `click` for controls, and `perform-secondary-action` only for listed action names.",
"- After any UI-changing action, use the returned state or rerun `get-app-state` before choosing the next element index.",
"- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; synthetic keyboard delivery is reported as unverified, so inspect the returned state before assuming text landed.",
"- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; unless the result says `verified focusedText`, inspect the returned state before assuming text landed.",
"- Use `press-key` for single/navigation keys such as Return, Escape, Tab, and arrows. Use `hotkey` only for one modifier chord plus one key, such as `CmdOrCtrl+A` or `CmdOrCtrl+Shift+P`; prefer `CmdOrCtrl+...` for cross-platform combos.",
"- Shortcuts that quit or close the app, log out, lock the screen, force-quit or end the session (such as `Cmd+Q` on macOS, `Alt+F4` on Windows and Linux, `Ctrl+Q` on Linux) are refused with `system_shortcut_blocked`. The error says what the shortcut would do; reach that goal another way, such as the window's own close button or menu item. Add `--allow-system-shortcut` only when the user asked for that exact shortcut.",
"- Use `click --modifiers <chord>` for modifier-clicks. Never synthesize separate modifier-down and modifier-up commands around a click; interruption can leave a modifier logically held.",
"- Coordinates are window-local; use coordinates from the latest screenshot/state for the same target window.",
"",
@@ -1201,6 +1209,7 @@ const ORCA_COMPUTER_USE_MARKDOWN = [
"",
"- `app_not_found`: run `list-apps` and retry with the bundle ID. If the target is a web app such as Gmail, choose the desktop browser app/window that contains it; do not retry `ORCA computer ... --app Gmail` unchanged because `orca computer` app selectors refer to desktop apps, not website names.",
"- `app_blocked`: stop; the target is intentionally blocked from computer-use.",
"- `system_shortcut_blocked`: the error message says what the shortcut would do. Reach the same goal with a semantic action, such as a button or menu item, or ask the user; retry with `--allow-system-shortcut` only when the user asked for that exact shortcut.",
"- `window_not_found` / `window_stale`: run `list-windows`, choose a current selector, then rerun `get-app-state`.",
"- `background_unavailable`: for a popup or `<select>`, use `set-value` on it; otherwise retry once with `--restore-window` after telling the user it will take over their screen.",
"- `window_not_focused`: retry once with `--restore-window`; if the message says restore was already requested, stop retrying restore and bring the app forward manually or check permissions. For editable fields prefer `set-value`, then inspect before assuming keyboard input worked.",
+6 -6
View File
@@ -44,8 +44,8 @@ export const COMPUTER_COMMAND_SPECS: CommandSpec[] = [
path: ['computer', 'get-app-state'],
summary: 'Capture a compact accessibility snapshot of an app',
usage:
'orca computer get-app-state --app <name|bundle|pid:N> [--window-id <id> | --window-index <n>] [--worktree <selector> | --session <id>] [--query <text>] [--max-elements <n>] [--restore-window] [--no-screenshot] [--json]',
allowedFlags: [...COMPUTER_OBSERVE_FLAGS, 'query', 'max-elements']
'orca computer get-app-state --app <name|bundle|pid:N> [--window-id <id> | --window-index <n>] [--worktree <selector> | --session <id>] [--element-index <n>] [--query <text>] [--max-elements <n>] [--restore-window] [--no-screenshot] [--json]',
allowedFlags: [...COMPUTER_OBSERVE_FLAGS, 'element-index', 'query', 'max-elements']
},
{
path: ['computer', 'wait'],
@@ -121,15 +121,15 @@ export const COMPUTER_COMMAND_SPECS: CommandSpec[] = [
path: ['computer', 'press-key'],
summary: 'Press a single key such as Return or Escape',
usage:
'orca computer press-key --app <app> --key <key> [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--full-tree] [--json]',
allowedFlags: [...COMPUTER_ACTION_FLAGS, 'key']
'orca computer press-key --app <app> --key <key> [--window-id <id> | --window-index <n>] [--allow-system-shortcut] [--restore-window] [--no-screenshot] [--full-tree] [--json]',
allowedFlags: [...COMPUTER_ACTION_FLAGS, 'key', 'allow-system-shortcut']
},
{
path: ['computer', 'hotkey'],
summary: 'Press a platform-aware key combination',
usage:
'orca computer hotkey --app <app> --key <key-combo> [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--full-tree] [--json]',
allowedFlags: [...COMPUTER_ACTION_FLAGS, 'key']
'orca computer hotkey --app <app> --key <key-combo> [--window-id <id> | --window-index <n>] [--allow-system-shortcut] [--restore-window] [--no-screenshot] [--full-tree] [--json]',
allowedFlags: [...COMPUTER_ACTION_FLAGS, 'key', 'allow-system-shortcut']
},
{
path: ['computer', 'paste-text'],
@@ -0,0 +1,126 @@
import { readTestHostJournal } from '../../native-chat/agent-session-journal/journal-observation.test-support'
import { afterEach, expect, it } from 'vitest'
import { readAgentJournalTurn } from '../../../shared/agent-session/lifecycle/agent-session-turn-record'
import { closeProviderTimelineRigs } from '../../native-chat/agent-session-timeline/provider-timeline-assembler-test-support'
import {
HOST_TEST_SESSION as SESSION,
hostTestAttachParams
} from '../../native-chat/agent-session-wire/host/testing/structured-agent-session-host-test-data.test-fixture'
import {
createRestTestRig,
REST_TEST_CALLER,
restTestSend,
sweepOnce,
type RestTestRig
} from '../../native-chat/agent-session-wire/host/testing/structured-agent-session-rest-test-rig.test-fixture'
import { StructuredAgentRegistry } from '../../native-chat/agent-session-wire/providers/structured-agent-registry'
import { STRUCTURED_AGENT_SESSION_IDLE_MS as IDLE_MS } from '../../native-chat/agent-session-wire/termination/structured-agent-session-idle-sweep'
import { acpLaunchSpecFor } from '../launch/acp-launch-registry'
import { readAcpFixture } from '../timeline/testing/acp-timeline-fixture.test-support'
import { acpStructuredAgentDefinition } from './acp-structured-agent-definitions'
import { openAcpAdapterRig, waitFor } from './testing/acp-structured-adapter.test-support'
import { GrokFixtureReplay } from './testing/acp-structured-fixture-replay.test-support'
afterEach(closeProviderTimelineRigs)
async function openChat(agent: 'omp' | 'grok', replay?: GrokFixtureReplay) {
const spec = acpLaunchSpecFor(agent)!
let hostRig: RestTestRig | undefined
const acp = await openAcpAdapterRig({
spec,
script: (scripted) => {
if (replay) {
replay.attach(scripted)
} else {
scripted.on('session/prompt', (frame) => scripted.reply(frame, { stopReason: 'end_turn' }))
}
},
deps: {
onEvent: (event) => hostRig?.host.handleAdapterEvent(event),
onDispatchSettledLate: (settlement) => void hostRig?.host.settleLateDispatch(settlement),
now: () => hostRig?.clock.now ?? 0
}
})
const rig = await createRestTestRig({
adapter: acp.adapter,
agents: new StructuredAgentRegistry([
{ definition: acpStructuredAgentDefinition(spec), adapter: acp.adapter }
])
})
hostRig = rig
rig.host.collaboratorsForTests().lifetime.idleSweep.dispose()
const attached = await rig.host.attach(
REST_TEST_CALLER,
hostTestAttachParams(null, {
provider: agent,
agent,
accountHome: {
variable: agent === 'omp' ? 'PI_CODING_AGENT_DIR' : 'GROK_HOME',
path: rig.root
},
providerHandle: undefined
})
)
expect(attached).toMatchObject({ ok: true })
return { rig, acp }
}
async function sendAndFinish(rig: RestTestRig, text: string, completed: number) {
expect(await rig.host.send(REST_TEST_CALLER, restTestSend(text))).toMatchObject({ ok: true })
await waitFor(async () => {
const snapshot = await readTestHostJournal(rig.host, SESSION)
const turns = snapshot.items.flatMap((item) => readAgentJournalTurn(item.body) ?? [])
expect(turns).toHaveLength(completed)
expect(turns.every((turn) => turn.state === 'completed')).toBe(true)
expect(snapshot.submissions).toHaveLength(completed)
expect(
snapshot.submissions.every((submission) => submission.dispatchState === 'accepted')
).toBe(true)
})
await rig.host.drainStatusWrites()
}
it('retains an aged OMP child after recorded parent completion, accepts another send, and closes explicitly', async () => {
const replay = new GrokFixtureReplay(await readAcpFixture('omp-v17-reply-without-thought'))
const { rig, acp } = await openChat('omp', replay)
try {
await sendAndFinish(rig, 'First question', 1)
const child = acp.child()
rig.clock.now += IDLE_MS + 1
await sweepOnce(rig.host)
expect(child.exited).toBe(false)
expect(rig.host.hasSession(SESSION)).toBe(true)
// Completion uncertainty remains uncertainty even long after the normal idle window.
rig.clock.now += 24 * 60 * 60_000
await sweepOnce(rig.host)
expect(child.exited).toBe(false)
await sendAndFinish(rig, 'Follow-up question', 2)
expect(replay.awaiting).toBeNull()
expect(acp.child()).toBe(child)
await rig.host.close(SESSION, 'user-close')
expect(child.exited).toBe(true)
expect(rig.host.hasSession(SESSION)).toBe(false)
} finally {
await acp.adapter.closeAll()
await closeProviderTimelineRigs()
await rig.dispose()
}
})
it('still retires an idle ACP child whose provider has no extra idle confirmation', async () => {
const { rig, acp } = await openChat('grok')
try {
await sendAndFinish(rig, 'First question', 1)
const child = acp.child()
rig.clock.now += IDLE_MS + 1
await sweepOnce(rig.host)
expect(child.exited).toBe(true)
expect(rig.host.hasSession(SESSION)).toBe(false)
} finally {
await acp.adapter.closeAll()
await closeProviderTimelineRigs()
await rig.dispose()
}
})
@@ -222,6 +222,14 @@ export class AcpStructuredSessionAdapter implements SessionAdapter {
return session ? childState.isAcpReadySession(session) && session.turns.running : false
}
confirmIdle: NonNullable<SessionAdapter['confirmIdle']> = (sessionId) => {
const session = this.children.sessions.get(sessionId)
// OMP cannot prove that a finished parent has no background work or pending results.
return session?.spec.agent === 'omp' && !session.connection.exited
? Promise.resolve('unknown')
: undefined
}
holdsLiveProviderProcess = (sessionId: string, generation: string): boolean => {
const session = this.children.sessions.get(sessionId)
return session?.acquisitionGeneration === generation && !session.connection.exited
@@ -91,13 +91,9 @@ export abstract class AgentBrowserBridgeKeyCommands extends AgentBrowserBridgeCa
if (typeof step === 'string') {
// Text no key types (emoji, controls) goes in like a paste, between the keys around it.
await flush()
const route = await routePageTextInsertion(wc, { text: step }, send, guardFocus)
try {
assertExpectedPageUrlForStep()
await route.deliver({ text: step })
} finally {
route.release()
}
const route = await routePageTextInsertion(wc, send)
assertExpectedPageUrlForStep()
await route.deliver({ text: step })
continue
}
const event = {
@@ -4,7 +4,7 @@ import type { CdpClientResponseWriter } from '../transport/cdp-client-response-w
import type { CdpDebuggerChannel } from '../debugger/cdp-debugger-channel'
import type { TextInsertionRoute } from './guest-direct-key-input'
/** Replay DOM.focus before insertion because native focusing can blur the target. */
/** Replay DOM.focus before insertion so pipelined inserts use the resolved element. */
export class CdpDomFocusReplay {
private pendingDomFocusBySession = new Map<
string | undefined,
@@ -72,7 +72,7 @@ export class CdpDomFocusReplay {
return pendingFocus
}
/** Choose the route before replay: native fallback focus can blur the target. */
/** Choose a page-bound route before replaying the selected element. */
async insertText(
client: WebSocket,
clientId: number,
@@ -97,18 +97,14 @@ export class CdpDomFocusReplay {
this.responder.sendError(clientId, err instanceof Error ? err.message : String(err), client)
return
}
try {
await this.replayAndDeliver(
client,
clientId,
params,
effectiveSessionId,
pendingFocusParams,
route
)
} finally {
route.release()
}
await this.replayAndDeliver(
client,
clientId,
params,
effectiveSessionId,
pendingFocusParams,
route
)
}
private async replayAndDeliver(
@@ -12,6 +12,7 @@ import {
import {
dispatchPageKeyEvent,
isEmbeddedGuest,
pageInputRouteUnavailableError,
routePageTextInsertion
} from './guest-direct-key-input'
import type { GuestSessionCdpSender } from './guest-frame-input-sessions'
@@ -84,10 +85,9 @@ export class CdpPageInputCommands {
'Input.insertText',
insertParams,
sessionId
),
release: () => {}
)
})
: routePageTextInsertion(this.webContents, params, this.sendRoot, this.guardFocus)
: routePageTextInsertion(this.webContents, this.sendRoot)
)
)
return true
@@ -122,17 +122,16 @@ export class CdpPageInputCommands {
withAgentInputInjection(this.webContents, () =>
this.debuggerChannel.sendDebuggerCommand(method, params, sessionId)
)
const routedByFocus = !sessionId && isFocusRoutedCdpInput(method)
if (!routedByFocus && !isFocusTakingCdpInput(method, params)) {
if (!sessionId && isFocusRoutedCdpInput(method)) {
throw pageInputRouteUnavailableError()
}
if (!isFocusTakingCdpInput(method, params)) {
return await send()
}
// Emulate page focus while the pane guards and restores user focus.
await enablePageFocusEmulation(this.sendRoot)
const release = await this.guardFocus()
try {
if (routedByFocus) {
this.webContents.focus()
}
return await send()
} finally {
release()
@@ -0,0 +1,54 @@
import { describe, expect, it, vi } from 'vitest'
import { dispatchPageKeyEvent, routePageTextInsertion } from './guest-direct-key-input'
import { createMockWebContents, mockAsWebContents } from '../transport/cdp-ws-proxy-test-harness'
import type * as FocusedFrameRouteModule from './guest-focused-frame-route'
const forceUnknownRoute = vi.hoisted(() => ({ enabled: false }))
vi.mock('./guest-focused-frame-route', async (importActual) => {
const actual = await importActual<typeof FocusedFrameRouteModule>()
return {
resolveFocusedFrameRoute: (...args: Parameters<typeof actual.resolveFocusedFrameRoute>) =>
forceUnknownRoute.enabled
? Promise.resolve({ kind: 'unknown' })
: actual.resolveFocusedFrameRoute(...args)
}
})
function createGuest() {
const mock = createMockWebContents()
mock.webContents.getType.mockReturnValue('webview')
return mock
}
describe('guest input route admission', () => {
it('refuses keys when the focused guest frame cannot be resolved', async () => {
forceUnknownRoute.enabled = true
try {
const mock = createGuest()
await expect(
dispatchPageKeyEvent(
mockAsWebContents(mock),
{ type: 'keyDown', key: 'a', text: 'a' },
async () => ({})
)
).rejects.toMatchObject({ code: 'browser_input_target_unavailable' })
} finally {
forceUnknownRoute.enabled = false
}
})
it('refuses text when the root frame probe cannot confirm focus', async () => {
const mock = createGuest()
const send = vi.fn(async (method: string) => {
if (method === 'Runtime.evaluate') {
throw new Error('probe context destroyed')
}
return {}
})
await expect(routePageTextInsertion(mockAsWebContents(mock), send)).rejects.toMatchObject({
code: 'browser_input_target_unavailable'
})
})
})
@@ -1,14 +1,16 @@
import { assertExpectedPageUrlForStep } from './browser-expected-page-url'
import type { WebContents } from 'electron'
import { enablePageFocusEmulation, type AgentInputFocusGuard } from './agent-input-focus-guard'
import type { AgentInputFocusGuard } from './agent-input-focus-guard'
import { BrowserError } from '../../browser-error'
import { withAgentInputInjection } from './agent-input-injection'
import { resolveFocusedFrameRoute } from './guest-focused-frame-route'
import { prepareGuestTextInsertion } from './guest-renderer-text-insertion'
import {
settleWithin,
type GuestSessionCdpSender,
sendFrameSessionInput
} from './guest-frame-input-sessions'
import { planCdpKeyEvent, planInsertText, type GuestEditCommand } from './guest-key-event-plan'
import { planCdpKeyEvent, type GuestEditCommand } from './guest-key-event-plan'
import { sendGuestCdpCommand } from '../debugger/guest-cdp-command'
import { acquireElectronDebugger } from '../debugger/electron-debugger-lease'
import { normalizeCdpPointerButton } from './cdp-pointer-targets'
@@ -68,16 +70,11 @@ function runEditCommand(webContents: WebContents, command: GuestEditCommand): vo
const NO_FOCUS_GUARD: AgentInputFocusGuard = async () => () => {}
// Restore user focus after CDP fallback; emulation keeps the page from seeing blur.
async function focusGuestForCdpInput(
webContents: WebContents,
send: GuestSessionCdpSender,
guardFocus: AgentInputFocusGuard
): Promise<() => void> {
await enablePageFocusEmulation(send)
const release = await guardFocus()
webContents.focus()
return release
export function pageInputRouteUnavailableError(): BrowserError {
return new BrowserError(
'browser_input_target_unavailable',
'Input could not be routed directly to the browser page, so it was not sent.'
)
}
// Cross-site frame sessions route CDP input without borrowing user focus.
@@ -93,7 +90,7 @@ async function sendToFrameSession(
)
}
/** Prefer guest widget or frame-session events; borrow focus for embedded-guest CDP fallback. */
/** Guest input must target its widget or frame session without borrowing window focus. */
export async function dispatchPageKeyEvent(
webContents: WebContents,
params: Record<string, unknown>,
@@ -190,17 +187,7 @@ export async function dispatchPageKeyEvents(
}
return {}
}
const release = await focusGuestForCdpInput(webContents, send, guardFocus)
try {
for (const params of remaining) {
result = await withAgentInputInjection(webContents, () =>
send('Input.dispatchKeyEvent', params)
)
}
return result
} finally {
release()
}
throw pageInputRouteUnavailableError()
}
/** Release input held by a revoked host, routing keys to the focused frame. */
@@ -235,49 +222,45 @@ export async function releaseHeldPageInput(
export type TextInsertionRoute = {
deliver: (params: Record<string, unknown>) => Promise<unknown>
// Release borrowed focus even if delivery is abandoned.
release: () => void
}
/** Route before DOM.focus replay: CDP fallback can blur the selected element. */
/** Guest edits target their renderer without borrowing window focus. */
export async function routePageTextInsertion(
webContents: WebContents,
params: Record<string, unknown>,
send: GuestSessionCdpSender,
guardFocus: AgentInputFocusGuard = NO_FOCUS_GUARD
send: GuestSessionCdpSender
): Promise<TextInsertionRoute> {
const sendText = (insertParams: Record<string, unknown>): Promise<unknown> =>
send('Input.insertText', insertParams)
if (!isEmbeddedGuest(webContents)) {
webContents.focus()
return { deliver: sendText, release: () => {} }
return { deliver: sendText }
}
const route = await resolveFocusedFrameRoute(webContents, send)
if (route.kind === 'session') {
const { sessionId } = route
return {
deliver: (insertParams) =>
sendToFrameSession(webContents, send, sessionId, 'Input.insertText', insertParams),
release: () => {}
sendToFrameSession(webContents, send, sessionId, 'Input.insertText', insertParams)
}
}
const events = planInsertText(params.text)
if (events && route.kind === 'widget') {
if (route.kind === 'widget' && route.hasFocusedLocalFrame) {
const insertText = await prepareGuestTextInsertion(send, route.contextId)
return {
deliver: () =>
deliver: (insertParams) =>
withAgentInputInjection(webContents, async () => {
for (const event of events) {
webContents.sendInputEvent(event)
assertExpectedPageUrlForStep()
if (typeof insertParams.text !== 'string') {
throw new BrowserError('invalid_argument', 'Browser text must be a string')
}
await waitForPageToHandleInput(send)
await insertText(webContents, insertParams.text)
return {}
}),
release: () => {}
})
}
}
const release = await focusGuestForCdpInput(webContents, send, guardFocus)
return {
deliver: (insertParams) => withAgentInputInjection(webContents, () => sendText(insertParams)),
release
if (route.kind === 'unknown') {
throw pageInputRouteUnavailableError()
}
throw new BrowserError(
'browser_input_target_unavailable',
'The browser page has no focused element to receive text, so it was not sent. Focus a field first.'
)
}
@@ -11,6 +11,8 @@ function fakeGuest(): WebContents {
type FrameModel = {
// Document keys: '' = root, session ID = cross-site frame, 'frame:<id>' = same-site context.
focusedFrameBySession: Record<string, string | null>
focusedElementBySession?: Record<string, boolean>
rootProbeFails?: boolean
crossSiteTargets: string[]
sessionForTarget: Record<string, string>
deadSessions?: Set<string>
@@ -29,12 +31,20 @@ function frameSender(model: FrameModel) {
}
switch (method) {
case 'Runtime.evaluate': {
if (model.rootProbeFails && session === '' && params.contextId === undefined) {
throw new Error('Execution context was destroyed')
}
const contextFrame =
typeof params.contextId === 'number' ? frameByContext.get(params.contextId) : undefined
const documentKey = contextFrame ? `frame:${contextFrame}` : session
const frameId = model.focusedFrameBySession[documentKey] ?? null
if (!frameId) {
return { result: { type: 'object', subtype: 'null' } }
return {
result: {
type: 'boolean',
value: model.focusedElementBySession?.[documentKey] ?? false
}
}
}
const objectId = `object-${(nextObject += 1)}`
frameByObject.set(objectId, frameId)
@@ -12,11 +12,12 @@ import {
// Cross-site frames need their own CDP session; widget input only reaches the guest renderer.
export type FocusedFrameRoute =
| { kind: 'widget' }
// The renderer still validates the editable target when committing text.
| { kind: 'widget'; hasFocusedLocalFrame: boolean; contextId?: number }
| { kind: 'session'; sessionId: string }
| { kind: 'unknown' }
// Deeper nesting uses the focus-borrowing fallback.
// Refuse input if deeper nesting prevents a direct route.
const MAX_FRAME_DEPTH = 6
const PROBE_OBJECT_GROUP = 'orca-focused-frame-probe'
const PROBE_WORLD_NAME = 'orca-focused-frame-probe'
@@ -26,11 +27,14 @@ const FOCUSED_FRAME_ELEMENT_EXPRESSION = `(() => {
while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement
const tag = element?.tagName
const owner = tag === 'IFRAME' || tag === 'FRAME' || tag === 'FENCEDFRAME' || tag === 'OBJECT' || tag === 'EMBED'
return owner ? element : null
// A fresh body is not a text target, but an editable body is a valid editing host.
return owner ? element : Boolean(element) && (element.isContentEditable || (element !== document.body && element !== document.documentElement))
})()`
type ProbeSite = { sessionId?: string; contextId?: number; targetId?: string }
type FocusProbe = { outcome: 'ok'; frameId: string | null } | { outcome: 'failed' | 'timeout' }
type FocusProbe =
| { outcome: 'ok'; frameId: string | null; focusedElement: boolean }
| { outcome: 'failed' | 'timeout' }
async function probeFocusedFrame(send: GuestSessionCdpSender, at: ProbeSite): Promise<FocusProbe> {
const evaluated = await settle(
@@ -53,12 +57,16 @@ async function probeFocusedFrame(send: GuestSessionCdpSender, at: ProbeSite): Pr
}
const objectId = readString(result, 'objectId')
if (!objectId) {
return { outcome: 'ok', frameId: null }
return { outcome: 'ok', frameId: null, focusedElement: result.value === true }
}
try {
const described = await settle(send('DOM.describeNode', { objectId }, at.sessionId))
return described.outcome === 'ok'
? { outcome: 'ok', frameId: readString(readRecord(described.value, 'node'), 'frameId') }
? {
outcome: 'ok',
frameId: readString(readRecord(described.value, 'node'), 'frameId'),
focusedElement: false
}
: described
} finally {
void send(
@@ -87,11 +95,19 @@ async function frameContext(
return typeof contextId === 'number' ? contextId : null
}
function routeFor(at: ProbeSite): FocusedFrameRoute {
return at.sessionId ? { kind: 'session', sessionId: at.sessionId } : { kind: 'widget' }
function routeFor(at: ProbeSite, focusedElement: boolean): FocusedFrameRoute {
if (at.sessionId) {
return { kind: 'session', sessionId: at.sessionId }
}
// Entering a same-site frame through its focused owner proves that frame holds focus.
return {
kind: 'widget',
hasFocusedLocalFrame: focusedElement || at.contextId !== undefined,
...(at.contextId === undefined ? {} : { contextId: at.contextId })
}
}
/** Follow focus through same-site contexts and cross-site sessions; unknown routes borrow focus. */
/** Follow focus through same-site contexts and cross-site sessions. */
export async function resolveFocusedFrameRoute(
webContents: WebContents,
send: GuestSessionCdpSender
@@ -113,11 +129,14 @@ export async function resolveFocusedFrameRoute(
at = { sessionId: fresh, targetId: at.targetId }
continue
}
// Main-frame navigation can invalidate the probe context; fall back to widget input.
return at.sessionId || at.contextId !== undefined ? { kind: 'unknown' } : { kind: 'widget' }
// Navigation can invalidate the root probe; widget keys still reach the guest, but text cannot
// confirm a focused frame.
return at.sessionId || at.contextId !== undefined
? { kind: 'unknown' }
: { kind: 'widget', hasFocusedLocalFrame: false }
}
if (!probe.frameId) {
return routeFor(at)
return routeFor(at, probe.focusedElement)
}
let crossSite = await crossSiteFrameTargets(webContents, send, { refresh: false })
if (!crossSite.has(probe.frameId)) {
@@ -79,7 +79,7 @@ function electronModifiers(mask: unknown): ElectronModifier[] {
return CDP_MODIFIER_NAMES.filter(([bit]) => (bits & bit) !== 0).map(([, name]) => name)
}
// Blink drops control characters from char events; use the fallback to preserve them.
// Blink drops control characters from char events.
function typeableText(text: unknown): text is string {
return (
typeof text === 'string' &&
@@ -146,7 +146,3 @@ export function planCdpKeyEvent(params: Record<string, unknown>): GuestKeyPlan |
: []
return { events: [event, ...enterChar], commands }
}
export function planInsertText(text: unknown): GuestKeyboardEvent[] | null {
return typeableText(text) ? charEvents(text, []) : null
}
@@ -0,0 +1,143 @@
import type { WebContents } from 'electron'
import { BrowserError } from '../../browser-error'
import {
readRecord,
readString,
settle,
settleWithin,
type GuestSessionCdpSender
} from './guest-frame-input-sessions'
const INSERT_TIMEOUT_MS = 10_000
const INSERT_WORLD_NAME = 'orca-guest-text-insertion'
const insertionScript = (
text: string,
expiresAt: number,
requireFrameFocus: boolean
): string => `(() => {
const expiresAt = ${expiresAt}
const text = ${JSON.stringify(text)}
const focusedField = () => {
if (${requireFrameFocus} && !document.hasFocus()) return null
let doc = document
for (let depth = 0; depth < 64; depth++) {
let field = doc.activeElement
while (field?.shadowRoot?.activeElement) field = field.shadowRoot.activeElement
if (!field) return null
if (field.tagName === 'IFRAME' || field.tagName === 'FRAME') {
doc = field.contentDocument
if (!doc) return null
continue
}
if (field.disabled || field.readOnly) return null
const editable = field.isContentEditable || field.tagName === 'TEXTAREA' ||
(field.tagName === 'INPUT' && ['text', 'search', 'url', 'tel', 'email', 'password', 'number'].includes(field.type))
return editable ? { doc, field } : null
}
return null
}
if (Date.now() >= expiresAt) return 'expired'
const target = focusedField()
if (!target) return 'unavailable'
const event = new InputEvent('beforeinput', {
bubbles: true, composed: true, cancelable: true, inputType: 'insertText', data: text
})
if (!target.field.dispatchEvent(event)) return 'cancelled'
// Page handlers can move focus, navigate, or block beyond the deadline.
const current = focusedField()
if (!current || current.field !== target.field || current.doc !== target.doc) return 'unavailable'
if (Date.now() >= expiresAt) return 'expired'
return Document.prototype.execCommand.call(target.doc, 'insertText', false, text) ? 'inserted' : 'unavailable'
})()`
/** Bind edits to this document's isolated world, so navigation invalidates queued work. */
export async function prepareGuestTextInsertion(
send: GuestSessionCdpSender,
focusedContextId?: number
): Promise<(webContents: WebContents, text: string) => Promise<void>> {
if (focusedContextId !== undefined) {
// Virtual page focus preserves background input while hasFocus fences a stale child route.
const emulated = await settle(send('Emulation.setFocusEmulationEnabled', { enabled: true }))
if (emulated.outcome !== 'ok') {
throw unavailableError()
}
return (webContents, text) => insertGuestText(webContents, send, focusedContextId, text, true)
}
const tree = await settleWithin(send('Page.getFrameTree', {}), null)
const frameId = readString(readRecord(readRecord(tree, 'frameTree'), 'frame'), 'id')
const world = frameId
? await settleWithin(
send('Page.createIsolatedWorld', { frameId, worldName: INSERT_WORLD_NAME }),
null
)
: null
const contextId =
world && typeof world === 'object' && 'executionContextId' in world
? world.executionContextId
: null
if (typeof contextId !== 'number') {
throw unavailableError()
}
return (webContents, text) => insertGuestText(webContents, send, contextId, text, false)
}
function unavailableError(): BrowserError {
return new BrowserError(
'browser_input_target_unavailable',
'The browser page has no focused editable element to receive text, so it was not sent. Focus a field first.'
)
}
function timeoutError(): BrowserError {
return new BrowserError('browser_timeout', 'The browser page did not confirm the text in time.')
}
async function insertGuestText(
webContents: WebContents,
send: GuestSessionCdpSender,
contextId: number,
text: string,
requireFrameFocus: boolean
): Promise<void> {
const expiresAt = Date.now() + INSERT_TIMEOUT_MS
let stop = (): void => {}
const abandoned = new Promise<never>((_resolve, reject) => {
const fail = (): void =>
reject(
new BrowserError(
'browser_error',
'The browser page crashed or closed before it confirmed the text.'
)
)
const timer = setTimeout(() => reject(timeoutError()), INSERT_TIMEOUT_MS)
webContents.once('render-process-gone', fail)
webContents.once('destroyed', fail)
stop = () => {
clearTimeout(timer)
webContents.removeListener('render-process-gone', fail)
webContents.removeListener('destroyed', fail)
}
})
try {
const response = await Promise.race([
send('Runtime.evaluate', {
expression: insertionScript(text, expiresAt, requireFrameFocus),
contextId,
returnByValue: true
}).catch(() => {
throw unavailableError()
}),
abandoned
])
const outcome = readRecord(response, 'result')?.value
if (outcome === 'expired') {
throw timeoutError()
}
if (outcome !== 'inserted' && outcome !== 'cancelled') {
throw unavailableError()
}
} finally {
stop()
}
}
@@ -0,0 +1,237 @@
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { build } from 'esbuild'
import { expect, it } from 'vitest'
import { runBrowserRouteEgressElectron } from '../../network/egress/browser-route-egress-electron-launch'
const MAIN = String.raw`
import { app, BrowserWindow } from 'electron'
import { readFileSync, writeFileSync } from 'node:fs'
import { createServer } from 'node:http'
import { routePageTextInsertion } from './src/main/browser/cdp/input/guest-direct-key-input'
if (process.platform === 'darwin') app.setActivationPolicy('accessory')
app.commandLine.appendSwitch('site-per-process')
const { resultPath, scenario } = JSON.parse(readFileSync(process.argv[2], 'utf8'))
const answer = (value) => { writeFileSync(resultPath, JSON.stringify(value)); app.exit(0) }
app.whenReady().then(async () => {
const server = createServer((req, res) => {
res.setHeader('Content-Type', 'text/html')
res.end(req.url === '/frame' ? '<textarea id="field"></textarea>' :
'<textarea id="field"></textarea><iframe src="http://localhost:' + server.address().port + '/frame"></iframe>')
})
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve))
const win = new BrowserWindow({ show: false, width: 960, height: 640,
webPreferences: { offscreen: true, webviewTag: true } })
const attached = new Promise(resolve => win.webContents.once('did-attach-webview', (_event, guest) => resolve(guest)))
await win.loadURL('data:text/html,' + encodeURIComponent('<textarea id="draft">Unsent draft</textarea>' +
'<webview style="width:800px;height:500px" src="http://127.0.0.1:' + server.address().port + '/"></webview>'))
const guest = await attached
if (guest.isLoading()) await new Promise(resolve => guest.once('did-finish-load', resolve))
let crashed = false
guest.on('render-process-gone', () => { crashed = true })
guest.debugger.attach('1.3')
const send = (method, params, sessionId) => guest.debugger.sendCommand(method, params, sessionId)
const insert = async text => (await routePageTextInsertion(guest, send)).deliver({ text })
await guest.executeJavaScript('field.focus()')
if (scenario.startsWith('child-')) {
const childServer = createServer((_req,res) => {
res.setHeader('Content-Type','text/html')
res.end('<textarea id="field"></textarea>')
})
await new Promise(resolve => childServer.listen(0,'127.0.0.1',resolve))
const loaded = new Promise(resolve => guest.once('did-frame-finish-load',resolve))
await guest.executeJavaScript('document.querySelector("iframe").src="http://127.0.0.1:' + childServer.address().port + '/";void 0')
await loaded
const { frameTree } = await send('Page.getFrameTree',{})
const frameId = frameTree.childFrames[0].frame.id
const { executionContextId } = await send('Page.createIsolatedWorld',{frameId,worldName:'fixture-child'})
const child = expression => send('Runtime.evaluate',{expression,contextId:executionContextId,returnByValue:true})
const { targetInfos } = await send('Target.getTargets',{})
if (targetInfos.some(target => target.type === 'iframe' && target.targetId === frameId)) throw Error('Fixture child must share the root renderer')
await child('field.focus()')
if (scenario === 'child-focus-race') await child("document.designMode='on';document.body.innerHTML='';document.body.focus()")
await win.webContents.executeJavaScript('draft.focus()')
const route = await routePageTextInsertion(guest,send)
if (scenario === 'child-focus-race') await guest.executeJavaScript('field.focus()')
if (scenario === 'child-detached') await guest.executeJavaScript('document.querySelector("iframe").remove()')
if (scenario === 'child-navigation') {
const reloaded = new Promise(resolve => guest.once('did-frame-finish-load',resolve))
await guest.executeJavaScript('document.querySelector("iframe").src+="?new";void 0')
await reloaded
}
if (scenario === 'child-expired') {
const busy = new Promise(resolve => guest.once('console-message',resolve))
void child("console.log('busy');{const end=performance.now()+11000;while(performance.now()<end){}}")
await busy
}
let outcome = 'ok'
try { await route.deliver({text:'Child 🙏'}) } catch(error) {outcome=error.code}
const after = scenario === 'child-detached' ? null : scenario === 'child-navigation'
? await guest.mainFrame.frames[0].executeJavaScript('field.value')
: (await child(scenario === 'child-focus-race' ? 'document.body.textContent' : 'field.value')).result.value
await guest.executeJavaScript('field.focus()')
await insert('Next 🙏')
const result = {outcome,after,root:await guest.executeJavaScript('field.value'),host:await win.webContents.executeJavaScript('draft.value'),hostFocus:await win.webContents.executeJavaScript('document.activeElement.id')}
guest.debugger.detach()
win.destroy()
answer(result)
return
}
if (scenario === 'editing') {
await guest.executeJavaScript("field.value='Replace me';field.setSelectionRange(0,10);window.inputText='';field.addEventListener('input',()=>{inputText=field.value})")
await insert('First\nSecond 🙏')
const selection = await guest.executeJavaScript('field.value')
const inputText = await guest.executeJavaScript('inputText')
await guest.executeJavaScript("document.execCommand('undo')")
const undo = await guest.executeJavaScript('field.value')
await guest.executeJavaScript("field.addEventListener('beforeinput', e => e.preventDefault(), {once:true})")
await insert('Cancelled')
const cancelled = await guest.executeJavaScript('field.value')
await guest.executeJavaScript("const rich=document.createElement('div');rich.id='rich';rich.contentEditable='true';document.body.append(rich);rich.focus()")
await insert('Rich 🙏')
const rich = await guest.executeJavaScript('rich.textContent')
await guest.executeJavaScript("const host=document.createElement('div');host.id='shadow';document.body.append(host);host.attachShadow({mode:'open'}).innerHTML='<textarea></textarea>';host.shadowRoot.firstChild.focus()")
await insert('Shadow')
const shadow = await guest.executeJavaScript('shadow.shadowRoot.firstChild.value')
await guest.executeJavaScript("const frame=document.createElement('iframe');frame.id='local';document.body.append(frame);frame.contentDocument.body.innerHTML='<textarea id=localfield></textarea>';frame.contentDocument.querySelector('textarea').focus();Date.now=()=>0;Document.prototype.execCommand=()=>{throw Error('page hook ran')};frame.contentWindow.Document.prototype.execCommand=()=>{throw Error('child hook ran')};void 0")
await insert('Local 🙏')
const local = await guest.executeJavaScript('local.contentDocument.querySelector("textarea").value')
await guest.executeJavaScript("local.contentDocument.designMode='on';local.contentDocument.body.innerHTML='';local.contentWindow.focus()")
await insert('Design')
const design = await guest.executeJavaScript('local.contentDocument.body.textContent')
await guest.executeJavaScript("field.focus();field.addEventListener('beforeinput',()=>shadow.shadowRoot.firstChild.focus(),{once:true})")
let moved = 'ok'
try { await insert('Wrong target') } catch(error) { moved=error.code }
await guest.executeJavaScript("document.body.innerHTML='';document.body.contentEditable='true';document.body.focus()")
await insert('Body 🙏')
const body = await guest.executeJavaScript('document.body.textContent')
answer({ selection, inputText, undo, cancelled, rich, shadow, local, design, moved, body,
host: await win.webContents.executeJavaScript('draft.value') })
return
}
if (scenario === 'detached') {
await guest.executeJavaScript("const frame=document.createElement('iframe');frame.id='local';document.body.append(frame);frame.contentDocument.body.innerHTML='<textarea></textarea>';frame.contentDocument.querySelector('textarea').focus()")
}
const route = await routePageTextInsertion(guest, send)
if (scenario === 'focus-race') {
const { targetInfos } = await send('Target.getTargets', {})
const target = targetInfos.find(info => info.type === 'iframe')
if (!target) throw new Error('Cross-site frame target missing')
const { sessionId } = await send('Target.attachToTarget', { targetId: target.targetId, flatten: true })
const { root } = await send('DOM.getDocument', {}, sessionId)
const { nodeId } = await send('DOM.querySelector', { nodeId: root.nodeId, selector: '#field' }, sessionId)
await send('DOM.focus', { nodeId }, sessionId)
} else if (scenario === 'navigation') {
await guest.loadURL('http://127.0.0.1:' + server.address().port + '/')
await guest.executeJavaScript('field.focus()')
} else if (scenario === 'detached') {
await guest.executeJavaScript('local.remove()')
} else if (scenario === 'handler-expired') {
await guest.executeJavaScript("field.addEventListener('beforeinput',()=>{const end=performance.now()+11000;while(performance.now()<end){}},{once:true})")
} else {
const busy = new Promise(resolve => guest.once('console-message', resolve))
void guest.executeJavaScript("Date.now=()=>0;console.log('busy');{const end=performance.now()+11000;while(performance.now()<end){}};void 0").catch(() => {})
await busy
}
let outcome = 'ok'
const delivered = route.deliver({ text: 'LATE' })
if (scenario === 'renderer-dies') guest.forcefullyCrashRenderer()
try { await delivered } catch (error) { outcome = error.code }
if (scenario === 'renderer-dies') {
answer({ outcome, host: await win.webContents.executeJavaScript('draft.value') })
return
}
if (crashed) { answer({ crashed, outcome }); return }
const after = await guest.executeJavaScript('field.value')
await guest.executeJavaScript('field.focus()')
const next = await routePageTextInsertion(guest, send)
await next.deliver({ text: 'Next 🙏' })
answer({ crashed, outcome, after, next: await guest.executeJavaScript('field.value'),
host: await win.webContents.executeJavaScript('draft.value') })
}).catch(error => answer({ error: String(error.stack || error) }))
`
it.each([
'focus-race',
'expired',
'navigation',
'detached',
'editing',
'handler-expired',
'renderer-dies',
'child-insert',
'child-focus-race',
'child-expired',
'child-navigation',
'child-detached'
])(
'handles %s input without leaking or replaying abandoned text',
async (scenario) => {
const root = mkdtempSync(join(tmpdir(), 'orca-guest-text-'))
try {
const mainPath = join(root, 'main.cjs')
await build({
stdin: { contents: MAIN, resolveDir: process.cwd(), loader: 'ts' },
outfile: mainPath,
bundle: true,
platform: 'node',
format: 'cjs',
external: ['electron', '*.node']
})
writeFileSync(
join(root, 'config.json'),
JSON.stringify({ resultPath: join(root, 'result.json'), scenario })
)
const result = await runBrowserRouteEgressElectron(root, mainPath)
if (scenario.startsWith('child-')) {
expect(result).toEqual({
outcome:
scenario === 'child-insert'
? 'ok'
: scenario === 'child-expired'
? 'browser_timeout'
: 'browser_input_target_unavailable',
after:
scenario === 'child-insert' ? 'Child 🙏' : scenario === 'child-detached' ? null : '',
root: 'Next 🙏',
host: 'Unsent draft',
hostFocus: 'draft'
})
return
}
if (scenario === 'editing') {
expect(result).toEqual({
selection: 'First\nSecond 🙏',
inputText: 'First\nSecond 🙏',
undo: 'Replace me',
cancelled: 'Replace me',
rich: 'Rich 🙏',
shadow: 'Shadow',
local: 'Local 🙏',
design: 'Design',
moved: 'browser_input_target_unavailable',
body: 'Body 🙏',
host: 'Unsent draft'
})
return
}
if (scenario === 'renderer-dies') {
expect(result).toEqual({ outcome: 'browser_error', host: 'Unsent draft' })
return
}
expect(result).toEqual({
crashed: false,
outcome: scenario.endsWith('expired')
? 'browser_timeout'
: 'browser_input_target_unavailable',
after: '',
next: 'Next 🙏',
host: 'Unsent draft'
})
} finally {
rmSync(root, { recursive: true, force: true })
}
},
45_000
)
@@ -11,6 +11,12 @@ import type { ClaudeCommandConfig, ClaudeCommandOptions } from './sign-in/claude
import type { ClaudeAccountSelectionTarget } from './selection/runtime-selection'
import { ClaudeAccountService } from './service'
// Selection checks must never reach the real Keychain.
vi.mock('../macos-keychain/generic-password', async (importOriginal) => ({
...(await importOriginal<object>()),
readKeychainPassword: async () => null
}))
const roots: string[] = []
afterEach(() => roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })))
@@ -190,4 +196,19 @@ describe('ClaudeAccountService', () => {
expect(f.runtimeAuth.removeAccountFolder).not.toHaveBeenCalled()
expect(f.settings().claudeManagedAccounts.map((entry) => entry.id)).toEqual(['a', 'b'])
})
it('refuses to select an account that has its identity but no login to start Claude with', async () => {
const f = fixture()
f.signIn('b', 'b@example.test')
await expect(f.service.selectAccount('b')).rejects.toThrow(
'Sign in to this Claude account again before selecting it.'
)
expect(f.settings().activeClaudeManagedAccountId).toBe('a')
writeFileSync(
join(f.home('b'), '.credentials.json'),
JSON.stringify({ claudeAiOauth: { refreshToken: 'refresh' } })
)
await expect(f.service.selectAccount('b')).resolves.toMatchObject({ activeAccountId: 'b' })
})
})
@@ -6,6 +6,8 @@ import type {
import type { GlobalSettings } from '../../../shared/global-settings-types'
import type { RateLimitService } from '../../rate-limits/service/rate-limit-service'
import { claudeStateFile, readClaudeFolderLogin } from '../sign-in/claude-account-folder'
import { readClaudeOAuthCredentials } from '../../rate-limits/providers/claude/claude-oauth-credentials'
import { resolveLoginShellEnvironment } from '../../startup/shell-environment/login-shell-environment'
import { savedWslClaudeAccountHome } from '../profiles/wsl/claude-profile-wsl-paths'
import type { ClaudeRuntimeAuthService } from '../runtime-auth-service'
import {
@@ -193,8 +195,13 @@ export class ClaudeAccountSelection {
) {
throw new Error('That Claude account belongs to a different runtime.')
}
// CLI and paired clients bypass the renderer's sign-in check.
if (this.summarize(account).needsSignIn) {
// CLI and paired clients bypass the renderer's sign-in check. An identity file alone can't
// start Claude, and chats follow the selection on their next start.
const summary = this.summarize(account)
if (
summary.needsSignIn ||
(summary.home && !summary.runsOnSystemDefault && (await accountHoldsNoLogin(summary.home)))
) {
throw new Error(CLAUDE_ACCOUNT_NEEDS_SIGN_IN_MESSAGE)
}
effectiveTarget = accountTarget
@@ -271,3 +278,16 @@ export class ClaudeAccountSelection {
}
}
}
/** The read usage already makes for this account; an unreadable store is no answer. */
async function accountHoldsNoLogin(home: string): Promise<boolean> {
// Claude then keeps every login elsewhere, so an empty account folder proves nothing.
if ((await resolveLoginShellEnvironment()).CLAUDE_SECURESTORAGE_CONFIG_DIR !== undefined) {
return false
}
const login = await readClaudeOAuthCredentials({
credentialsFileConfigDir: home,
keychainConfigDir: home
})
return !login.token && !login.hasRefreshableCredentials && !login.unavailable
}
@@ -138,7 +138,8 @@ export function projectCodexSessionOptions(
/** Capture only the immutable capability fields consumed by the message compiler. */
export function captureCodexMessageCompilerInputs(
session: Parameters<typeof codexSessionCapabilities>[0]
session: Parameters<typeof codexSessionCapabilities>[0] &
Partial<Pick<CodexSession, 'appliedTurnInputOptions'>>
): CodexMessageCompilerInputs {
const capabilities = codexSessionCapabilities(session)
const defaults: Record<string, string> = {}
@@ -157,6 +158,7 @@ export function captureCodexMessageCompilerInputs(
}
return {
active: { ...session.appliedTurnOptions },
activeInput: { ...session.appliedTurnInputOptions },
defaults,
models: capabilities.models.map((model) => ({
id: model.id,
@@ -158,6 +158,7 @@ export class CodexStructuredSessionAdapter implements SessionAdapter {
return admission
}
if (event.type === 'notification') {
turnEnd.publishCodexTurnOpened(session, event, this.sessions)
// Only an admitted turn end settles; a refused one settles on the retry that lands.
turnEnd.settleCodexSendsInEndedTurn(session, event)
// Child records observe refused frames only after a retry reaches the journal.
@@ -338,7 +339,6 @@ export class CodexStructuredSessionAdapter implements SessionAdapter {
hasOwnedAcquisition = (sessionId: string): boolean =>
this.acquisitions.get(sessionId) !== undefined || this.sessions.has(sessionId)
private session(sessionId: string): CodexSession {
return requireLiveCodexSession(this.sessions, sessionId)
}
private session = (sessionId: string): CodexSession =>
requireLiveCodexSession(this.sessions, sessionId)
}
@@ -28,6 +28,7 @@ import {
reportedCodexSessionOptions
} from '../catalog/codex-structured-session-options'
import { CodexSkillReport } from '../catalog/codex-structured-skill-report'
import { codexMessageInputOptions } from '../../../../shared/agent-session/options/codex-message-compiler'
import { codexAcquireCatalogAccess } from '../catalog/codex-structured-acquire-catalog'
import {
CODEX_DEFAULT_SERVICE_TIER,
@@ -187,6 +188,10 @@ export async function openCodexStructuredSession(
...(opened.effort ? { effort: opened.effort } : {}),
...(serviceTier ? { serviceTier } : {})
},
appliedTurnInputOptions: codexMessageInputOptions(input.options ?? {}, {
serviceTier: CODEX_DEFAULT_SERVICE_TIER,
interactionMode: 'default'
}),
appliedSettings: attempt.window.appliedSettings,
...(catalogAccess ? { catalogAccess } : {}),
dispatchEchoes: start.dispatchEchoes,
@@ -178,6 +178,8 @@ export type CodexSession = {
nativeDefaults?: { personality?: string; serviceTier?: string }
/** Acquisition facts overlaid by controls actually sent, independently of reports or saved picks. */
appliedTurnOptions?: Readonly<Record<string, string>>
/** Saved/start input choices before protocol shaping; later catalogs cannot change eligibility. */
appliedTurnInputOptions?: Readonly<Record<string, string>>
appliedSettings?: CodexAppliedSettings
/** Optional for adapters without a host catalog store. */
catalogAccess?: CodexSessionCatalogAccess
@@ -4,7 +4,10 @@ import { CodexAppliedSettings } from '../catalog/codex-structured-applied-settin
import { CodexAppServerRequestError } from '../../app-server/codex-app-server-request-error'
import { changeCodexThreadGoal } from './codex-structured-thread-goal'
import type { CodexSession } from '../session-lifecycle/codex-structured-session-state'
import { waitsForCodexTurnOptions } from '../../../native-chat/agent-session-wire/options/structured-agent-session-message-options'
import {
codexTurnOptionsDiffer,
waitsForCodexTurnOptions
} from '../../../native-chat/agent-session-wire/options/structured-agent-session-message-options'
const THREAD = 'thread-1'
@@ -89,30 +92,16 @@ describe('direct goal settings application', () => {
threadId: 'thread-abc',
threadSettings: { model, effort }
})
const goal = (model = 'gpt-live', effort?: string) =>
const goal = (model = 'gpt-live', effort?: string, signal?: AbortSignal) =>
adapter.changeThreadGoal({
sessionId: 'session-1',
fence: 7,
change: { kind: 'status', status: 'active' },
replacesGoal: false,
executionOptions: { model, ...(effort ? { effort } : {}) }
executionOptions: { model, ...(effort ? { effort } : {}) },
signal
})
const state = acquiredSettingsTestSeam(adapter)
return { codex, adapter, apply, goal, settings: state.settings, state, acquisition }
}
it('keeps literal goal settings separate from the applied turn settings that hold a follow-up, and recovers on a successful turn', async () => {
const { codex, adapter, apply, goal, acquisition } = await fixture()
const choices = { model: 'unlisted-model-a', effort: 'high' }
const dispatch = () =>
adapter.dispatch({
sessionId: 'session-1',
fence: 7,
clientMessageId: `message-${codex.connections[0].calls.length}`,
body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'Continue' }] },
executionOptions: choices
})
const held = (options = choices) =>
const held = (options: Record<string, string>) =>
waitsForCodexTurnOptions(
{ provider: 'codex', sessionId: 'session-1' },
{
@@ -134,8 +123,37 @@ describe('direct goal settings application', () => {
{ fence: 7, generation: acquisition.acquisitionGeneration, phase: 'ready' },
adapter
)
return {
codex,
adapter,
apply,
goal,
held,
get settings() {
return acquiredSettingsTestSeam(adapter).settings
},
get state() {
return acquiredSettingsTestSeam(adapter)
},
acquisition
}
}
it('preserves goal choice eligibility through readback, refusal and successful-turn recovery', async () => {
const { codex, adapter, apply, goal, held: heldOptions } = await fixture()
const choices = { model: 'unlisted-model-a', effort: 'high' }
const dispatch = () =>
adapter.dispatch({
sessionId: 'session-1',
fence: 7,
clientMessageId: `message-${codex.connections[0].calls.length}`,
body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'Continue' }] },
executionOptions: choices
})
const held = (options = choices) => heldOptions(options)
try {
expect(await dispatch()).toMatchObject({ state: 'admitted' })
expect(await held()).toBe(false)
codex.routes['thread/settings/update'] = (params) => {
apply(String(params?.model), String(params?.effort))
return {}
@@ -149,12 +167,18 @@ describe('direct goal settings application', () => {
params: { threadId: 'thread-abc', model: 'unlisted-model-b', effort: 'low' }
}
])
const appliedGoal = { model: 'unlisted-model-b', effort: 'low', interactionMode: 'default' }
const goalChoices = { model: 'unlisted-model-b', effort: 'low' }
expect(await held(goalChoices)).toBe(false)
expect(await held()).toBe(true)
apply('observation-only', 'medium')
expect(await held(goalChoices)).toBe(false)
expect(await held()).toBe(true)
codex.routes['thread/settings/update'] = () => {
throw new CodexAppServerRequestError('thread/settings/update', -32602, 'invalid model')
}
expect(await goal('refused-model', 'high')).toMatchObject({ ok: false })
expect(await held(goalChoices)).toBe(false)
expect(await held()).toBe(true)
expect(
codex.connections[0].calls.filter((call) => call.method === 'thread/goal/set')
).toHaveLength(1)
@@ -163,7 +187,8 @@ describe('direct goal settings application', () => {
(call) => call.method === 'thread/settings/update'
).length
expect(await goal('observation-only-c', 'medium')).toEqual({ ok: true })
expect(await held(appliedGoal)).toBe(false)
expect(await held()).toBe(true)
expect(await held(goalChoices)).toBe(false)
expect(await held({ model: 'observation-only-c', effort: 'medium' })).toBe(true)
expect(
codex.connections[0].calls.filter((call) => call.method === 'thread/settings/update')
@@ -174,25 +199,147 @@ describe('direct goal settings application', () => {
expect(await goal('uncertain-model', 'low')).toMatchObject({ ok: false })
expect(await held()).toBe(true)
expect(await dispatch()).toMatchObject({ state: 'admitted' })
expect(await held()).toBe(false)
codex.routes['thread/settings/update'] = () => {
throw new CodexAppServerRequestError('thread/settings/update', -32601, 'method not found')
}
expect(await goal('unlisted-model-b', 'low')).toEqual({ ok: true })
expect(await held()).toBe(false)
const callsBeforeCached = codex.connections[0].calls.filter(
(call) => call.method === 'thread/settings/update'
).length
expect(await goal('unlisted-model-b', 'low')).toEqual({ ok: true })
expect(await held()).toBe(false)
expect(await held(appliedGoal)).toBe(true)
expect(await held(goalChoices)).toBe(true)
expect(
codex.connections[0].calls.filter((call) => call.method === 'thread/settings/update')
).toHaveLength(callsBeforeCached)
expect(await dispatch()).toMatchObject({ state: 'admitted' })
expect(await held()).toBe(false)
} finally {
await adapter.closeAll()
}
})
it.each([
['success', false],
['refusal', false],
['success', true],
['refusal', true]
] as const)(
'preserves a newer start after late goal %s (replaced acquisition: %s)',
async (outcome, replaceAcquisition) => {
const { codex, adapter, goal, acquisition } = await fixture()
let generation = acquisition.acquisitionGeneration
const dispatch = (model: string) =>
adapter.dispatch({
sessionId: 'session-1',
fence: 7,
clientMessageId: model,
body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'Continue' }] },
executionOptions: { model, effort: 'high' }
})
const answer = Promise.withResolvers<unknown>()
const authority = new AbortController()
try {
expect(await dispatch('model-a')).toMatchObject({ state: 'admitted' })
const request = vi.spyOn(codex.connections[0], 'request')
codex.routes['thread/settings/update'] = () => answer.promise
const pending = goal('model-b', 'high', authority.signal)
const rpc = request.mock.results.at(-1)?.value
expect(rpc).toBeInstanceOf(Promise)
authority.abort(new Error('stopped'))
expect(await pending).toMatchObject({ ok: false })
if (replaceAcquisition) {
await adapter.closeSession('session-1')
const { acquireReadyCodexForTest, identityFor } =
await import('../session-lifecycle/codex-structured-session-adapter-fixture')
const replacement = await acquireReadyCodexForTest(adapter, {
identity: identityFor('session-1'),
fence: 7,
spawnToken: 'replacement'
})
generation = replacement.acquisitionGeneration
expect(
adapter.captureMessageSteeringInputs({
sessionId: 'session-1',
fence: 7,
acquisitionGeneration: acquisition.acquisitionGeneration
})
).toEqual({ kind: 'codex-unavailable' })
}
expect(await dispatch('model-c')).toMatchObject({ state: 'admitted' })
if (outcome === 'success') {
answer.resolve({})
} else {
answer.reject(
new CodexAppServerRequestError('thread/settings/update', -32602, 'invalid model')
)
}
await Promise.allSettled([rpc])
const differs = (model: string) =>
codexTurnOptionsDiffer(
{ provider: 'codex', sessionId: 'session-1' },
{ model, effort: 'high' },
{ fence: 7, generation, phase: 'ready' },
adapter
)
expect(differs('model-c')).toBe(false)
expect(differs('model-a')).toBe(true)
expect(differs('model-b')).toBe(true)
expect(
codex.connections
.flatMap((connection) => connection.calls)
.some((call) => call.method.startsWith('thread/goal/'))
).toBe(false)
} finally {
await adapter.closeAll()
}
}
)
it.each(['success', 'refusal'] as const)(
'settles an owned late %s after abort without resuming the goal',
async (outcome) => {
const { codex, adapter, goal, held } = await fixture()
const answer = Promise.withResolvers<unknown>()
const authority = new AbortController()
try {
await adapter.dispatch({
sessionId: 'session-1',
fence: 7,
clientMessageId: 'prior',
body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'Continue' }] },
executionOptions: { model: 'model-a', effort: 'high' }
})
const request = vi.spyOn(codex.connections[0], 'request')
codex.routes['thread/settings/update'] = () => answer.promise
const pending = goal('model-b', 'high', authority.signal)
const rpc = request.mock.results.at(-1)?.value
expect(rpc).toBeInstanceOf(Promise)
authority.abort(new Error('stopped'))
expect(await pending).toMatchObject({ ok: false })
if (outcome === 'success') {
answer.resolve({})
} else {
answer.reject(
new CodexAppServerRequestError('thread/settings/update', -32602, 'invalid model')
)
}
await Promise.allSettled([rpc])
expect(await held({ model: 'model-b', effort: 'high' })).toBe(outcome !== 'success')
expect(await held({ model: 'model-a', effort: 'high' })).toBe(outcome === 'success')
expect(
codex.connections
.flatMap((connection) => connection.calls)
.some((call) => call.method.startsWith('thread/goal/'))
).toBe(false)
} finally {
await adapter.closeAll()
}
}
)
it.each(['success', 'rejection', 'stale'] as const)(
'rechecks current application after its RPC reply, preserving pre-reply evidence (%s)',
async (mode) => {
@@ -120,7 +120,16 @@ export async function changeCodexThreadGoalWithSettings(
try {
if (startsWork) {
const requested = resolvedCodexGoalSettings(session, input.executionOptions)
const previousApplied = session.appliedTurnOptions
let previousApplied = session.appliedTurnOptions
let previousInput = session.appliedTurnInputOptions
let invalidatedApplied: CodexSession['appliedTurnOptions']
let invalidatedInput: CodexSession['appliedTurnInputOptions']
const ownsAcquisition = () =>
current() &&
!session.ended &&
!session.connection.closed &&
session.fence === input.fence &&
session.acquisitionGeneration === generation
const installed = await session.appliedSettings?.install(
session.connection,
session.threadId,
@@ -128,19 +137,44 @@ export async function changeCodexThreadGoalWithSettings(
timeoutMs,
input.signal,
() => {
session.appliedTurnOptions = { ...session.appliedTurnOptions, ...requested }
if (!ownsAcquisition()) {
return
}
if (session.appliedTurnOptions === invalidatedApplied) {
session.appliedTurnOptions = { ...invalidatedApplied, ...requested }
}
if (session.appliedTurnInputOptions === invalidatedInput) {
session.appliedTurnInputOptions = { ...invalidatedInput, ...requested }
}
},
() => {
session.appliedTurnOptions = previousApplied
if (!ownsAcquisition()) {
return
}
if (session.appliedTurnOptions === invalidatedApplied) {
session.appliedTurnOptions = previousApplied
}
if (session.appliedTurnInputOptions === invalidatedInput) {
session.appliedTurnInputOptions = previousInput
}
},
() => {
previousApplied = session.appliedTurnOptions
previousInput = session.appliedTurnInputOptions
const known = { ...previousApplied }
const knownInput = { ...previousInput }
for (const key of ['model', 'effort'] as const) {
if (requested[key] !== undefined && requested[key] !== known[key]) {
delete known[key]
}
if (requested[key] !== undefined && requested[key] !== knownInput[key]) {
delete knownInput[key]
}
}
session.appliedTurnOptions = known
invalidatedApplied = known
invalidatedInput = knownInput
session.appliedTurnOptions = invalidatedApplied
session.appliedTurnInputOptions = invalidatedInput
}
)
if (installed && !installed.ok) {
@@ -1,7 +1,13 @@
import { describe, expect, it } from 'vitest'
import { describe, expect, it, vi } from 'vitest'
import type { AgentJournalItemBody } from '../../../../shared/agent-session/journal/agent-session-journal-types'
import { agentJournalSubmissionKey } from '../../../../shared/agent-session/journal/agent-session-journal-item-key'
import type { StructuredAgentSessionEventSink } from '../../../native-chat/agent-session-wire/journal-ingress/structured-agent-session-event-sink'
import {
createDeferredStructuredAgentSessionEventSink,
type StructuredAgentSessionEventSink
} from '../../../native-chat/agent-session-wire/journal-ingress/structured-agent-session-event-sink'
import { createFixture } from '../../../native-chat/agent-session-journal/store-lifecycle/journal-store-contract-test-fixture'
import { testEventSinkLogging } from '../../../native-chat/agent-session-wire/host/testing/structured-agent-session-logger.test-fixture'
import { StructuredAgentSessionCurrentWork } from '../../../native-chat/agent-session-wire/observations/structured-agent-session-current-work'
import { MAX_CODEX_PENDING_DISPATCH_ECHOES } from './codex-structured-dispatch-echo'
import {
acquiredCodexAdapter,
@@ -43,6 +49,26 @@ function lifecycleRecorder(): {
}
describe('codex dispatch admission', () => {
it('releases acknowledged steers without an echo and retains each late receipt', async () => {
const codex = fakeCodexAppServer({ 'turn/steer': () => ({ turnId: 'turn-1' }) })
const settlements: LateSettlement[] = []
const adapter = await acquiredCodexAdapter({ codex, settlements })
const connection = codex.connections[0]!
startTurn(connection, 'turn-1')
expect(await send(adapter, 'client-1')).toMatchObject({ state: 'admitted' })
expect(adapter.heldSends('session-1')).toEqual(new Set())
expect(await send(adapter, 'client-2')).toMatchObject({ state: 'admitted' })
expect(adapter.heldSends('session-1')).toEqual(new Set())
echoUserMessage(connection, { turnId: 'turn-1', itemId: 'late-1', clientId: 'client-1' })
echoUserMessage(connection, { turnId: 'turn-1', itemId: 'late-2', clientId: 'client-2' })
expect(settlements.map(({ clientMessageId }) => clientMessageId)).toEqual([
'client-1',
'client-2'
])
expect(connection.calls.filter(({ method }) => method === 'turn/steer')).toHaveLength(2)
await adapter.closeSession('session-1')
})
it('correlates each send by client message id, not queue order', async () => {
const codex = fakeCodexAppServer({ 'turn/steer': () => ({ turnId: 'turn-1' }) })
const settlements: LateSettlement[] = []
@@ -190,3 +216,96 @@ describe('codex dispatch admission', () => {
expect(settlements).toEqual([])
})
})
describe('ordered Codex admission publication', () => {
const fixture = createFixture()
it('holds a replied start through refused and delayed journal writes, then releases without an echo', async () => {
const journal = await fixture.open()
const deferred = createDeferredStructuredAgentSessionEventSink(testEventSinkLogging())
deferred.bind({ journal, fence: 7, publish: () => {} })
let refuse = true
const revise = deferred.sink.tryReviseResolvedItem!
deferred.sink.tryReviseResolvedItem = (...args) =>
refuse ? { accepted: false, reason: 'backpressure' } : revise(...args)
const write = journal.appendSteps.bind(journal)
const delay = Promise.withResolvers<void>()
let delayed = false
vi.spyOn(journal, 'appendSteps').mockImplementation(async (...args) => {
delayed = true
await delay.promise
return write(...args)
})
const codex = fakeCodexAppServer({ 'turn/start': () => ({ turn: { id: 'ordered-turn' } }) })
let observeWork: (() => Promise<StructuredAgentSessionCurrentWork>) | undefined
const observed: Promise<{ active: string | null; working: boolean }>[] = []
const adapter = await acquiredCodexAdapter({
codex,
settlements: [],
sink: deferred.sink,
onHeldSendsChanged: () => {
if (observeWork) {
observed.push(
observeWork().then((work) => ({ active: work.activeTurnId(), working: work.working() }))
)
}
}
})
const work = async () => {
const { state } = await journal.readWork(
() => ({
handle: journal.observationHandle(),
record: {
currentFence: 7,
ownerObservation: { claimStatus: 'live', unreconciled: false, handoffStage: null }
},
child: { fence: 7, heldSends: [...(adapter.heldSends('session-1') ?? [])] },
handingOver: [],
continuingUnadmitted: [],
finalizing: [],
openedThrough: journal.openingCursor(),
operationalRevision: 0
}),
{ kind: 'state' }
)
return new StructuredAgentSessionCurrentWork(journal, state, journal.cursor())
}
observeWork = work
await journal.appendSubmission({
clientMessageId: 'ordered',
payloadFingerprint: 'ordered-input',
body: CODEX_TEST_USER_MESSAGE,
fence: 7
})
await send(adapter, 'ordered')
const submission = (await journal.reads.submission('ordered'))!
expect((await work()).publishedSubmission(submission)).toMatchObject({
dispatchState: 'pending'
})
expect((await work()).working()).toBe(true)
startTurn(codex.connections[0]!, 'ordered-turn')
expect((await work()).activeTurnId()).toBeNull()
expect((await work()).working()).toBe(true)
refuse = false
await vi.waitFor(() => expect(delayed).toBe(true))
expect((await work()).activeTurnId()).toBeNull()
expect((await work()).publishedSubmission(submission)).toMatchObject({
dispatchState: 'pending'
})
delay.resolve()
await deferred.sink.written!()
await vi.waitFor(async () =>
expect((await work()).publishedSubmission(submission)).toMatchObject({
dispatchState: 'unknown',
recovered: true,
reason: 'provider_write_outcome_unknown'
})
)
expect((await work()).activeTurnId()).toBe('ordered-turn')
expect((await work()).working()).toBe(true)
expect(await Promise.all(observed)).toEqual([{ active: 'ordered-turn', working: true }])
await adapter.closeSession('session-1')
deferred.close()
})
})
@@ -1,4 +1,4 @@
import type { CodexTurnHost } from './codex-structured-turn-start'
import type { CodexTurnHost } from './codex-structured-turn-contract'
import { StructuredAgentSessionInputNotAdmitted } from '../../../native-chat/agent-session-wire/delivery/structured-agent-session-input-admission'
import { codexDispatchRejection } from './codex-structured-turn-end-settlement'
import { agentSessionFailureFact } from '../../../../shared/agent-session/failures/agent-session-failure'
@@ -73,7 +73,7 @@ it.each(['start', 'steer'] as const)(
describe('the sends Codex holds', () => {
const held = (echoes: ReturnType<typeof createCodexDispatchEchoes>) => [...echoes.heldSends()]
it('holds a send from its write until the turn it was answered into ends, echo or not', () => {
it('holds a replied start until its turn is published without consuming its echo', () => {
const changes: string[] = []
const echoes = createCodexDispatchEchoes(() => changes.push(held(echoes).join(',')))
echoes.arm('sent')
@@ -82,12 +82,34 @@ describe('the sends Codex holds', () => {
echoes.bindTurn('sent', 'thread-1', 'turn-1', 'start', echoes.tick())
expect(held(echoes)).toEqual(['sent'])
// A hook can refuse input and complete the turn without echoing it.
echoes.endTurn('thread-1', 'turn-1', { status: 'completed' })
echoes.opened('thread-1', 'turn-1')
expect(held(echoes)).toEqual(['sent'])
echoes.published('thread-1', 'turn-1')
expect(held(echoes)).toEqual([])
expect(changes).toEqual([''])
})
it('releases a replied steer into a published turn while retaining exact late correlation', () => {
const echoes = createCodexDispatchEchoes()
echoes.published('thread-1', 'turn-1')
echoes.arm('steered')
echoes.expect('steered', 'thread-1', 'steer', [], 'turn-1')
expect(held(echoes)).toEqual(['steered'])
echoes.bindTurn('steered', 'thread-1', 'turn-1', 'steer', echoes.tick())
expect(held(echoes)).toEqual([])
expect(
readCodexDispatchEcho(
{
type: 'userMessage',
id: 'late-user',
clientId: 'steered'
},
CODEX_IDENTITY,
echoes
)?.clientMessageId
).toBe('steered')
})
it('holds a send answered into no turn only while a status read after its answer says running', () => {
const echoes = createCodexDispatchEchoes()
echoes.arm('timed-out')
@@ -8,10 +8,7 @@ import type { ProviderDiagnostic } from '../../../../shared/agent-session/failur
import type { AgentJournalItemIdentity } from '../../../../shared/agent-session/journal/agent-session-journal-identity'
import type { AgentJournalTurnJoin } from '../../../../shared/agent-session/journal/agent-session-journal-types'
export {
MAX_CODEX_PENDING_DISPATCH_ECHOES,
MAX_CODEX_RECORDED_TURN_ENDS
} from './codex-dispatch-receipt-correlation'
export { MAX_CODEX_PENDING_DISPATCH_ECHOES, MAX_CODEX_RECORDED_TURN_ENDS }
export type CodexTurnEnd =
| { status: 'completed' }
@@ -31,7 +28,7 @@ export type CodexDispatchEchoes = {
answered: (clientMessageId: string, at: number) => void
/** Only status received after a no-turn answer determines whether that send is held. */
observeStatus: (active: boolean) => void
/** Sends still executing; ended turns release them even without an echo. */
/** Sends awaiting admission to a published turn; exact echoes remain independently armed. */
heldSends: () => ReadonlySet<string>
/** False at capacity preserves older pending sends. */
arm: (clientMessageId: string, requestedAt?: number) => boolean
@@ -63,6 +60,8 @@ export type CodexDispatchEchoes = {
/** Mark a timed-out open wait so later sends do not wait again. */
leftUnopened: (threadId: string, turnId: string) => void
opened: (threadId: string, turnId: string) => void
/** The turn's admitted journal writes landed; receipt observation alone cannot release work. */
published: (threadId: string, turnId: string) => void
/** An error naming a never-opened turn counts as its end. */
hasOpened: (threadId: string, turnId: string) => boolean
/** Latest answered turn still unopened after its wait was exhausted. */
@@ -105,12 +104,14 @@ export function createCodexDispatchEchoes(
>()
let clock = 0
let status: { active: boolean; at: number } | null = null
const publishedTurns = new Set<string>()
const heldSends = (): Set<string> =>
new Set(
[...armed].flatMap(([clientMessageId, entry]) =>
entry.answeredAt === undefined ||
(entry.turn
? entry.turnEnded !== true
? entry.turnEnded !== true &&
!publishedTurns.has(turnKey(entry.turn.threadId, entry.turn.turnId))
: status?.active === true && status.at > entry.answeredAt)
? [clientMessageId]
: []
@@ -224,6 +225,16 @@ export function createCodexDispatchEchoes(
openedTurns.delete(oldest)
}
},
published: (threadId, turnId) =>
reporting(() => {
publishedTurns.add(turnKey(threadId, turnId))
for (const oldest of publishedTurns) {
if (publishedTurns.size <= MAX_CODEX_RECORDED_TURN_ENDS) {
break
}
publishedTurns.delete(oldest)
}
}),
hasOpened: (threadId, turnId) => openedTurns.has(turnKey(threadId, turnId)),
endTurn: (threadId, turnId, end) =>
reporting(() => {
@@ -286,6 +297,7 @@ export function createCodexDispatchEchoes(
receiptCorrelation.clear()
unopenedTurns.clear()
openedTurns.clear()
publishedTurns.clear()
nextSequence = 0
},
get size() {
@@ -99,6 +99,7 @@ export async function acquiredCodexAdapter(input: {
sink?: StructuredAgentSessionEventSink
requestTimeoutMs?: number
permissionMode?: AgentChatPermissionChoice
onHeldSendsChanged?: CodexStructuredSessionAdapterDeps['onHeldSendsChanged']
}): Promise<CodexStructuredSessionAdapter> {
const adapter = new CodexStructuredSessionAdapter({
resolveLaunch: async () => ({
@@ -114,6 +115,7 @@ export async function acquiredCodexAdapter(input: {
onDispatchSettledLate: (settlement) => {
input.settlements.push(settlement)
},
...(input.onHeldSendsChanged ? { onHeldSendsChanged: input.onHeldSendsChanged } : {}),
...(input.requestTimeoutMs === undefined ? {} : { requestTimeoutMs: input.requestTimeoutMs })
})
const identity: AgentSessionJournalIdentity = {
@@ -1,6 +1,6 @@
import {
structuredMessageSteeringOptionsDiffer,
compileCodexMessageOptions
compileCodexMessageOptions,
structuredMessageSteeringOptionsDiffer
} from '../../../../shared/agent-session/options/codex-message-compiler'
import { CODEX_SESSION_OPTION_CATALOG } from '../../../../shared/agent-session/options/agent-session-option-catalog-claude-codex'
import {
@@ -14,14 +14,28 @@ import { describe, expect, it, vi } from 'vitest'
import { AgentModelCatalogStore } from '../../../native-chat/agent-model-catalog/agent-model-catalog-store'
import { optionSession } from '../catalog/codex-structured-option-session.test-fixture'
import {
resolvedCodexMessageOptions,
readCodexAppliedInputOptions,
readCodexMessageSteeringInputs
readCodexMessageSteeringInputs,
resolvedCodexMessageOptions
} from './codex-structured-message-options'
import {
codexTurnOptionsDiffer,
waitsForCodexTurnOptions
} from '../../../native-chat/agent-session-wire/options/structured-agent-session-message-options'
import { startCodexTurn } from './codex-structured-turn-start'
import { CodexAppServerRequestError } from '../../app-server/codex-app-server-request-error'
import { CodexAppliedSettings } from '../catalog/codex-structured-applied-settings'
import { changeCodexThreadGoalWithSettings } from '../threads/codex-structured-thread-goal'
import type { CodexSession } from '../session-lifecycle/codex-structured-session-state'
import { USER_MESSAGE } from '../session-lifecycle/codex-structured-session-adapter-fixture'
import {
USER_MESSAGE,
fakeCodex,
adapterFor,
acquireReadyCodexForTest,
identityFor,
answerWithOpenedTurn
} from '../session-lifecycle/codex-structured-session-adapter-fixture'
import type { AgentJournalItemBody } from '../../../../shared/agent-session/journal/agent-session-journal-types'
function seedMessageDefaults(session: CodexSession): void {
const store = new AgentModelCatalogStore()
@@ -37,7 +51,7 @@ function seedMessageDefaults(session: CodexSession): void {
label: 'Model A',
isDefault: false,
defaultEffort: 'low',
serviceTiers: [{ value: 'priority', label: 'Priority' }],
serviceTiers: [{ value: 'priority', label: 'Fast' }],
efforts: [
{ value: 'low', label: 'Low' },
{ value: 'high', label: 'High' }
@@ -220,11 +234,246 @@ it('sends native Plan and Build on fresh turns while a steer keeps its running m
expect(request.mock.calls.at(-1)?.[1]).not.toHaveProperty('collaborationMode')
})
it.each(['priority', 'default'])(
'keeps the same priority input eligible after an actual %s start and catalog change',
async (initialTier) => {
const request = vi.fn(async () => ({ turn: { id: 'tier-turn' } }))
const session = optionSession(request)
const store = new AgentModelCatalogStore()
const catalog = (priority: boolean) =>
store.recordSuccess(
'input-intent',
'codex',
{
origin: 'probe',
models: [
{
id: 'gpt-live',
label: 'GPT',
isDefault: true,
efforts: [{ value: 'high', label: 'High' }],
serviceTiers: priority ? [{ value: 'priority', label: 'Fast' }] : []
}
]
},
'discovery'
)
session.catalogAccess = { store, fingerprint: 'input-intent', accountHomePath: '/test-account' }
session.reportedOptions = { model: 'gpt-live', effort: 'high', serviceTier: 'default' }
catalog(initialTier === 'priority')
const choices = { model: 'gpt-live', effort: 'high', serviceTier: 'priority' }
await startCodexTurn(session, {
clientMessageId: 'priority-input',
body: USER_MESSAGE,
executionOptions: choices,
turnOptions: await resolvedCodexMessageOptions(session, choices)
})
expect(request).toHaveBeenCalledWith(
'turn/start',
expect.objectContaining({
serviceTier: initialTier
}),
expect.anything()
)
catalog(false)
const differs = (next: Record<string, string>) =>
codexTurnOptionsDiffer(
{ provider: 'codex', sessionId: 'session-1' },
next,
{ fence: 1, generation: 'generation-1', phase: 'ready' },
{ captureMessageSteeringInputs: (input) => readCodexMessageSteeringInputs(session, input) }
)
expect(differs(choices)).toBe(false)
expect(differs({ ...choices, serviceTier: 'default' })).toBe(true)
}
)
it('holds Plan behind a successful sparse Default start, then starts Plan after completion', async () => {
const codex = fakeCodex()
const adapter = adapterFor(codex)
const bodies: AgentJournalItemBody[] = []
const acquisition = await acquireReadyCodexForTest(adapter, {
identity: identityFor('session-1'),
fence: 7,
spawnToken: 'spawn',
events: {
appendItem: (_identity, body) => bodies.push(body),
appendTombstone: () => {},
publish: () => {}
}
})
const dispatch = (id: string, options: Record<string, string>) =>
adapter.dispatch({
sessionId: 'session-1',
fence: 7,
clientMessageId: id,
body: USER_MESSAGE,
executionOptions: options
})
const complete = (turnId: string) =>
codex.connections[0].handlers.onNotification?.('turn/completed', {
threadId: 'thread-abc',
turn: { id: turnId, status: 'completed' }
})
const held = (interactionMode: 'default' | 'plan') =>
waitsForCodexTurnOptions(
{ provider: 'codex', sessionId: 'session-1' },
{
reads: {
messageOptions: async () => ({
ordinaryMessage: true,
runningTurn: bodies.findLast((body) => body.kind === 'turn')?.state === 'running'
})
}
},
{
clientMessageId: 'next-plan',
fence: 7,
payloadFingerprint: 'fingerprint',
dispatchState: 'pending',
providerItemId: null,
reason: null,
submittedAt: 1,
resolvedAt: null,
options: { interactionMode }
},
{ fence: 7, generation: acquisition.acquisitionGeneration, phase: 'ready' },
adapter
)
try {
codex.routes['turn/start'] = answerWithOpenedTurn(codex, 'plan-first')
expect(await dispatch('first', { interactionMode: 'plan' })).toMatchObject({
state: 'admitted'
})
complete('plan-first')
codex.routes['turn/start'] = answerWithOpenedTurn(codex, 'default-second')
expect(await dispatch('second', {})).toMatchObject({ state: 'admitted' })
expect(
codex.connections[0].calls.findLast((call) => call.method === 'turn/start')?.params
).toMatchObject({ collaborationMode: { mode: 'default' } })
expect(await held('plan')).toBe(true)
expect(await held('default')).toBe(false)
complete('default-second')
expect(await held('plan')).toBe(false)
codex.routes['turn/start'] = answerWithOpenedTurn(codex, 'plan-third')
expect(await dispatch('third', { interactionMode: 'plan' })).toMatchObject({
state: 'admitted'
})
expect(
codex.connections[0].calls.findLast((call) => call.method === 'turn/start')?.params
).toMatchObject({ collaborationMode: { mode: 'plan' } })
} finally {
await adapter.closeAll()
}
})
it('uses exact successful start controls when the first message omits model and effort', async () => {
const session = optionSession(vi.fn(async () => ({ turn: { id: 'first-turn' } })))
await startCodexTurn(session, {
clientMessageId: 'first',
body: USER_MESSAGE,
executionOptions: {},
turnOptions: {}
})
const read = (next: Record<string, string>) =>
readCodexAppliedInputOptions(session, {
fence: 1,
acquisitionGeneration: 'generation-1',
next
})
expect(read({ effort: 'high' })).toMatchObject({
active: { model: 'gpt-live', effort: 'high' },
next: { model: 'gpt-live', effort: 'high' }
})
expect(read({ effort: 'low' })).toMatchObject({
active: { effort: 'high' },
next: { effort: 'low' }
})
})
it('inherits retained sparse intent ahead of canonical execution controls', async () => {
const session = optionSession(vi.fn(async () => ({ turn: { id: 'sparse-turn' } })))
session.appliedTurnOptions = { model: 'gpt-canonical', effort: 'high' }
session.appliedTurnInputOptions = { model: 'gpt-alias', effort: 'low' }
const choices = { model: 'gpt-alias' }
await startCodexTurn(session, {
clientMessageId: 'sparse',
body: USER_MESSAGE,
executionOptions: choices,
turnOptions: await resolvedCodexMessageOptions(session, choices)
})
expect(
readCodexAppliedInputOptions(session, {
fence: 1,
acquisitionGeneration: 'generation-1',
next: { model: 'gpt-alias', effort: 'low' }
})
).toMatchObject({
active: { model: 'gpt-alias', effort: 'low' },
next: { model: 'gpt-alias', effort: 'low' }
})
})
it('retains legacy fast intent as the successful native tier after a standard turn', async () => {
const session = optionSession(vi.fn(async () => ({ turn: { id: 'fast-turn' } })))
seedMessageDefaults(session)
session.appliedTurnInputOptions = { model: 'model-a', serviceTier: 'default' }
const choices = { model: 'model-a', fastMode: 'true' as const }
await startCodexTurn(session, {
clientMessageId: 'legacy-fast',
body: USER_MESSAGE,
executionOptions: choices,
turnOptions: await resolvedCodexMessageOptions(session, choices)
})
expect(Object.fromEntries(session.options)).toEqual({ model: 'model-a', serviceTier: 'priority' })
expect(
readCodexAppliedInputOptions(session, {
fence: 1,
acquisitionGeneration: 'generation-1',
next: { serviceTier: 'priority' }
})
).toMatchObject({ active: { serviceTier: 'priority' }, next: { serviceTier: 'priority' } })
})
it('invalidates uncertain goal choices independently from shaped execution evidence', async () => {
const session = optionSession(
vi.fn(async () => {
throw new Error('lost after write')
})
)
session.appliedSettings = new CodexAppliedSettings()
session.appliedSettings.observe({ model: 'gpt-canonical', effort: 'high' })
session.appliedTurnOptions = { model: 'gpt-canonical', effort: 'high' }
session.appliedTurnInputOptions = { model: 'gpt-alias', effort: 'low', serviceTier: 'default' }
expect(
await changeCodexThreadGoalWithSettings(
session,
{
sessionId: 'session-1',
fence: 1,
change: { kind: 'set', objective: 'Ship it' },
replacesGoal: false,
executionOptions: { model: 'gpt-canonical', effort: 'low' }
},
1000,
() => true
)
).toMatchObject({ ok: false })
expect(
readCodexAppliedInputOptions(session, {
fence: 1,
acquisitionGeneration: 'generation-1',
next: {}
})?.active
).toEqual({ effort: 'low', serviceTier: 'default' })
})
it('compiles copied live steering inputs with exact sparse defaults and tier semantics', async () => {
const session = optionSession(vi.fn(async () => ({})))
seedMessageDefaults(session)
session.options = new Map([['effort', 'high']])
session.appliedTurnOptions = { model: 'model-a', effort: 'low', serviceTier: 'default' }
session.appliedTurnInputOptions = { ...session.appliedTurnOptions }
const binding = { fence: 1, acquisitionGeneration: 'generation-1' }
const input = readCodexMessageSteeringInputs(session, binding)
if (input.kind !== 'codex') {
@@ -235,7 +484,7 @@ it('compiles copied live steering inputs with exact sparse defaults and tier sem
expect(await resolvedCodexMessageOptions(session, undefined)).toBeUndefined()
expect(
compileCodexMessageOptions(input.compiler, { model: 'model-a', fastMode: 'true' })
).toEqual({ model: 'model-a', serviceTier: 'default' })
).toEqual({ model: 'model-a', serviceTier: 'priority' })
expect(
compileCodexMessageOptions(input.compiler, { model: 'model-b', fastMode: 'true' })
).toEqual({ model: 'model-b', serviceTier: 'fast' })
@@ -265,7 +514,7 @@ it('compiles copied live steering inputs with exact sparse defaults and tier sem
kind: 'codex-unavailable'
})
expect(structuredMessageSteeringOptionsDiffer({ kind: 'codex-unavailable' }, undefined)).toBe(
true
false
)
expect(structuredMessageSteeringOptionsDiffer({ kind: 'codex-unavailable' }, {})).toBe(false)
expect(structuredMessageSteeringOptionsDiffer({ kind: 'other' }, { effort: 'high' })).toBe(false)
@@ -1,7 +1,7 @@
import { pickedSkillsInText } from '../../../../shared/agent-session/requests/structured-agent-session-send-mutation'
import type { AgentJournalMessageItem } from '../../../../shared/agent-session/journal/agent-session-journal-types'
import type { NativeChatBlock } from '../../../../shared/native-chat/native-chat-types'
import type { CodexTurnHost } from './codex-structured-turn-start'
import type { CodexTurnHost } from './codex-structured-turn-contract'
import { waitForPromiseWithSignal } from '../../../../shared/abort-signal-reason'
import { StructuredAgentSessionInputNotAdmitted } from '../../../native-chat/agent-session-wire/delivery/structured-agent-session-input-admission'
import { agentSessionFailureFact } from '../../../../shared/agent-session/failures/agent-session-failure'
@@ -12,6 +12,7 @@ import type { CodexSession } from '../session-lifecycle/codex-structured-session
import { captureCodexMessageCompilerInputs } from '../catalog/codex-structured-service-tier'
import {
compileCodexMessageOptions,
codexMessageInputOptions,
type StructuredMessageSteeringInputs
} from '../../../../shared/agent-session/options/codex-message-compiler'
@@ -36,7 +37,7 @@ export async function resolvedCodexMessageOptions(
return compileCodexMessageOptions(captureCodexMessageCompilerInputs(session), choices)
}
/** The same sparse controls as dispatch, compared under the exact acquisition that took them. */
/** Intent belongs to this exact acquisition; catalog/readback differences are not send barriers. */
export function readCodexAppliedInputOptions(
session: CodexSession | undefined,
input: {
@@ -51,19 +52,13 @@ export function readCodexAppliedInputOptions(
session.connection.closed ||
session.fence !== input.fence ||
session.acquisitionGeneration !== input.acquisitionGeneration ||
!session.appliedTurnOptions
!session.appliedTurnInputOptions
) {
return null
}
const active = session.appliedTurnOptions
const active = session.appliedTurnInputOptions
const requested = input.next ?? Object.fromEntries(session.options)
return {
active,
next: {
...active,
...compileCodexMessageOptions(captureCodexMessageCompilerInputs(session), requested)
}
}
return { active, next: codexMessageInputOptions(requested, active, session.appliedTurnOptions) }
}
/** An exact child-bound observation, with candidate-independent compiler inputs. */
@@ -77,7 +72,7 @@ export function readCodexMessageSteeringInputs(
session.connection.closed ||
session.fence !== input.fence ||
session.acquisitionGeneration !== input.acquisitionGeneration ||
!session.appliedTurnOptions
!session.appliedTurnInputOptions
) {
return { kind: 'codex-unavailable' }
}
@@ -0,0 +1,39 @@
import type { AgentSessionAccountKind } from '../../../../shared/agent-session/agent-session-availability'
import type { AgentJournalMessageItem } from '../../../../shared/agent-session/journal/agent-session-journal-types'
import type { AgentSessionExecutionOptions } from '../../../../shared/agent-session/options/agent-session-execution-options'
import type { StructuredAgentSessionAdmitFreshInput } from '../../../native-chat/agent-session-wire/delivery/structured-agent-session-input-admission'
import type { CodexAppServerConnection } from '../../app-server/codex-app-server-connection-types'
import type { CodexSkillReport } from '../catalog/codex-structured-skill-report'
import type { CodexSession } from '../session-lifecycle/codex-structured-session-state'
import type { CodexDispatchEchoes } from './codex-structured-dispatch-echo'
import type { CodexTurnOpenWaits } from './codex-structured-turn-open-wait'
export type CodexTurnHost = {
account?: AgentSessionAccountKind
skillReport?: Pick<CodexSkillReport, 'hasSkillPath'>
connection: Pick<CodexAppServerConnection, 'request'>
threadId: string
permissions?: CodexSession['permissions']
options: Map<string, string>
appliedTurnOptions?: Readonly<Record<string, string>>
appliedTurnInputOptions?: Readonly<Record<string, string>>
appliedSettings?: CodexSession['appliedSettings']
reportedOptions?: CodexSession['reportedOptions']
catalogAccess?: CodexSession['catalogAccess']
dispatchEchoes: CodexDispatchEchoes
activeTurnIds?: ReadonlySet<string>
/** Running turns whose interrupt Codex already answered: aborted, so never steered. */
abortedTurnIds?: ReadonlySet<string>
turnOpenWaits: Pick<CodexTurnOpenWaits, 'wait'>
}
export type CodexTurnInput = {
clientMessageId: string
body: AgentJournalMessageItem
requestedAt?: number
turnOptions?: Readonly<Record<string, string>>
executionOptions?: AgentSessionExecutionOptions
signal?: AbortSignal
isCurrent?: () => boolean
admitFreshInput?: StructuredAgentSessionAdmitFreshInput
}
@@ -97,6 +97,36 @@ export function noteCodexTurnOpened(
}
}
export function publishCodexTurnOpened(
session: CodexSession,
frame: { sessionId: string; method: string; params: unknown },
sessions: ReadonlyMap<string, CodexSession>
): void {
const { method, params } = frame
const turnId = method === 'turn/started' ? readCodexTurnId(params) : null
if (!turnId || (readCodexThreadId(params) ?? session.threadId) !== session.threadId) {
return
}
const publish = (): void => {
if (!session.ended && sessions.get(frame.sessionId) === session) {
session.dispatchEchoes.published(session.threadId, turnId)
}
}
const written = session.events?.written?.()
if (written) {
void written.then(
(result) => {
if (result.ok) {
publish()
}
},
() => {}
)
} else {
publish()
}
}
export function settleCodexSendsInEndedTurn(
session: Pick<CodexSession, 'threadId' | 'dispatchEchoes' | 'account'>,
frame: { sessionId: string; method: string; params: unknown }
@@ -1,67 +1,32 @@
import type { CodexTurnHost, CodexTurnInput } from './codex-structured-turn-contract'
import { captureCodexMessageCompilerInputs } from '../catalog/codex-structured-service-tier'
import {
codexMessageInputOptions,
compileCodexMessageOptions
} from '../../../../shared/agent-session/options/codex-message-compiler'
import { steerCodexTurn } from './codex-structured-turn-steer'
import { checkDispatchAuthority } from './codex-structured-dispatch-authority'
import {
StructuredAgentSessionInputNotAdmitted,
type StructuredAgentSessionAdmitFreshInput
} from '../../../native-chat/agent-session-wire/delivery/structured-agent-session-input-admission'
import { StructuredAgentSessionInputNotAdmitted } from '../../../native-chat/agent-session-wire/delivery/structured-agent-session-input-admission'
import { turnInputFor } from './codex-structured-message-options'
import { reportedCodexPermissionMode } from '../threads/codex-structured-thread-open'
import { admitCodexTurnPermission } from './codex-structured-turn-permission'
import type { CodexSkillReport } from '../catalog/codex-structured-skill-report'
import type { AgentSessionAccountKind } from '../../../../shared/agent-session/agent-session-availability'
import {
agentSessionFailureFact,
providerDiagnosticOf
} from '../../../../shared/agent-session/failures/agent-session-failure'
import type {
AgentJournalMessageItem,
AgentJournalTurnJoin
} from '../../../../shared/agent-session/journal/agent-session-journal-types'
import type { AgentJournalTurnJoin } from '../../../../shared/agent-session/journal/agent-session-journal-types'
import type { AgentSessionDispatchOutcome } from '../../../native-chat/agent-session-wire/providers/structured-agent-session-adapter'
import { isCodexAppServerRequestError } from '../../app-server/codex-app-server-request-error'
import type { CodexAppServerConnection } from '../../app-server/codex-app-server-connection-types'
import { isCodexAppServerUnsupportedError } from '../../app-server/codex-app-server-session'
import type { CodexDispatchEchoes } from './codex-structured-dispatch-echo'
import { readCodexTurnId } from '../../app-server/codex-structured-thread-facts'
import { codexTurnRequestOptions } from '../catalog/codex-structured-service-tier'
import type { CodexSession } from '../session-lifecycle/codex-structured-session-state'
import {
codexRunningOrOpeningTurn,
codexStopTarget,
type CodexTurnOpenWaits
} from './codex-structured-turn-open-wait'
import { codexRunningOrOpeningTurn, codexStopTarget } from './codex-structured-turn-open-wait'
import { codexDispatchRejection } from './codex-structured-turn-end-settlement'
export type CodexTurnHost = {
account?: AgentSessionAccountKind
skillReport?: Pick<CodexSkillReport, 'hasSkillPath'>
connection: Pick<CodexAppServerConnection, 'request'>
threadId: string
permissions?: CodexSession['permissions']
options: Map<string, string>
appliedTurnOptions?: Readonly<Record<string, string>>
appliedSettings?: CodexSession['appliedSettings']
reportedOptions?: CodexSession['reportedOptions']
catalogAccess?: CodexSession['catalogAccess']
dispatchEchoes: CodexDispatchEchoes
activeTurnIds?: ReadonlySet<string>
/** Running turns whose interrupt Codex already answered: aborted, so never steered. */
abortedTurnIds?: ReadonlySet<string>
turnOpenWaits: Pick<CodexTurnOpenWaits, 'wait'>
}
/** false means correlation capacity refused the send before writing; null turnId means an unnamed answer. */
export async function startCodexTurn(
host: CodexTurnHost,
input: {
clientMessageId: string
body: AgentJournalMessageItem
requestedAt?: number
input: CodexTurnInput & {
timeoutMs?: number
turnOptions?: Readonly<Record<string, string>>
signal?: AbortSignal
isCurrent?: () => boolean
admitFreshInput?: StructuredAgentSessionAdmitFreshInput
onReply?: () => void
}
): Promise<{ turnId: string | null; via: AgentJournalTurnJoin } | false> {
@@ -90,8 +55,11 @@ export async function startCodexTurn(
return steered
}
let previousApplied = host.appliedTurnOptions
let previousInput = host.appliedTurnInputOptions
let previousOptions: Map<string, string> | null | undefined
let requested: Readonly<Record<string, string>> = {}
let inputChoices: Readonly<Record<string, string | undefined>> = {}
let requestControls: Record<string, string> = {}
let enqueued = false
let discardSettingsAttempt: (() => void) | undefined
const enqueue = (
@@ -110,14 +78,24 @@ export async function startCodexTurn(
throw new StructuredAgentSessionInputNotAdmitted(undefined, 'permission')
}
previousApplied = host.appliedTurnOptions
previousInput = host.appliedTurnInputOptions
inputChoices = input.executionOptions ?? input.turnOptions ?? Object.fromEntries(host.options)
previousOptions = input.turnOptions ? new Map(host.options) : null
if (input.turnOptions) {
host.options.clear()
for (const [key, value] of Object.entries(input.turnOptions)) {
for (const [key, value] of Object.entries(
codexMessageInputOptions(inputChoices, {}, input.turnOptions)
)) {
host.options.set(key, value)
}
}
requested = { interactionMode: 'default', ...codexTurnRequestOptions(host) }
requested = {
interactionMode: 'default',
...compileCodexMessageOptions(
captureCodexMessageCompilerInputs(host),
input.turnOptions ?? Object.fromEntries(host.options)
)
}
const { interactionMode, ...nativeOptions } = requested
const collaborationMode = {
mode: interactionMode,
@@ -128,11 +106,21 @@ export async function startCodexTurn(
developer_instructions: null
}
}
requestControls = {
...requested,
...(collaborationMode.settings.model ? { model: collaborationMode.settings.model } : {}),
...(collaborationMode.settings.reasoning_effort
? { effort: collaborationMode.settings.reasoning_effort }
: {})
}
const known = { ...previousApplied }
for (const key of Object.keys(requested)) {
const knownInput = { ...previousInput }
for (const key of new Set([...Object.keys(requested), ...Object.keys(inputChoices)])) {
delete known[key]
delete knownInput[key]
}
host.appliedTurnOptions = known
host.appliedTurnInputOptions = knownInput
discardSettingsAttempt = host.appliedSettings?.attempt(requested)
enqueued = true
host.dispatchEchoes.expect(
@@ -202,9 +190,8 @@ export async function startCodexTurn(
}
if (isCodexAppServerRequestError(error) || isCodexAppServerUnsupportedError(error)) {
discardSettingsAttempt?.()
}
if (isCodexAppServerRequestError(error) || isCodexAppServerUnsupportedError(error)) {
host.appliedTurnOptions = previousApplied
host.appliedTurnInputOptions = previousInput
if (previousOptions) {
host.options.clear()
for (const [key, value] of previousOptions) {
@@ -215,22 +202,23 @@ export async function startCodexTurn(
throw error
}
host.appliedTurnOptions = { ...previousApplied, ...requested }
host.appliedTurnInputOptions = codexMessageInputOptions(
inputChoices,
{
serviceTier: 'default',
...requestControls,
...previousInput,
interactionMode: requestControls.interactionMode
},
requested
)
return { turnId: readCodexTurnId(answer), via: 'start' }
}
/** Replies establish admission; exact echoes establish identity. Timeouts cannot disprove delivery. */
export async function dispatchCodexTurn(
session: CodexTurnHost,
input: {
sessionId: string
clientMessageId: string
body: AgentJournalMessageItem
requestedAt?: number
turnOptions?: Readonly<Record<string, string>>
signal?: AbortSignal
isCurrent?: () => boolean
admitFreshInput?: StructuredAgentSessionAdmitFreshInput
},
input: CodexTurnInput & { sessionId: string },
timeoutMs: number | undefined
): Promise<AgentSessionDispatchOutcome> {
let answer: { turnId: string | null; via: AgentJournalTurnJoin } | false
@@ -270,12 +258,6 @@ export async function dispatchCodexTurn(
return { state: 'rejected', ...codexDispatchRejection(agentSessionFailureFact('queueFull')) }
}
// An answer read after the turn it names already ended is settled by that end.
if (!answer.turnId) {
session.dispatchEchoes.answered(
input.clientMessageId,
answeredAt ?? session.dispatchEchoes.tick()
)
}
if (answer.turnId) {
session.dispatchEchoes.bindTurn(
input.clientMessageId,
@@ -284,6 +266,11 @@ export async function dispatchCodexTurn(
answer.via,
answeredAt
)
} else {
session.dispatchEchoes.answered(
input.clientMessageId,
answeredAt ?? session.dispatchEchoes.tick()
)
}
return { state: 'admitted', ranOptions: { ...session.appliedTurnOptions } }
}
@@ -2,7 +2,7 @@ import {
StructuredAgentSessionInputNotAdmitted,
type StructuredAgentSessionAdmitFreshInput
} from '../../../native-chat/agent-session-wire/delivery/structured-agent-session-input-admission'
import type { CodexTurnHost } from './codex-structured-turn-start'
import type { CodexTurnHost } from './codex-structured-turn-contract'
import type { AgentJournalMessageItem } from '../../../../shared/agent-session/journal/agent-session-journal-types'
import { checkDispatchAuthority } from './codex-structured-dispatch-authority'
import { turnInputFor } from './codex-structured-message-options'
@@ -0,0 +1,77 @@
import { describe, expect, it } from 'vitest'
import { validateComputerProviderActionParams } from './computer-provider-action-validation'
function sendKey(
method: 'hotkey' | 'pressKey',
key: string,
platform: NodeJS.Platform,
allowSystemShortcut?: boolean
): Promise<string> {
return validateComputerProviderActionParams(
method,
{ app: 'Target', key, ...(allowSystemShortcut ? { allowSystemShortcut } : {}) },
platform
)
}
const BLOCKED: [NodeJS.Platform, 'hotkey' | 'pressKey', string][] = [
['darwin', 'hotkey', 'Cmd+Q'],
['darwin', 'hotkey', 'CmdOrCtrl+q'],
['darwin', 'hotkey', 'Shift+Command+Q'],
['darwin', 'hotkey', 'Cmd+Option+Shift+Q'],
['darwin', 'hotkey', 'Ctrl+Meta+Q'],
['darwin', 'hotkey', 'Cmd+Alt+Esc'],
['darwin', 'hotkey', 'Cmd+Shift+Delete'],
['win32', 'hotkey', 'Alt+F4'],
['win32', 'hotkey', 'Win+L'],
['win32', 'hotkey', 'Ctrl+Shift+Escape'],
['linux', 'hotkey', 'Ctrl+Q'],
['linux', 'hotkey', 'CmdOrCtrl+Shift+Q'],
['linux', 'hotkey', 'Alt+F4'],
['linux', 'hotkey', 'Super+L'],
['linux', 'hotkey', 'ctrl+alt+Delete'],
['linux', 'hotkey', 'ctrl+alt+BackSpace'],
['linux', 'hotkey', 'Ctrl+Alt+F2'],
['linux', 'hotkey', 'Meta+F4'],
['linux', 'hotkey', 'Ctrl+0x71'],
['linux', 'hotkey', 'Alt+0xffc1'],
['linux', 'hotkey', 'Alt+70'],
['linux', 'hotkey', 'Alt+70suffix'],
['linux', 'hotkey', 'Alt+70\rsuffix'],
['linux', 'hotkey', 'Ctrl+Alt+L1'],
['linux', 'hotkey', 'Ctrl+XF86Switch_VT_1'],
['linux', 'pressKey', 'XF86_Switch_VT_2'],
['linux', 'pressKey', 'Terminate_Server'],
['linux', 'pressKey', 'XF86PowerOff'],
['linux', 'pressKey', 'XF86_Sleep'],
['linux', 'pressKey', 'ScreenSaver'],
['linux', 'pressKey', '0x1008FF2A']
]
const ALLOWED: [NodeJS.Platform, 'hotkey' | 'pressKey', string][] = [
['darwin', 'hotkey', 'Cmd+A'],
['darwin', 'hotkey', 'Ctrl+Q'],
['darwin', 'hotkey', 'Cmd+ForwardDelete'],
['darwin', 'pressKey', 'q'],
['win32', 'hotkey', 'Ctrl+Q'],
['win32', 'hotkey', 'Ctrl+Alt+Delete'],
['linux', 'hotkey', 'Cmd+Q'],
['linux', 'hotkey', 'Ctrl+Delete'],
['linux', 'hotkey', 'Meta+L'],
['linux', 'hotkey', 'Ctrl+u'],
['linux', 'hotkey', 'Ctrl+1'],
['linux', 'pressKey', 'Escape']
]
describe('blocked system shortcuts', () => {
it.each(BLOCKED)('refuses %s %s %s unless allowed', async (platform, method, key) => {
await expect(sendKey(method, key, platform)).rejects.toMatchObject({
code: 'system_shortcut_blocked'
})
await expect(sendKey(method, key, platform, true)).resolves.toBe('Target')
})
it.each(ALLOWED)('sends %s %s %s', async (platform, method, key) => {
await expect(sendKey(method, key, platform)).resolves.toBe('Target')
})
})
@@ -0,0 +1,177 @@
import { RuntimeClientError } from '../runtime-client-error'
// Design ported from vercel-labs/agent-desktop's per-platform blocked_combo lists (Apache-2.0).
type BlockedKeyCombo = {
modifiers: readonly string[]
keys: readonly string[]
effect: string
}
type PlatformKeyRules = {
modifierAliases: Readonly<Record<string, string>>
keyAliases: Readonly<Record<string, string>>
canonicalKey?: (key: string) => string
// Most specific first so the error names what the combo would actually do.
blocked: readonly BlockedKeyCombo[]
}
const SHARED_KEY_ALIASES = { esc: 'escape' }
// Aliases mirror how each provider maps modifier names to physical keys.
const MACOS_RULES: PlatformKeyRules = {
modifierAliases: {
cmd: 'cmd',
command: 'cmd',
meta: 'cmd',
super: 'cmd',
win: 'cmd',
cmdorctrl: 'cmd',
commandorcontrol: 'cmd',
ctrl: 'ctrl',
control: 'ctrl',
alt: 'alt',
option: 'alt',
shift: 'shift'
},
// The macOS provider sends both names as the same physical Delete key.
keyAliases: { ...SHARED_KEY_ALIASES, delete: 'backspace' },
blocked: [
{ modifiers: ['cmd', 'ctrl'], keys: ['q'], effect: 'lock the screen' },
{ modifiers: ['cmd', 'shift'], keys: ['q'], effect: 'log out of macOS' },
{ modifiers: ['cmd', 'alt'], keys: ['escape'], effect: 'open the Force Quit window' },
{ modifiers: ['cmd', 'shift'], keys: ['backspace'], effect: 'empty the Trash' },
{ modifiers: ['cmd'], keys: ['q'], effect: 'quit the app' }
]
}
const WINDOWS_RULES: PlatformKeyRules = {
modifierAliases: {
ctrl: 'ctrl',
control: 'ctrl',
cmdorctrl: 'ctrl',
commandorcontrol: 'ctrl',
meta: 'win',
super: 'win',
win: 'win',
cmd: 'win',
command: 'win',
alt: 'alt',
option: 'alt',
shift: 'shift'
},
keyAliases: SHARED_KEY_ALIASES,
// Ctrl+Alt+Delete is absent: Windows never honors a synthesized Secure Attention Sequence.
blocked: [
{ modifiers: ['alt'], keys: ['f4'], effect: 'close the app' },
{ modifiers: ['win'], keys: ['l'], effect: 'lock Windows' },
{ modifiers: ['ctrl', 'shift'], keys: ['escape'], effect: 'open Task Manager' }
]
}
const NUMERIC_KEYSYM = 'numeric keysym'
const LINUX_RULES: PlatformKeyRules = {
modifierAliases: {
ctrl: 'ctrl',
control: 'ctrl',
cmdorctrl: 'ctrl',
commandorcontrol: 'ctrl',
// xdotool sends Meta_L, which default XKB layouts put on the Alt modifier.
meta: 'alt',
super: 'super',
win: 'super',
cmd: 'super',
command: 'super',
alt: 'alt',
option: 'alt',
shift: 'shift'
},
// X11 names F11 and F12 L1 and L2 too.
keyAliases: { ...SHARED_KEY_ALIASES, l1: 'f11', l2: 'f12' },
// Xlib and GDK resolve 0x/U hex keysyms and drop an optional XF86 prefix; xdotool reads any
// other token starting with a digit as a raw keycode (atoi ignores trailing text).
canonicalKey: (key) =>
/^(\d[\s\S]+|u[0-9a-f]+)$/.test(key) ? NUMERIC_KEYSYM : key.replace(/^xf86_?/, ''),
blocked: [
{
modifiers: [],
keys: [NUMERIC_KEYSYM],
effect: 'send a numeric keysym or keycode that computer use cannot check'
},
{
modifiers: [],
keys: ['poweroff', 'powerdown', 'logoff', 'sleep', 'suspend', 'hibernate', 'standby'],
effect: 'power off, suspend or log out of the session'
},
{ modifiers: [], keys: ['screensaver'], effect: 'lock the screen' },
{
modifiers: [],
keys: Array.from({ length: 12 }, (_, index) => `switch_vt_${index + 1}`),
effect: 'switch to another virtual terminal'
},
{ modifiers: [], keys: ['terminate_server'], effect: 'end the X session' },
{
modifiers: ['ctrl', 'alt'],
keys: ['delete'],
effect: 'open the log out and power off dialog'
},
{ modifiers: ['ctrl', 'alt'], keys: ['backspace'], effect: 'end the X session' },
{
modifiers: ['ctrl', 'alt'],
keys: ['f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12'],
effect: 'switch to another virtual terminal'
},
{ modifiers: ['ctrl', 'alt'], keys: ['l'], effect: 'lock the screen' },
{ modifiers: ['super'], keys: ['l'], effect: 'lock the screen' },
{ modifiers: ['alt'], keys: ['f4'], effect: 'close the window' },
{ modifiers: ['ctrl'], keys: ['q'], effect: 'quit the app' }
]
}
function platformKeyRules(platform: NodeJS.Platform): PlatformKeyRules {
if (platform === 'darwin') {
return MACOS_RULES
}
return platform === 'win32' ? WINDOWS_RULES : LINUX_RULES
}
// Extra modifiers still match: Cmd+Alt+Shift+Q logs out just like Cmd+Shift+Q.
function findBlockedKeyCombo(key: string, platform: NodeJS.Platform): BlockedKeyCombo | null {
const rules = platformKeyRules(platform)
const parts = key === '+' ? ['+'] : key.split('+')
const modifiers = new Set<string>()
let keyName = ''
for (const part of parts) {
const normalized = part.trim().toLowerCase()
const modifier = rules.modifierAliases[normalized.replace(/[\s_-]/g, '')]
if (modifier) {
modifiers.add(modifier)
} else {
const aliased = rules.keyAliases[normalized] ?? normalized
keyName = rules.canonicalKey ? rules.canonicalKey(aliased) : aliased
}
}
return (
rules.blocked.find(
(combo) =>
combo.keys.includes(keyName) && combo.modifiers.every((modifier) => modifiers.has(modifier))
) ?? null
)
}
export function assertComputerKeyComboAllowed(
params: Record<string, unknown>,
key: string,
platform: NodeJS.Platform
): void {
if (params.allowSystemShortcut === true) {
return
}
const blocked = findBlockedKeyCombo(key, platform)
if (blocked) {
throw new RuntimeClientError(
'system_shortcut_blocked',
`'${key}' would ${blocked.effect}, so computer use refuses it unless the caller passes --allow-system-shortcut`
)
}
}
@@ -3,6 +3,7 @@ import {
computerUseHotkeyValidationMessage,
computerUsePressKeyValidationMessage
} from '../../../shared/computer-use-key-spec'
import { assertComputerKeyComboAllowed } from './computer-blocked-key-combos'
import { validateComputerClipboardPasteTextWithBoundedYield } from './computer-clipboard-paste-validation'
import { RuntimeClientError } from '../runtime-client-error'
@@ -19,7 +20,8 @@ type ComputerProviderActionMethod =
export async function validateComputerProviderActionParams(
method: ComputerProviderActionMethod,
params: Record<string, unknown>
params: Record<string, unknown>,
platform: NodeJS.Platform = process.platform
): Promise<string> {
const app = requireNonEmptyString(params, 'app')
validateWindowTarget(params)
@@ -49,10 +51,10 @@ export async function validateComputerProviderActionParams(
await validatePasteText(params)
return app
case 'pressKey':
validatePressKey(params)
validatePressKey(params, platform)
return app
case 'hotkey':
validateHotkey(params)
validateHotkey(params, platform)
return app
case 'setValue':
requireNonNegativeInteger(params, 'elementIndex')
@@ -182,20 +184,22 @@ function validateScrollDirection(params: Record<string, unknown>): void {
}
}
function validatePressKey(params: Record<string, unknown>): void {
function validatePressKey(params: Record<string, unknown>, platform: NodeJS.Platform): void {
const key = requireNonEmptyString(params, 'key')
const message = computerUsePressKeyValidationMessage(key)
if (message) {
throw new RuntimeClientError('invalid_argument', message)
}
assertComputerKeyComboAllowed(params, key, platform)
}
function validateHotkey(params: Record<string, unknown>): void {
function validateHotkey(params: Record<string, unknown>, platform: NodeJS.Platform): void {
const key = requireNonEmptyString(params, 'key')
const message = computerUseHotkeyValidationMessage(key)
if (message) {
throw new RuntimeClientError('invalid_argument', message)
}
assertComputerKeyComboAllowed(params, key, platform)
}
function validatePasteText(params: Record<string, unknown>): Promise<void> | void {
@@ -40,6 +40,7 @@ import {
isRuntimeHostUnavailable
} from './runtime-host/desktop-script-runtime-host'
import { DesktopScriptSnapshotStore } from './snapshots/desktop-script-snapshot-store'
import { readSubtreeSnapshot } from './snapshots/desktop-script-subtree-snapshot'
import { normalizeBridgeApp, renderSnapshot } from './snapshots/desktop-script-snapshot-rendering'
import { normalizeComputerActionResult } from '../action-contracts/computer-action-verification-normalization'
import { RuntimeClientError } from '../runtime-client-error'
@@ -119,6 +120,14 @@ export class DesktopScriptProviderClient {
async snapshot(params: Record<string, unknown>): Promise<ComputerSnapshotResult> {
const app = stringParam(params, 'app')
const rootElementIndex = optionalNumberParam(params, 'rootElementIndex')
if (rootElementIndex !== undefined) {
return await readSubtreeSnapshot(this.snapshotStore, (request) => this.callBridge(request), {
app,
rootElementIndex,
params
})
}
const response = await this.callBridge({
tool: 'get_app_state',
app,
@@ -77,6 +77,8 @@ export type BridgeSnapshot = {
focusedElementId?: number | null
selectedText?: string | null
elements?: BridgeElement[]
/** Set by the client when treeLines show only this element's subtree. */
rootElementIndex?: number
}
export type BridgeWindow = {
@@ -128,6 +130,8 @@ export type BridgeRequest = {
requestId?: number
app?: string
element?: BridgeElement
/** get_app_state renders only this element's subtree. */
root?: BridgeElement
fromElement?: BridgeElement
toElement?: BridgeElement
x?: number
@@ -49,6 +49,9 @@ export function renderSnapshot(
treeText,
elementCount: snapshot.elements?.length ?? 0,
focusedElementId,
...(snapshot.rootElementIndex === undefined
? {}
: { rootElementIndex: snapshot.rootElementIndex }),
truncation: {
truncated: snapshot.truncation?.truncated === true,
maxNodes: snapshot.truncation?.maxNodes,
@@ -125,6 +128,9 @@ function renderTreeText(snapshot: BridgeSnapshot): string {
const lines = [
`App=${appRef} (pid ${snapshot.app.pid})`,
`Window: "${sanitize(snapshot.windowTitle ?? snapshot.app.name)}", App: ${sanitize(snapshot.app.name)}.`,
...(snapshot.rootElementIndex === undefined
? []
: [`Showing element ${snapshot.rootElementIndex} and its descendants.`]),
'',
...(snapshot.treeLines ?? [])
]
@@ -0,0 +1,72 @@
import { afterEach, describe, expect, it } from 'vitest'
import {
createDesktopScriptProviderClient,
mockBridgeResponse,
resetDesktopScriptProviderTestHarness,
sampleBridgeSnapshot,
sampleCapabilities
} from '../desktop-script-provider-test-harness'
function bridgeSnapshot(treeLines: string[], elements: { index: number; runtimeId: number[] }[]) {
return { ...sampleBridgeSnapshot('Mail', ''), windowIndex: 0, treeLines, elements }
}
describe('DesktopScriptProviderClient subtree reads', () => {
afterEach(resetDesktopScriptProviderTestHarness)
it('keeps shown indexes, numbers hidden children after them, and resolves them for actions', async () => {
const client = await createDesktopScriptProviderClient('linux', '/tmp/runtime.py')
mockBridgeResponse({
ok: true,
snapshot: bridgeSnapshot(
['0 frame Mail', '\t1 list Inbox', '\t\t2 button A', '\t\t... 1 more not shown'],
[
{ index: 0, runtimeId: [0] },
{ index: 1, runtimeId: [0, 0] },
{ index: 2, runtimeId: [0, 0, 0] }
]
)
})
await client.snapshot({ app: 'Mail' })
let sentRoot: unknown
mockBridgeResponse(
{
ok: true,
snapshot: bridgeSnapshot(
['0 list Inbox', '\t1 button A', '\t2 button B'],
[
{ index: 0, runtimeId: [0, 0] },
{ index: 1, runtimeId: [0, 0, 0] },
{ index: 2, runtimeId: [0, 0, 1] }
]
)
},
(operation) => {
sentRoot = operation.root
}
)
const subtree = await client.snapshot({ app: 'Mail', rootElementIndex: 1 })
expect(sentRoot).toMatchObject({ index: 1, runtimeId: [0, 0] })
expect(subtree.snapshot.rootElementIndex).toBe(1)
expect(subtree.snapshot.treeText.split('\n').slice(2, 7)).toEqual([
'Showing element 1 and its descendants.',
'',
'1 list Inbox',
'\t2 button A',
'\t3 button B'
])
let clicked: unknown
mockBridgeResponse({ ok: true, capabilities: sampleCapabilities() })
mockBridgeResponse(
{ ok: true, snapshot: bridgeSnapshot(['0 frame Mail'], [{ index: 0, runtimeId: [0] }]) },
(operation) => {
clicked = operation.element
}
)
await client.action('click', { app: 'Mail', elementIndex: 3 })
expect(clicked).toMatchObject({ runtimeId: [0, 0, 1] })
})
})
@@ -0,0 +1,103 @@
import type { ComputerSnapshotResult } from '../../../../shared/computer-use-runtime-types'
import { renumberComputerTreeLine } from '../../../../shared/computer-use-tree-text'
import { RuntimeClientError } from '../../runtime-client-error'
import { elementParam } from '../actions/desktop-script-action'
import { optionalNumberParam } from '../desktop-script-provider-params'
import type {
BridgeElement,
BridgeRequest,
BridgeResponse,
BridgeSnapshot
} from '../desktop-script-provider-types'
import type { DesktopScriptSnapshotStore } from './desktop-script-snapshot-store'
import { renderSnapshot } from './desktop-script-snapshot-rendering'
/** Reads one element's subtree of the window the store last saw, the target of --element-index. */
export async function readSubtreeSnapshot(
store: DesktopScriptSnapshotStore,
callBridge: (request: BridgeRequest) => Promise<BridgeResponse>,
{
app,
rootElementIndex,
params
}: { app: string; rootElementIndex: number; params: Record<string, unknown> }
): Promise<ComputerSnapshotResult> {
const base = store.current(app, optionalNumberParam(params, 'windowId'), params)
if (!base) {
throw new RuntimeClientError(
'element_not_found',
'--element-index needs a get-app-state snapshot of this app/window first'
)
}
const response = await callBridge({
tool: 'get_app_state',
app,
root: elementParam(base, rootElementIndex),
windowId: base.windowId ?? undefined,
windowIndex: base.windowIndex ?? undefined,
noScreenshot: params.noScreenshot === true,
restoreWindow: params.restoreWindow === true
})
if (!response.snapshot) {
throw new RuntimeClientError('accessibility_error', 'desktop provider returned no snapshot')
}
const { shown, cached } = indexSubtreeSnapshot(base, response.snapshot, rootElementIndex)
store.remember(app, cached, params)
return renderSnapshot(shown, params.noScreenshot === true)
}
/**
* Gives a subtree read the indexes the window's cached snapshot already uses: elements it showed
* keep theirs, new ones get unused indexes. Returns what to show and what to cache, where the
* cache keeps the earlier elements so their indexes still resolve for actions.
*/
function indexSubtreeSnapshot(
base: BridgeSnapshot,
subtree: BridgeSnapshot,
rootElementIndex: number
): { shown: BridgeSnapshot; cached: BridgeSnapshot } {
const baseElements = base.elements ?? []
const baseIndexByRuntimeId = new Map<string, number>()
for (const element of baseElements) {
const key = runtimeIdKey(element)
if (key !== null && !baseIndexByRuntimeId.has(key)) {
baseIndexByRuntimeId.set(key, element.index)
}
}
let nextIndex = baseElements.reduce((max, element) => Math.max(max, element.index), -1) + 1
const indexes = new Map<number, number>()
const claimed = new Set<number>()
for (const element of subtree.elements ?? []) {
const key = runtimeIdKey(element)
const known = key === null ? undefined : baseIndexByRuntimeId.get(key)
const index = known !== undefined && !claimed.has(known) ? known : nextIndex++
claimed.add(index)
indexes.set(element.index, index)
}
const walked = (subtree.elements ?? []).map((element) => ({
...element,
index: indexes.get(element.index) ?? element.index
}))
const focusedElementId =
subtree.focusedElementId === null || subtree.focusedElementId === undefined
? subtree.focusedElementId
: (indexes.get(subtree.focusedElementId) ?? null)
const shown: BridgeSnapshot = {
...subtree,
treeLines: subtree.treeLines?.map((line) => renumberComputerTreeLine(line, indexes)),
focusedElementId,
elements: walked,
rootElementIndex
}
const cached: BridgeSnapshot = {
...shown,
elements: [...baseElements.filter((element) => !claimed.has(element.index)), ...walked]
}
return { shown, cached }
}
function runtimeIdKey(element: BridgeElement): string | null {
return Array.isArray(element.runtimeId) && element.runtimeId.length > 0
? JSON.stringify(element.runtimeId)
: null
}
@@ -72,6 +72,7 @@ export function automaticSubmissionFixture(cleanups: (() => void)[]) {
kind: 'codex',
compiler: {
active: { model: 'active' },
activeInput: { model: 'active' },
defaults: { model: 'default' },
models: [],
listedServiceTiers: []
@@ -63,7 +63,13 @@ it('evaluates the first nonmail fallback itself and waits on it instead of bypas
expect(f.card('person')?.state).toBe('waiting')
f.selection.steering = {
kind: 'codex',
compiler: { active: { model: 'different' }, defaults: {}, models: [], listedServiceTiers: [] }
compiler: {
active: { model: 'different' },
activeInput: { model: 'different' },
defaults: {},
models: [],
listedServiceTiers: []
}
}
expect((await f.claim('send', f.prepare(100, 'unavailable'))).value).toEqual({
kind: 'claimed',
@@ -497,7 +497,8 @@ it('uses scoped failure facts for cuts and request outcomes', () => {
turnItemId: 'turn',
resetsAt: 999,
turnEndedAt: 300,
chain: 0
chain: 0,
fence: 1
})
expect(readJournalSqlStatusState(context()).summary).toMatchObject({
status: 'idle',
@@ -86,9 +86,11 @@ export function readJournalSqlUsageLimitCut(
const displayed = state.displayed(item.itemId)
const limit = displayed && nativeChatUsageLimitOfRow(displayed)
if (limit) {
const fence = state.itemFence(latest.item.itemId)
return {
turnItemId: latest.item.itemId,
chain: readUsageLimitChain(state, latest.item.itemId),
...(fence === undefined ? {} : { fence }),
...(limit.resetsAt !== undefined ? { resetsAt: limit.resetsAt } : {}),
turnEndedAt: latest.item.recoveredAt ?? latest.lifecycle.completedAt
}
@@ -14,7 +14,7 @@ import type {
} from '../../../shared/agent-session/journal/agent-session-journal-identity'
import { readJournalSqlWorkAnswer } from './journal-sql-work-answer'
import { JournalEpochReplacements } from './context-epochs/journal-epoch-replacements'
import { journalWroteBeforeOpen } from './replay/journal-cursor'
import { journalWroteBeforeOpen, sameJournalCursor } from './replay/journal-cursor'
import type { JournalSqlDatabase } from './database/journal-sql-database'
import type { JournalSqlOwner } from './journal-sql-owner'
import type { JournalWriterCommit } from './journal-writer-changes'
@@ -211,6 +211,7 @@ export class SqlAgentSessionJournal extends SqlJournalAppends {
if (commit.goals) {
this.goalAttachments.adopt(commit.goals)
}
const advanced = !sameJournalCursor(this.acknowledged, commit.cursor)
this.acknowledged = { ...commit.cursor }
if (commit.queueRevision !== undefined) {
this.queuedMessages.adoptRevision(commit.queueRevision)
@@ -220,7 +221,9 @@ export class SqlAgentSessionJournal extends SqlJournalAppends {
} else if (operation.kind === 'epoch-replace') {
this.queuedMessages.editLeases.adoptDeadline(null)
}
this.listener?.(commit)
if (advanced || commit.inputChanged || commit.items.length > 0) {
this.listener?.(commit)
}
return commit
})
return this.epochReplacements.notifyFor(operation.kind, written)
@@ -102,6 +102,7 @@ export async function acquireOwner(
sessionId: record.sessionId,
fence,
...(acquired.link ? { link: acquired.link } : {}),
...(acquired.accountHome ? { launchedAccountHome: acquired.accountHome } : {}),
...(acquired.accountRecipe
? {
accountAcquisition: {
@@ -23,7 +23,10 @@ import {
type MutationPlan
} from '../operations/structured-agent-session-mutation-plans'
import type { CommandReceiptResult } from '../../agent-session-journal/writing/command-receipt-schema'
import { structuredQueueHold } from './structured-agent-session-queued-gate'
import {
canSteerStructuredQueuedMessage,
structuredQueueHold
} from './structured-agent-session-queued-gate'
import { refuse } from '../../../../shared/agent-session/failures/agent-session-wire-refusals'
import { queuedMessageOwed } from './structured-agent-session-queued-mail'
import { resumeStructuredQueue } from './structured-agent-session-queued-pause'
@@ -131,6 +134,30 @@ export function sendQueuedStructuredAgentMessage(
if (row.body.command && work.working()) {
return invalid("A command can't be sent while the agent is working.")
}
if (
(work.activeTurnId() !== null || work.queueState().owedEnteredSubmissions.length > 0) &&
(work.activeTurnId() === null ||
work.queueState().owedEnteredSubmissions.length > 0 ||
!canSteerStructuredQueuedMessage(
{
record,
work,
child: context.sessions.get(ctx.sessionId)?.child,
stopping: await context.readStopping(ctx.sessionId),
adapter: context.deps.adapter
},
row
))
) {
return {
ok: false,
refusal: refuse(
'agent_session_operation_invalid',
{ reason: 'turnActive' },
'The current turn cannot accept this queued message yet.'
)
}
}
const submissionId = operationId
const owed = queuedMessageOwed(context.deps, ctx.sessionId, row.body)
try {
@@ -1,4 +1,11 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
import type { CodexAppServerConnection } from '../../../../codex/app-server/codex-app-server-connection-types'
import { optionSession } from '../../../../codex/structured/catalog/codex-structured-option-session.test-fixture'
import {
readCodexMessageSteeringInputs,
resolvedCodexMessageOptions
} from '../../../../codex/structured/turns/codex-structured-message-options'
import { dispatchCodexTurn } from '../../../../codex/structured/turns/codex-structured-turn-start'
import {
createQueuedMessageTestRig,
QUEUED_RIG_CALLER
@@ -17,7 +24,26 @@ import { agentJournalSubmissionKey } from '../../../../../shared/agent-session/j
let rig: QueuedMessageTestRig
beforeEach(async () => {
const request = vi.fn<CodexAppServerConnection['request']>(async () => ({}))
const native = optionSession(request)
native.threadId = THREAD
rig = await createQueuedMessageTestRig({ restartable: true })
rig.host.deps.adapter.captureMessageSteeringInputs = (input) =>
input.sessionId === SESSION
? readCodexMessageSteeringInputs(native, input)
: { kind: 'codex-unavailable' }
rig.host.deps.adapter.holdsLiveProviderProcess = (sessionId, generation) =>
sessionId === SESSION &&
generation === native.acquisitionGeneration &&
!native.ended &&
!native.connection.closed
rig.dispatch.mockImplementation(async (input) =>
dispatchCodexTurn(
native,
{ ...input, turnOptions: await resolvedCodexMessageOptions(native, input.executionOptions) },
undefined
)
)
})
afterEach(() => rig.dispose())
@@ -2,7 +2,10 @@ import { eventually } from './structured-agent-session-queued-message-rig-provid
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../../../shared/agent-session/journal/agent-session-journal-types'
import type { CodexAppServerConnection } from '../../../../codex/app-server/codex-app-server-connection-types'
import { resolvedCodexMessageOptions } from '../../../../codex/structured/turns/codex-structured-message-options'
import {
resolvedCodexMessageOptions,
readCodexMessageSteeringInputs
} from '../../../../codex/structured/turns/codex-structured-message-options'
import { optionSession } from '../../../../codex/structured/catalog/codex-structured-option-session.test-fixture'
import { dispatchCodexTurn } from '../../../../codex/structured/turns/codex-structured-turn-start'
import {
@@ -36,6 +39,10 @@ async function createGoalRig(): Promise<QueuedMessageTestRig> {
reportsModel: 'gpt-ran',
reportsEffort: 'high'
})
created.host.deps.adapter.captureMessageSteeringInputs = (input) =>
input.sessionId === SESSION
? readCodexMessageSteeringInputs(native, input)
: { kind: 'codex-unavailable' }
created.host.deps.adapter.holdsLiveProviderProcess = (sessionId, generation) =>
sessionId === SESSION &&
generation === native.acquisitionGeneration &&
@@ -103,3 +110,44 @@ it('holds a Plan choice until the goal turn ends', async () => {
await eventually(() => expect(rig.dispatch).toHaveBeenCalledTimes(2))
expect(rig.dispatch.mock.calls[1]?.[0].executionOptions).toEqual(choices)
})
it('keeps an incompatible Send-now card intact until a new turn can use its choices', async () => {
await startGoalTurn()
const choices = { model: 'gpt-ran', effort: 'high', serviceTier: 'priority' }
const queued = rig.send('Keep my selected speed', 'queue-if-active', { carried: choices })
expect(await queued.result).toMatchObject({ ok: true, value: { queued: {} } })
const operationId = hostTestOperationId()
for (let press = 0; press < 2; press += 1) {
expect(await rig.sendNow(queued.id, operationId)).toMatchObject({
ok: false,
refusal: { details: { reason: 'turnActive' } }
})
}
expect(await rig.handoff(queued.id)).toBeUndefined()
expect(
await rig.host
.collaboratorsForTests()
.sessions.get(SESSION)
?.journal.queuedMessages.get(queued.id)
).toMatchObject({
state: 'waiting',
body: { blocks: [{ type: 'text', text: 'Keep my selected speed' }] },
options: choices
})
native.activeTurnIds = new Set()
rig
.providerEvents()
.appendItem(
{ provider: 'codex', threadId: THREAD, turnId: 'goal-turn', ordinal: 99 },
{ kind: 'turn', turnId: 'goal-turn', state: 'completed' },
{ turnScope: AGENT_JOURNAL_THREAD_SCOPE }
)
await rig.host.flushStreamedEvents(SESSION)
await eventually(async () => expect((await rig.handoff(queued.id))?.handedOverAt).toBeDefined())
expect(rig.dispatch.mock.calls[1]?.[0].executionOptions).toEqual(choices)
expect(await rig.sendNow(queued.id, operationId)).toMatchObject({
ok: true,
value: { submission: { queuedMessageId: queued.id } }
})
})
@@ -1,3 +1,4 @@
import type { AgentSessionAccountHome } from '../../../../../shared/agent-session/lifecycle/agent-session-account-home'
import { readTestJournalSnapshot } from '../../../agent-session-journal/journal-observation.test-support'
import type { AgentMessageSource } from '../../../../../shared/agent-session/agent-session-message-source'
import { vi, type Mock } from 'vitest'
@@ -19,6 +20,7 @@ import {
} from './structured-agent-session-host-test-data.test-fixture'
export type QueuedRigProviderOptions = {
acquiredAccountHome?: () => AgentSessionAccountHome
agent?: 'claude'
rewind?: NonNullable<StructuredAgentSessionAdapter['rewind']>
recoverRewind?: NonNullable<StructuredAgentSessionAdapter['recoverRewind']>
@@ -91,6 +93,7 @@ export function createQueuedRigProvider(
const thread = context?.pendingClear
? `${THREAD}-context-${record!.providerContextBoundary!.operationId}`
: (context?.head?.handle.nativeId ?? THREAD)
const accountHome = options.acquiredAccountHome?.()
return {
process: {
hostId: 'local',
@@ -99,6 +102,7 @@ export function createQueuedRigProvider(
spawnToken
},
acquisitionGeneration: 'generation-1',
...(accountHome ? { accountHome } : {}),
link: {
linkId: `link-${fence}`,
handle:
@@ -116,6 +120,7 @@ export function createQueuedRigProvider(
kind: 'codex',
compiler: {
active: reported(),
activeInput: reported(),
defaults: reported(),
models: [],
listedServiceTiers: []
@@ -33,7 +33,9 @@ async function queuedDraft(text: string): Promise<string> {
describe('a repeated draft press under a fresh operation id', () => {
it('Send-now pressed twice sends the draft once and answers both presses with its submission', async () => {
await openRigTurnFor(rig, await rig.workingSend())
const opening = await rig.workingSend()
await rig.settleAccepted(opening, 'opening')
await openRigTurnFor(rig, opening)
const draftId = await queuedDraft('send me now')
const [first, second] = await Promise.all([rig.sendNow(draftId), rig.sendNow(draftId)])
expect(first).toMatchObject({ ok: true, replayed: false })
@@ -136,10 +136,11 @@ describe("Stop's event", () => {
it('a stopped unconfirmed hand-off stays ordinary while Resume drains waiting cards', async () => {
const working = await rig.workingSend()
await openRigTurnFor(rig, working)
await rig.settleAccepted(working, 'stopped')
const steered = await queuedDraft('sent now into the turn')
const opening = await queuedDraft('opens its own turn')
const last = await queuedDraft('waiting behind them')
await rig.sendNow(steered)
expect(await rig.sendNow(steered)).toMatchObject({ ok: true })
await eventually(async () => expect((await rig.handoff(steered))?.handedOverAt).toBeDefined())
const steer = await rig.handoffId(steered)
// Codex ends the stopped turn without taking the steer: the agent may have it, nothing proves
@@ -154,7 +155,6 @@ describe("Stop's event", () => {
},
via: 'steer'
})
await rig.settleAccepted(working, 'stopped')
await openRigTurnFor(rig, working, 'interrupted')
await expectHeld('stopped', opening, last)
expect(await rig.drafts()).toEqual([
@@ -22,6 +22,7 @@ afterEach(async () => rig?.dispose())
async function heldChat() {
await rig?.dispose()
let clock = HOST_TEST_NOW
let launchHome = '/homes/a'
let reading: ProviderRateLimits | null = null
let enabled = true
let answer: ((now: number) => ProviderRateLimits) | undefined
@@ -39,6 +40,8 @@ async function heldChat() {
let attributed: UsageLimitAccountScope = accountA
const current = await createQueuedMessageTestRig({
now: () => clock,
restartable: true,
acquiredAccountHome: () => ({ variable: 'CODEX_HOME', path: launchHome }),
usageLimitAutoContinue: {
isEnabled: () => enabled,
isWorkspaceAsleep: () => false,
@@ -47,6 +50,10 @@ async function heldChat() {
}
})
rig = current
current.host.deps.launchAccountHome = async () => ({ variable: 'CODEX_HOME', path: launchHome })
await eventually(() =>
expect(current.host.collaboratorsForTests().sessions.get(SESSION)?.child?.phase).toBe('ready')
)
await current.store.setSessionTabVisibility(SESSION, true)
const journal = current.host.collaboratorsForTests().sessions.get(SESSION)!.journal
const cut = await appendRewindUsageLimitedTurn(journal, clock)
@@ -79,6 +86,10 @@ async function heldChat() {
launchUnder: (next: UsageLimitAccountScope) => {
attributed = next
},
/** The cut's runtime launched on account A; a start of this chat would now launch on `path`. */
switchAccount: async (path: string) => {
launchHome = path
},
setEnabled: (next: boolean) => {
enabled = next
},
@@ -138,6 +149,33 @@ it('holds again on spending inside the grace, and leaves an unknown reset once r
await settleOne(await refused.pass(room + 15_000, (now) => weekly(0, null, now)))
})
it('continues before the cut resets once the chat would start on another account with room', async () => {
const chat = await heldChat()
const start = chat.cut.resetsAt - 60_000
await chat.switchAccount('/homes/b')
expect(await chat.pass(start + 5_000, (now) => weekly(0, null, now))).toEqual([])
expect(await chat.state()).toMatchObject({ state: 'scheduled', continueAt: start + 5_000 })
await settleOne(await chat.pass(start + 20_000, (now) => weekly(0, null, now)))
})
it('keeps the new account spent from a reading before the switch until a later one shows room', async () => {
const chat = await heldChat()
const start = chat.cut.resetsAt - 60_000
const newAccountReset = start + 2 * HOUR
await chat.switchAccount('/homes/b')
expect(await chat.pass(start + 5_000, () => weekly(100, newAccountReset, start + 1_000))).toEqual(
[]
)
expect(await chat.state()).toMatchObject({ continueAt: newAccountReset })
expect(await chat.pass(start + 10 * 60_000, () => weekly(100, newAccountReset, start))).toEqual(
[]
)
const room = start + 20 * 60_000
expect(await chat.pass(room, (now) => weekly(0, null, now))).toEqual([])
expect(await chat.state()).toMatchObject({ continueAt: room })
await settleOne(await chat.pass(room + 15_000, (now) => weekly(0, null, now)))
})
it('retires a hold when the launch moves to billing the reading does not describe', async () => {
const chat = await heldChat()
const weeklyReset = chat.cut.resetsAt + 3 * HOUR
@@ -0,0 +1,22 @@
import type { AgentJournalSubmission } from '../../../../shared/agent-session/journal/agent-session-journal-types'
import {
DISPATCH_DOUBT_PROVIDER_ENDED_UNANSWERED,
DISPATCH_DOUBT_WRITE_OUTCOME_UNKNOWN
} from '../../agent-session-journal/projection/journal-dispatch-doubt-reasons'
/** Admission releases presentation only; the journal still awaits an exact receipt. */
export function admittedStructuredSubmission(
submission: AgentJournalSubmission,
generationEnded: boolean
): AgentJournalSubmission {
return {
...submission,
dispatchState: 'unknown',
recovered: true,
reason:
submission.reason ??
(generationEnded
? DISPATCH_DOUBT_PROVIDER_ENDED_UNANSWERED
: DISPATCH_DOUBT_WRITE_OUTCOME_UNKNOWN)
}
}
@@ -16,7 +16,7 @@ import type { AgentSessionRecord } from '../../../../shared/agent-session/lifecy
import type { AgentSessionLatestTurn } from '../../../../shared/agent-session/agent-session-wire'
import type { StructuredAgentSessionWorkScope } from '../../../../shared/agent-session/view/structured-agent-session-main-agent-working'
import { isUnansweredStructuredAgentSessionDispatch } from '../../../../shared/agent-session/requests/structured-agent-session-unanswered-dispatch'
import { DISPATCH_DOUBT_PROVIDER_ENDED_UNANSWERED } from '../../agent-session-journal/projection/journal-dispatch-doubt-reasons'
import { admittedStructuredSubmission } from './structured-agent-session-submission-presentation'
export type StructuredAgentSessionWorkFacts = {
item: (id: string) => AgentJournalRenderItem | null
@@ -149,12 +149,7 @@ export class StructuredAgentSessionWorkPolicy {
* answer, reads in doubt: no client counts it as work, and its echo or refusal still settles it. */
publishedSubmission = (submission: AgentJournalSubmission): AgentJournalSubmission =>
this.unanswered(submission) && !this.holds(submission)
? {
...submission,
dispatchState: 'unknown',
recovered: true,
reason: submission.reason ?? DISPATCH_DOUBT_PROVIDER_ENDED_UNANSWERED
}
? admittedStructuredSubmission(submission, this.generationEnded(submission))
: submission
private unanswered(submission: AgentJournalSubmission): boolean {
@@ -10,8 +10,11 @@ import type { AgentSessionRecord } from '../../../../shared/agent-session/lifecy
import type { AgentSessionRecordStore } from '../../../runtime/agent-session/records/agent-session-record-store'
import {
agentSessionAccountHomePath,
agentSessionAccountHomesEqual,
isLegacyAgentSessionAccountHome,
type AgentSessionAccountHome
} from '../../../../shared/agent-session/lifecycle/agent-session-account-home'
import { normalizeRuntimePathForComparison } from '../../../../shared/cross-platform-path'
import { usageLimitContinueIdentity as identity } from './structured-agent-session-usage-limit-chain'
/** The polled usage of the account a chat continues on, as the host attributes it. */
@@ -75,6 +78,11 @@ export class UsageLimitAccountHolds {
private readonly scopes = new Map<string, UsageLimitAccountScope | undefined>()
/** The source witness each chat's attribution was computed under. */
private readonly witnesses = new Map<string, string | undefined>()
/** When each chat was first seen continuing on another account than the one its cut hit. */
private readonly movedSince = new Map<
string,
{ cut: string; to: AgentSessionAccountHome; at: number }
>()
/** Shared by every chat of a pass and keyed by source witness: work scales with accounts and
* their changes, not chats. */
private launchHomes = new Map<string, Promise<AgentSessionAccountHome | null>>()
@@ -90,15 +98,22 @@ export class UsageLimitAccountHolds {
this.readings = new Map()
}
/** Re-attributed each pass: the account a chat continues on follows the host's selection. */
async attribute(id: string, host: AttributionHost): Promise<void> {
/** Re-attributed each pass: the account a chat continues on follows the host's selection.
* `hit` names the cut and the account it hit, when known. */
async attribute(
id: string,
host: AttributionHost,
hit?: { cut: string; home: AgentSessionAccountHome | null },
now = Date.now()
): Promise<void> {
const record = host.store.getRecord(id)
const read = host.usageLimitAutoContinue?.accountUsage
const witness = record ? liveWitness(host, record.provider) : undefined
let scope: UsageLimitAccountScope | undefined
let movedTo: AgentSessionAccountHome | null = null
try {
// WSL readings never reach this host, so a WSL launch keeps the cut's own reset.
if (record && read && record.location.wslDistro === null) {
if (record && record.location.wslDistro === null) {
// Off WSL a launch home follows only the provider's selection, never the chat.
const target = JSON.stringify([witness, record.provider])
let launch = this.launchHomes.get(target)
@@ -107,8 +122,9 @@ export class UsageLimitAccountHolds {
this.launchHomes.set(target, launch)
}
const home = (await launch) ?? record.accountHome
movedTo = hit?.home && !sameAccountHome(home, hit.home) ? home : null
const path = agentSessionAccountHomePath(home)
if (home && path) {
if (read && home && path) {
const key = JSON.stringify([witness, record.provider, path])
let reading = this.readings.get(key)
if (!reading) {
@@ -129,6 +145,13 @@ export class UsageLimitAccountHolds {
}
this.scopes.set(id, scope)
this.witnesses.set(id, witness)
const since = this.movedSince.get(id)
if (!movedTo || !hit) {
this.movedSince.delete(id)
} else if (since?.cut !== hit.cut || !sameAccountHome(since.to, movedTo)) {
// Each destination counts from when it was chosen, so an older reading of it never releases.
this.movedSince.set(id, { cut: hit.cut, to: movedTo, at: now })
}
}
/** The witness the chat's attribution was computed under; a send binds to this one. */
@@ -155,20 +178,30 @@ export class UsageLimitAccountHolds {
}
}
}
for (const id of this.movedSince.keys()) {
if (!shown.has(id)) {
this.movedSince.delete(id)
}
}
}
clear(): void {
this.holds.clear()
this.scopes.clear()
this.witnesses.clear()
this.movedSince.clear()
this.beginPass()
}
verdict(
id: string,
cut: { turnItemId: string; resetsAt: number },
limited: { turnItemId: string; resetsAt: number },
now: number
): UsageLimitAccountVerdict {
// The account the chat moved to never hit this cut: from the move on, only its reading holds.
const moved = this.movedSince.get(id)
const movedCut = moved?.cut === identity(limited)
const cut = movedCut ? { ...limited, resetsAt: Math.min(limited.resetsAt, moved.at) } : limited
const scope = this.scopes.get(id)
let hold = this.holds.get(id)
// Nothing attributed and nothing held: the cut decides, and nothing is kept for it.
@@ -202,10 +235,11 @@ export class UsageLimitAccountHolds {
(hold.unknownReset && hold.spentSeenAt) || cut.resetsAt,
typeof covering === 'number' ? covering : cut.resetsAt
)
// Spending the new account showed before the move still holds; only a later reading releases it.
if (
account &&
reading &&
fresh(cut.resetsAt) &&
fresh(movedCut ? Number.NEGATIVE_INFINITY : cut.resetsAt) &&
now - anchor <= NATIVE_CHAT_USAGE_LIMIT_MAX_OVERDUE_MS
) {
if (typeof covering === 'number') {
@@ -220,7 +254,10 @@ export class UsageLimitAccountHolds {
hold.heldUntil = Math.max(hold.heldUntil ?? spentUntil, spentUntil)
}
hold.spentSeenAt = Math.max(hold.spentSeenAt ?? reading.updatedAt, reading.updatedAt)
} else if (reading.updatedAt > (hold.spentSeenAt ?? Number.NEGATIVE_INFINITY)) {
} else if (
fresh(cut.resetsAt) &&
reading.updatedAt > (hold.spentSeenAt ?? Number.NEGATIVE_INFINITY)
) {
// Room newer than any spending seen releases the hold at that reading, so it counts from now.
if (hold.unknownReset || (hold.heldUntil ?? Number.NEGATIVE_INFINITY) > reading.updatedAt) {
hold.heldUntil = reading.updatedAt
@@ -293,6 +330,14 @@ export class UsageLimitAccountHolds {
}
}
function sameAccountHome(left: AgentSessionAccountHome, right: AgentSessionAccountHome): boolean {
return isLegacyAgentSessionAccountHome(left) && isLegacyAgentSessionAccountHome(right)
? left.variable === right.variable &&
normalizeRuntimePathForComparison(left.path) ===
normalizeRuntimePathForComparison(right.path)
: agentSessionAccountHomesEqual(left, right)
}
/** Changes whenever a source the provider's attribution reads changes. */
function liveWitness(
host: AttributionHost,
@@ -165,18 +165,25 @@ export function usageLimitContinueChain(
return chain
}
/** A cut as the host's sweep reads it: with the chain that led to it. */
export type HostUsageLimitCut = NativeChatUsageLimitCut & { chain: number }
/** A cut as the host's sweep reads it: with the chain that led to it and the runtime fence that
* ran its turn, which names the account it hit. */
export type HostUsageLimitCut = NativeChatUsageLimitCut & { chain: number; fence?: number }
export function hostUsageLimitCut(
snapshot: {
items: readonly AgentJournalRenderItem[]
submissions: readonly ChainSubmission[]
},
cut: NativeChatUsageLimitCut | null
cut: NativeChatUsageLimitCut | null,
fenceOf: (itemId: string) => number | undefined
): HostUsageLimitCut | null {
const fence = cut ? fenceOf(cut.turnItemId) : undefined
return cut
? { ...cut, chain: usageLimitContinueChain(snapshot.items, snapshot.submissions, cut) }
? {
...cut,
chain: usageLimitContinueChain(snapshot.items, snapshot.submissions, cut),
...(fence === undefined ? {} : { fence })
}
: null
}
@@ -1,14 +1,15 @@
import type { HostUsageLimitCut } from './structured-agent-session-usage-limit-chain'
import type { StructuredAgentSessionHostDeps as HostDeps } from '../host/structured-agent-session-host-types'
type Deps = {
logger: HostDeps['logger']
export type UsageLimitCutReads = {
sessions: ReadonlyMap<string, unknown>
/** Reads an unopened chat's saved cut; history-sized, so the sweep budgets it. */
readSavedCut: (id: string) => Promise<HostUsageLimitCut | null>
readLoadedCut: (id: string) => Promise<HostUsageLimitCut | null>
}
type Deps = UsageLimitCutReads & { logger: HostDeps['logger'] }
/** The cuts of chats that aren't loaded, read from history a few per pass. */
export class UsageLimitColdCuts {
private readonly cuts = new Map<string, HostUsageLimitCut | null>()
@@ -0,0 +1,23 @@
import type { StructuredAgentSessionHostDeps as HostDeps } from '../host/structured-agent-session-host-types'
import type { AgentSessionRecordStore } from '../../../runtime/agent-session/records/agent-session-record-store'
import type { sendStructuredAgentSessionTurn } from '../operations/structured-agent-session-host-mutations'
import type { UsageLimitCutReads } from './structured-agent-session-usage-limit-cold-cuts'
export type UsageLimitContinueDeps = UsageLimitCutReads & {
host: Pick<HostDeps, 'usageLimitAutoContinue' | 'logger' | 'launchAccountHome'> & {
store: Pick<
AgentSessionRecordStore,
| 'listVisibleSessionIds'
| 'getRecord'
| 'hostId'
| 'isSessionTabVisible'
| 'cancelUsageLimitContinue'
>
}
serialize: <T>(id: string, run: () => Promise<T>) => Promise<T>
send: (
params: Parameters<typeof sendStructuredAgentSessionTurn>[2]
) => ReturnType<typeof sendStructuredAgentSessionTurn>
publish: (id: string) => void
now: () => number
}
@@ -1,3 +1,4 @@
import type { UsageLimitContinueDeps } from './structured-agent-session-usage-limit-continue-deps'
import { AgentSessionPreDispatchError } from '../operations/structured-agent-session-operation-settlement'
import { randomUUID } from 'node:crypto'
import { NATIVE_CHAT_USAGE_LIMIT_CONTINUE_MESSAGE } from '../../../../shared/native-chat/native-chat-usage-limit-cut'
@@ -10,37 +11,12 @@ import {
} from './structured-agent-session-usage-limit-chain'
import { structuredAgentSessionMessageSendMutation } from '../../../../shared/agent-session/requests/structured-agent-session-send-mutation'
import type { AgentSessionStatusSummary } from '../../../../shared/agent-session/agent-session-wire'
import type { StructuredAgentSessionHostDeps as HostDeps } from '../host/structured-agent-session-host-types'
import type { AgentSessionRecordStore } from '../../../runtime/agent-session/records/agent-session-record-store'
import type { sendStructuredAgentSessionTurn } from '../operations/structured-agent-session-host-mutations'
import { RestartContinuationSupersededError } from './structured-agent-session-restart-continuation'
import { UsageLimitAccountHolds } from './structured-agent-session-usage-limit-account-hold'
import { UsageLimitColdCuts } from './structured-agent-session-usage-limit-cold-cuts'
type Cut = HostUsageLimitCut & { resetsAt: number; turnEndedAt: number }
type Attempt = { identity: string; count: number; at: number; unconfirmed?: boolean }
type Deps = {
host: Pick<HostDeps, 'usageLimitAutoContinue' | 'logger' | 'launchAccountHome'> & {
store: Pick<
AgentSessionRecordStore,
| 'listVisibleSessionIds'
| 'getRecord'
| 'hostId'
| 'isSessionTabVisible'
| 'cancelUsageLimitContinue'
>
}
sessions: ReadonlyMap<string, unknown>
/** Reads an unopened chat's saved cut; history-sized, so the sweep budgets it. */
readSavedCut: (id: string) => Promise<HostUsageLimitCut | null>
serialize: <T>(id: string, run: () => Promise<T>) => Promise<T>
send: (
params: Parameters<typeof sendStructuredAgentSessionTurn>[2]
) => ReturnType<typeof sendStructuredAgentSessionTurn>
readLoadedCut: (id: string) => Promise<HostUsageLimitCut | null>
publish: (id: string) => void
now: () => number
}
export { readSavedUsageLimitCut } from './structured-agent-session-usage-limit-chain'
@@ -54,7 +30,7 @@ export class StructuredAgentSessionUsageLimitContinue {
private readonly holds = new UsageLimitAccountHolds()
private readonly cold: UsageLimitColdCuts
constructor(private readonly deps: Deps) {
constructor(private readonly deps: UsageLimitContinueDeps) {
this.cold = new UsageLimitColdCuts({ ...deps, logger: deps.host.logger })
}
@@ -141,7 +117,9 @@ export class StructuredAgentSessionUsageLimitContinue {
turnItemId: cut.turnItemId,
resetsAt: cut.resetsAt,
// While the reset is unknown, `hold.resetsAt` is a reading time, not a reset.
...(!hold.unknownReset && hold.resetsAt > cut.resetsAt ? { continueAt: hold.resetsAt } : {}),
...(!hold.unknownReset && hold.resetsAt !== cut.resetsAt
? { continueAt: hold.resetsAt }
: {}),
state
}
}
@@ -181,13 +159,7 @@ export class StructuredAgentSessionUsageLimitContinue {
if (this.running || this.disposed) {
return
}
// Off, nothing can be scheduled; the settings listener publishes the switch itself.
if (!this.enabled()) {
if (this.holds.size + this.cold.size + this.attempts.size > 0) {
this.retainShown(this.deps.host.store.listVisibleSessionIds())
}
return
}
// Off, nothing is sent, but an account switch still moves when Continue applies.
this.running = true
try {
const visible = this.deps.host.store.listVisibleSessionIds()
@@ -206,10 +178,10 @@ export class StructuredAgentSessionUsageLimitContinue {
if (!cut || previous?.identity !== identity(cut)) {
this.attempts.delete(id)
}
// Before the overdue check: after a restart only the reading still knows a later reset.
// Only a reading taken after the cut's reset counts, so attribution waits for it.
if (cut?.resetsAt !== undefined && this.deps.now() >= cut.resetsAt) {
await this.holds.attribute(id, this.deps.host)
// Before the overdue check: after a restart only the reading still knows a later reset,
// and a switched account frees the chat before it.
if (cut?.resetsAt !== undefined) {
await this.attribute(id, cut)
}
const attempt = this.attempts.get(id)
if (!cut || !this.eligible(id, cut) || attempt?.unconfirmed || (attempt?.count ?? 0) >= 5) {
@@ -235,6 +207,13 @@ export class StructuredAgentSessionUsageLimitContinue {
}
}
/** The account a cut hit is the one its turn's runtime launched under, when the record names it. */
private attribute(id: string, cut: HostUsageLimitCut): Promise<void> {
const launched = this.deps.host.store.getRecord(id)?.launchedAccount
const home = launched && launched.fence === cut.fence ? launched.accountHome : null
return this.holds.attribute(id, this.deps.host, { cut: identity(cut), home }, this.deps.now())
}
private async deferUnchangedCut(id: string, cut: Cut, at: number): Promise<void> {
const current = await this.cold.read(id)
// Admission also refuses temporary backlog; yield the next passes without spending a failure.
@@ -252,7 +231,7 @@ export class StructuredAgentSessionUsageLimitContinue {
private async send(id: string, cut: Cut): Promise<void> {
// Earlier sends take time; moved sources resolve again, once per new witness for every chat.
if (!this.holds.attributionCurrent(id, this.deps.host)) {
await this.holds.attribute(id, this.deps.host)
await this.attribute(id, cut)
}
const record = this.deps.host.store.getRecord(id)
if (!record) {
@@ -30,7 +30,8 @@ it('does no discovery on ordinary reads and drops preparation superseded by a se
usageLimitCut: async () =>
hostUsageLimitCut(
snapshot,
latestNativeChatUsageLimitCut(snapshot.items, snapshot.submissions)
latestNativeChatUsageLimitCut(snapshot.items, snapshot.submissions),
() => undefined
)
}
}
@@ -133,7 +134,8 @@ it('recovers an unpinned OMP native file using the owning host directory and flo
usageLimitCut: async () =>
hostUsageLimitCut(
snapshot,
latestNativeChatUsageLimitCut(snapshot.items, snapshot.submissions)
latestNativeChatUsageLimitCut(snapshot.items, snapshot.submissions),
() => undefined
)
}
}
@@ -38,7 +38,7 @@ import {
setOrcaCloudSignInLink,
startOrcaCloudSignInProgress
} from './profile-cloud-sign-in-progress'
import { clearRelayRegionPreferenceCache } from '../../runtime/mobile-cloud-relay/relay-region-preference'
import { clearRelayRegionPreferenceCache } from '../../runtime/mobile-cloud-relay/region-selection/relay-region-preference'
import './profile-cloud-capability-refresh'
@@ -109,7 +109,7 @@ export async function readClaudeOAuthCredentials(
return keychain
}
try {
return parseClaudeOAuthCredentialsJson(
const file = parseClaudeOAuthCredentialsJson(
await readFile(
path.join(
options?.credentialsFileConfigDir ?? path.join(homedir(), '.claude'),
@@ -119,6 +119,10 @@ export async function readClaudeOAuthCredentials(
),
'credentials-file'
)
// A file with no token must not hide a refreshable or unreadable Keychain login.
return file.token || !(keychain.hasRefreshableCredentials || keychain.unavailable)
? file
: keychain
} catch (error) {
return isDefinitiveAbsence(error) ? keychain : { ...keychain, unavailable: true }
}
@@ -1,5 +1,6 @@
import type { AgentSessionRecord } from '../../../../shared/agent-session/lifecycle/agent-session-record'
import type { AgentSessionProviderHandleLink } from '../../../../shared/agent-session/lifecycle/agent-session-provider-handle'
import type { AgentSessionAccountHome } from '../../../../shared/agent-session/lifecycle/agent-session-account-home'
import {
withAcquiredAgentSessionAccount,
type ClaudeAccountRecipe
@@ -21,9 +22,11 @@ export function createAgentSessionOwnerProof(
now: number
leaseTtlMs?: number
accountAcquisition?: { generation: string; recipe: ClaudeAccountRecipe }
/** The account this runtime launched under, when the provider reports one. */
launchedAccountHome?: AgentSessionAccountHome
}): Promise<AgentSessionRecord> =>
mutate(args.sessionId, (record) =>
withAcquiredAgentSessionAccount(
mutate(args.sessionId, (record) => {
const { launchedAccount: _previous, ...proved } = withAcquiredAgentSessionAccount(
proveAgentSessionOwner({
record,
fence: args.fence,
@@ -33,5 +36,12 @@ export function createAgentSessionOwnerProof(
}),
args.accountAcquisition
)
)
// A runtime that reports no account leaves no stale claim about where this fence ran.
return args.launchedAccountHome
? {
...proved,
launchedAccount: { fence: args.fence, accountHome: args.launchedAccountHome }
}
: proved
})
}
@@ -126,8 +126,8 @@ export function applyAgentSessionRecordCommand(
case 'process':
next = commitAgentSessionProcessIdentity({ ...command.args, record })
break
case 'prove':
next = withAcquiredAgentSessionAccount(
case 'prove': {
const { launchedAccount: _previous, ...proved } = withAcquiredAgentSessionAccount(
proveAgentSessionOwner({
...command.args,
record,
@@ -135,7 +135,17 @@ export function applyAgentSessionRecordCommand(
}),
command.args.accountAcquisition
)
next = command.args.launchedAccountHome
? {
...proved,
launchedAccount: {
fence: command.args.fence,
accountHome: command.args.launchedAccountHome
}
}
: proved
break
}
case 'evict':
next = evictAgentSessionOwner({ ...command.args, record })
break
@@ -1,6 +1,6 @@
import type { OrcaCloudAuthConfig } from '../../orca-profiles/cloud-auth/profile-cloud-auth-config'
import { ensureActiveOrcaProfile } from '../../orca-profiles/profile-index-store'
import { readFreshOrcaCloudSession } from '../../orca-profiles/cloud-session/profile-cloud-session-refresh'
import type { OrcaCloudAuthConfig } from '../../../orca-profiles/cloud-auth/profile-cloud-auth-config'
import { ensureActiveOrcaProfile } from '../../../orca-profiles/profile-index-store'
import { readFreshOrcaCloudSession } from '../../../orca-profiles/cloud-session/profile-cloud-session-refresh'
import type { RelayAuthContext } from './relay-auth-identity'
export async function readRelayAuthContext(
@@ -0,0 +1,65 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import '../desktop-relay-service'
import { RelayAuthCoordinator } from './relay-auth-coordinator'
import type { RelayAuthContext } from './relay-auth-identity'
import { RelayHttpError } from '../director/relay-http-client'
import '../session-lifecycle/relay-session-broker'
const context: RelayAuthContext = {
identity: { userId: 'user-1', profileId: 'profile-1', organizationId: 'org-1' },
accessToken: 'access-1',
relayEntitled: true
}
describe('relay readiness', () => {
afterEach(() => vi.useRealTimers())
it('stops waiting when a fence cancels the scheduled retry', async () => {
vi.useFakeTimers()
const coordinator = new RelayAuthCoordinator({
readContext: async () => context,
openBroker: async () => {
throw new RelayHttpError('token-exchange', 502)
},
onStatus: () => {},
random: () => 1
})
coordinator.reconcile()
const waiting = coordinator.waitForReadiness(20_000)
await vi.advanceTimersByTimeAsync(0)
coordinator.fenceAndCloseNow()
await expect(waiting).resolves.toEqual({
ready: false,
reason: 'control_not_active',
retryAt: null
})
})
function coldStartWithHungRetry(): RelayAuthCoordinator {
const openBroker = vi
.fn()
.mockRejectedValueOnce(new RelayHttpError('assignment', 503))
.mockImplementationOnce(() => new Promise(() => {}))
return new RelayAuthCoordinator({
readContext: async () => context,
openBroker,
onStatus: () => {},
random: () => 0.5
})
}
it('returns at its deadline while the retry it followed is still opening', async () => {
vi.useFakeTimers()
const coordinator = coldStartWithHungRetry()
coordinator.reconcile()
let settled: unknown = null
void coordinator.waitForReadiness(10_000).then((readiness) => (settled = readiness))
await vi.advanceTimersByTimeAsync(9_999)
expect(settled).toBeNull()
await vi.advanceTimersByTimeAsync(1)
expect(settled).toMatchObject({ ready: false, reason: 'director_assignment_503' })
coordinator.stop()
})
})

Some files were not shown because too many files have changed in this diff Show More