From 0f22e1e9051663a627bee78d2be91e44459fa5b2 Mon Sep 17 00:00:00 2001 From: Jinjing <6427696+AmethystLiang@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:16:40 -0700 Subject: [PATCH 01/76] Fix table header transparency with opaque background (#18499) Replace the translucent bg-muted/25 with an opaque color-mix blend (40% muted on background) to ensure scrolled rows don't show through the sticky header. Add test coverage for header styling and layout. --- .../AutomationListTableHeader.test.tsx | 45 +++++++++++++++++++ src/renderer/src/lib/list-table-layout.ts | 4 +- 2 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 src/renderer/src/components/automations/AutomationListTableHeader.test.tsx diff --git a/src/renderer/src/components/automations/AutomationListTableHeader.test.tsx b/src/renderer/src/components/automations/AutomationListTableHeader.test.tsx new file mode 100644 index 00000000000..5c5bbe8e568 --- /dev/null +++ b/src/renderer/src/components/automations/AutomationListTableHeader.test.tsx @@ -0,0 +1,45 @@ +// @vitest-environment happy-dom + +import { cleanup, render, screen } from '@testing-library/react' +import { afterEach, describe, expect, it } from 'vitest' +import { AutomationListTableHeader } from './AutomationListTableHeader' +import { + LIST_TABLE_HEADER_CLASS, + LIST_TABLE_STICKY_HEADER_CELL_CLASS +} from '@/lib/list-table-layout' + +describe('AutomationListTableHeader', () => { + afterEach(cleanup) + + it('renders all expected columns', () => { + render() + + expect(screen.getByText('Name')).toBeDefined() + expect(screen.getByText('Schedule')).toBeDefined() + expect(screen.getByText('Project')).toBeDefined() + expect(screen.getByText('Host')).toBeDefined() + expect(screen.getByText('Next run')).toBeDefined() + expect(screen.getByText('Last run')).toBeDefined() + expect(screen.getByText('Status')).toBeDefined() + expect(screen.getByText('Agent')).toBeDefined() + expect(screen.getByText('Actions')).toBeDefined() + }) + + it('uses opaque background and sticky positioning on the header row', () => { + const { container } = render() + const header = container.firstElementChild as HTMLElement + + expect(header.className).toContain(LIST_TABLE_HEADER_CLASS) + expect(header.className).toContain('sticky') + expect(header.className).toContain('top-0') + expect(header.className).toContain('bg-[color-mix(in_srgb,var(--muted)_40%,var(--background))]') + expect(header.className).not.toContain('bg-muted/25') + }) + + it('applies sticky cell styling to the first column', () => { + render() + const nameCell = screen.getByText('Name') + + expect(nameCell.className).toBe(LIST_TABLE_STICKY_HEADER_CELL_CLASS) + }) +}) diff --git a/src/renderer/src/lib/list-table-layout.ts b/src/renderer/src/lib/list-table-layout.ts index c40607ee944..89b85d0aab5 100644 --- a/src/renderer/src/lib/list-table-layout.ts +++ b/src/renderer/src/lib/list-table-layout.ts @@ -5,9 +5,9 @@ */ export const LIST_TABLE_CONTAINER_CLASS = 'rounded-md border border-border/50 bg-muted/20' -// Why: z-30 must beat the rows' sticky first cells (z-20) so the header still covers them. +// Why: z-30 and opaque wash ensure scrolled rows cannot show through the sticky header. export const LIST_TABLE_HEADER_CLASS = - 'sticky top-0 z-30 h-8 items-center gap-3 border-b border-border/50 bg-muted/25 px-3 text-[11px] font-medium uppercase tracking-[0.08em] text-muted-foreground' + 'sticky top-0 z-30 h-8 items-center gap-3 border-b border-border/50 bg-[color-mix(in_srgb,var(--muted)_40%,var(--background))] px-3 text-[11px] font-medium uppercase tracking-[0.08em] text-muted-foreground' // Why: keep keyboard-selected rows clear of the sticky table header. export const LIST_TABLE_ROW_CLASS = From 3cd817c250e41e91fa050e0e5ffe8c4c8757b032 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 18:31:46 +0000 Subject: [PATCH 02/76] Update README downloads badge --- docs/assets/readme-downloads.svg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/assets/readme-downloads.svg b/docs/assets/readme-downloads.svg index 39fbcf45af1..ef8ebb61bb4 100644 --- a/docs/assets/readme-downloads.svg +++ b/docs/assets/readme-downloads.svg @@ -1,5 +1,5 @@ - - downloads: 37m + + downloads: 38m @@ -15,7 +15,7 @@ downloads downloads - 37m - 37m + 38m + 38m From fbea749d07adee7840f1bb5c8b9f33ddf90c908a Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:36:41 -0400 Subject: [PATCH 03/76] chore(cloud): pin staging relay c3 to the director's image (#18508) * chore(cloud): pin staging relay c3 to the director's image Mirrors stablyai/orca-cloud#468. c3 stayed on sha-c91439af after the director and c4 moved to sha-e3e92d95, so the staging capacity proof's compatible-director-image check has failed since 2026-08-14. * test(cloud): scope the launch-image pin to staging C4 now that C3 shares the digest * test(cloud): keep the public workflow assertions; scope only the launch-image pin to C4 --- .../relay-staging-c4-refresh-workflow.test.mjs | 11 +++++++---- cloud/infra/terraform/environments/staging.tfvars | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs b/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs index 75bbb9ac8d5..0e777b06c38 100644 --- a/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs +++ b/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs @@ -39,14 +39,17 @@ const launchDigest = '5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70ca // so a file-wide count no longer isolates Asia. const asiaCells = ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'] -function productionCell(cellId) { - const start = productionTfvars.indexOf(`"${cellId}"`) +function cellBlock(tfvars, cellId) { + const start = tfvars.indexOf(`"${cellId}"`) assert.notEqual(start, -1, `${cellId} is missing`) - return productionTfvars.slice(start, productionTfvars.indexOf('\n }', start)) + return tfvars.slice(start, tfvars.indexOf('\n }', start)) } +const productionCell = (cellId) => cellBlock(productionTfvars, cellId) + +// Scoped to C4 by name: staging C3 serves this digest too since its 2026-09-03 re-pin. test('pins staging C4 and all production Asia cells to the same launch image', () => { - assert.equal(stagingTfvars.match(new RegExp(launchDigest, 'g'))?.length, 1) + assert.match(cellBlock(stagingTfvars, 'staging-gce-c4'), new RegExp(`relay@sha256:${launchDigest}"`)) for (const cellId of asiaCells) { assert.match(productionCell(cellId), new RegExp(`relay@sha256:${launchDigest}"`), cellId) } diff --git a/cloud/infra/terraform/environments/staging.tfvars b/cloud/infra/terraform/environments/staging.tfvars index 3ee108fe874..bb894744612 100644 --- a/cloud/infra/terraform/environments/staging.tfvars +++ b/cloud/infra/terraform/environments/staging.tfvars @@ -67,7 +67,7 @@ relay_gce_cells = { boot_disk_gb = 30 boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-7" capacity_requests = 4000 - image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:9fba2a189ab3fa29853800830e77c7551ab3aaa8f43f3cd9adbdea28b876a8b9" + image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563" initially_enabled = false connection_hard_cap = 1000 connection_unobserved_bound = 60 From 0746d82c019aa710c1ef19c78e70edb0d7ca7d63 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:57:00 -0400 Subject: [PATCH 04/76] chore(cloud): close the Workload Identity cutover onto stablyai/orca (#18509) Mirrors stablyai/orca-cloud#470. The private relay workflows are retired, so the dual accept has one live arm left. Add `github_workflow_file_prefix` for the primary repository's workflow filenames, point `github_repo`/ `github_repo_id` at `stablyai/orca` (`1183888342`), and empty `github_accepted_repositories` in both environments. Every relay provider goes back to a single arm naming `cloud-` prefixed workflow refs. `cloud/infra/terraform` stays byte-identical to the private branch. The two identity tests diverge here as they already did, so they take the same change rather than the same bytes: both now render the trusted ref head from the Terraform variable instead of this checkout's own workflow filenames, which is what lets the length pin be the same 791 characters in either repository. --- cloud/dev/scripts/relay-repository.mjs | 9 ++- cloud/dev/scripts/relay-repository.test.mjs | 3 + .../relay-staging-deploy-identity.test.mjs | 15 ++-- ...oad-identity-attribute-conditions.test.mjs | 69 +++++++++---------- cloud/infra/terraform/README.md | 43 +++++------- .../terraform/environments/production.tfvars | 18 ++--- .../terraform/environments/staging.tfvars | 18 ++--- cloud/infra/terraform/relay-shared.tf | 10 +-- cloud/infra/terraform/variables.tf | 28 +++++--- 9 files changed, 101 insertions(+), 112 deletions(-) diff --git a/cloud/dev/scripts/relay-repository.mjs b/cloud/dev/scripts/relay-repository.mjs index bf41bed8012..7e8b01e4799 100644 --- a/cloud/dev/scripts/relay-repository.mjs +++ b/cloud/dev/scripts/relay-repository.mjs @@ -15,9 +15,16 @@ export function relayWorkflowFile(name) { return `${RELAY_WORKFLOW_FILE_PREFIX}${name}` } +// Repository-relative path for a repository that renames its copies with `prefix`. Terraform's +// trusted prefix is a variable and need not be this checkout's, so callers rendering a +// workflow_ref from Terraform pass it in rather than assuming the local one. +export function prefixedRelayWorkflowPath(prefix, name) { + return `.github/workflows/${prefix}${name}` +} + // Repository-relative path, the shape GitHub reports in workflow_ref and evidence payloads. export function relayWorkflowPath(name) { - return `.github/workflows/${relayWorkflowFile(name)}` + return prefixedRelayWorkflowPath(RELAY_WORKFLOW_FILE_PREFIX, name) } export function relayWorkflowUrl(name) { diff --git a/cloud/dev/scripts/relay-repository.test.mjs b/cloud/dev/scripts/relay-repository.test.mjs index 56383db4a1e..cf33f869773 100644 --- a/cloud/dev/scripts/relay-repository.test.mjs +++ b/cloud/dev/scripts/relay-repository.test.mjs @@ -5,6 +5,7 @@ import { fileURLToPath } from 'node:url' import { RELAY_GITHUB_REPOSITORY, RELAY_WORKFLOW_FILE_PREFIX, + prefixedRelayWorkflowPath, readRelayWorkflow, relayWorkflowFile, relayWorkflowPath, @@ -21,6 +22,8 @@ test('workflow identity is derived, never restated', () => { assert.equal(relayWorkflowFile('deploy-relay-staging.yml'), `${RELAY_WORKFLOW_FILE_PREFIX}deploy-relay-staging.yml`) assert.equal(relayWorkflowPath('deploy-relay-staging.yml'), `.github/workflows/${relayWorkflowFile('deploy-relay-staging.yml')}`) assert.ok(relayWorkflowUrl('deploy-relay-staging.yml').pathname.endsWith(relayWorkflowPath('deploy-relay-staging.yml'))) + // A caller rendering Terraform's trusted ref supplies that prefix instead of this checkout's. + assert.equal(prefixedRelayWorkflowPath('cloud-', 'deploy-relay-staging.yml'), '.github/workflows/cloud-deploy-relay-staging.yml') assert.match(readRelayWorkflow('deploy-relay-staging.yml'), /^name:/m) assert.match(RELAY_GITHUB_REPOSITORY, /^[\w.-]+\/[\w.-]+$/) }) diff --git a/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs b/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs index fece62f9ea2..8afbfb5ca94 100644 --- a/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs +++ b/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs @@ -2,11 +2,7 @@ import assert from 'node:assert/strict' import { readFileSync } from 'node:fs' import test from 'node:test' import { readWorkflow, workflowFiles } from './cloud-sql-rollout-lock-census.mjs' -import { - RELAY_WORKFLOW_FILE_PREFIX, - relayWorkflowFile, - relayWorkflowPath -} from './relay-repository.mjs' +import { prefixedRelayWorkflowPath, relayWorkflowFile } from './relay-repository.mjs' const identity = readFileSync( new URL('../../infra/terraform/relay-staging-deploy-iam.tf', import.meta.url), @@ -128,8 +124,11 @@ test('the rendered attribute condition stays inside the provider limit', () => { `assertion.repository_id == '${variableDefault('github_repo_id')}'`, `assertion.repository_owner_id == '${variableDefault('github_owner_id')}'` ] + // The prefix is the Terraform variable, not this checkout's own workflow filenames: the + // condition names the files as the trusted repository carries them. + const prefix = variableDefault('github_workflow_file_prefix') const workflowRefs = providerWorkflowFiles().map( - (file) => `${repository}/${relayWorkflowPath(file)}@refs/heads/main` + (file) => `${repository}/${prefixedRelayWorkflowPath(prefix, file)}@refs/heads/main` ) const rendered = [ ...claims, @@ -138,9 +137,7 @@ test('the rendered attribute condition stays inside the provider limit', () => { `(${workflowRefs.map((ref) => `assertion.workflow_ref == '${ref}'`).join(' || ')})` ].join(' && ') assert.ok(rendered.length < 4096, `rendered condition is ${rendered.length} characters`) - // 797 is the private repository's rendered length. This copy prefixes every workflow filename, - // which is the only difference, so the pin still moves the moment a workflow is added or dropped. - assert.equal(rendered.length, 797 + workflowRefs.length * RELAY_WORKFLOW_FILE_PREFIX.length) + assert.equal(rendered.length, 791) }) // Why: the census is the point. A binding added here without a workflow step behind it, or one diff --git a/cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs b/cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs index f25e442d8c0..1d3f3ce4d79 100644 --- a/cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs +++ b/cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs @@ -13,13 +13,13 @@ const EXPECTED_CONDITIONS = { staging: { relay: { github_staging_relay_capacity: - "assertion.ref == 'refs/heads/main' && assertion.environment == 'staging' && ((assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && (assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/bootstrap-relay-staging-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/prove-relay-staging-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/recover-relay-staging-c4-image.yml@refs/heads/main')) || (assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-bootstrap-relay-staging-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-prove-relay-staging-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-recover-relay-staging-c4-image.yml@refs/heads/main')))", + "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'staging' && (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-bootstrap-relay-staging-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-prove-relay-staging-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-recover-relay-staging-c4-image.yml@refs/heads/main')", github_staging_relay_deploy: - "assertion.ref == 'refs/heads/main' && assertion.environment == 'staging' && ((assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && (assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/bootstrap-relay-staging-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-staging-gce-candidate.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-staging.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/power-relay-staging.yml@refs/heads/main')) || (assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-bootstrap-relay-staging-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-staging-gce-candidate.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-staging.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-power-relay-staging.yml@refs/heads/main')))", + "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'staging' && (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-bootstrap-relay-staging-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-staging-gce-candidate.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-staging.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-power-relay-staging.yml@refs/heads/main')", github_relay_asia_topology: - "assertion.ref == 'refs/heads/main' && assertion.environment == 'staging' && assertion.event_name == 'workflow_dispatch' && ((assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-asia-topology.yml@refs/heads/main') || (assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-asia-topology.yml@refs/heads/main'))", + "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'staging' && assertion.event_name == 'workflow_dispatch' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-asia-topology.yml@refs/heads/main'", github_relay_asia_proof: - "assertion.ref == 'refs/heads/main' && assertion.environment == 'staging' && assertion.event_name == 'workflow_dispatch' && ((assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/prove-relay-asia-staging.yml@refs/heads/main') || (assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-prove-relay-asia-staging.yml@refs/heads/main'))", + "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'staging' && assertion.event_name == 'workflow_dispatch' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-prove-relay-asia-staging.yml@refs/heads/main'", }, // The relay root creates this provider only in production, so staging has exactly one // definition and it lives here. @@ -31,15 +31,15 @@ const EXPECTED_CONDITIONS = { production: { relay: { github: - "assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && ((assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-fence-broker.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-director.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-multi-target.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/publish-relay-production.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/operate-relay-production-rehome.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca-cloud/.github/workflows/operate-relay-production-rehome-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-same-cap.yml@refs/heads/main' && (assertion.job_workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-same-cap-job.yml@refs/heads/main' || assertion.job_workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-same-cap.yml@refs/heads/main')))) || (assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && ((assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-fence-broker.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-director.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-publish-relay-production.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main' && (assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml@refs/heads/main' || assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main')))))", + "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-fence-broker.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-director.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-publish-relay-production.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main' && (assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml@refs/heads/main' || assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main')))", github_monitor: - "assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/monitor-relay-production.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca-cloud/.github/workflows/monitor-relay-production-job.yml@refs/heads/main') || (assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-monitor-relay-production.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-monitor-relay-production-job.yml@refs/heads/main'))", + "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-monitor-relay-production.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-monitor-relay-production-job.yml@refs/heads/main'", github_fence: - "assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-multi-target.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-multi-target.yml@refs/heads/main') || (assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main'))", + "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main'", github_production_relay_capacity: - "assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && ((assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-capacity.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-capacity-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-same-cap.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-production-same-cap-job.yml@refs/heads/main'))) || (assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && ((assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml@refs/heads/main'))))", + "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml@refs/heads/main'))", github_relay_asia_topology: - "assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.event_name == 'workflow_dispatch' && ((assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca-cloud/.github/workflows/deploy-relay-asia-topology.yml@refs/heads/main') || (assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-asia-topology.yml@refs/heads/main'))", + "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.event_name == 'workflow_dispatch' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-asia-topology.yml@refs/heads/main'", }, apps: { github_production_app_deploy: @@ -48,20 +48,21 @@ const EXPECTED_CONDITIONS = { }, } -// Every repository the relay root accepts while the public extraction runs, with the workflow-ref -// head each one contributes. The apps root is not part of the dual accept. -const ACCEPTED_REPOSITORIES = [ - { - claims: - "assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420'", - workflowHead: 'stablyai/orca-cloud/.github/workflows/' - }, - { +// The one repository each root trusts, with the workflow-ref head it contributes. The relay root +// moved to the public repository, where the workflow files carry the `cloud-` prefix; the apps +// root still deploys from the private one. +const ROOT_REPOSITORIES = { + relay: { claims: "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420'", workflowHead: 'stablyai/orca/.github/workflows/cloud-' + }, + apps: { + claims: + "assertion.repository == 'stablyai/orca-cloud' && assertion.repository_id == '1273841466' && assertion.repository_owner_id == '127256420'", + workflowHead: 'stablyai/orca-cloud/.github/workflows/' } -] +} // [root, provider, condition] for every provider the environment creates, across all roots. async function flatten(environment) { @@ -103,9 +104,7 @@ for (const environment of Object.keys(EXPECTED_CONDITIONS)) { test(`${environment} pins repository, branch, and environment on every provider`, async () => { for (const [root, provider, condition] of await flatten(environment)) { for (const pin of [ - "assertion.repository == 'stablyai/orca-cloud'", - "assertion.repository_id == '1273841466'", - "assertion.repository_owner_id == '127256420'", + ROOT_REPOSITORIES[root].claims, "assertion.ref == 'refs/heads/main'", `assertion.environment == '${environment}'` ]) { @@ -131,27 +130,23 @@ for (const environment of Object.keys(EXPECTED_CONDITIONS)) { }) } -// Why: the dual accept is only safe if each OR arm carries its own repository claims. An arm that -// inherited them, or a workflow ref that named the other repository, would let one repository's -// workflows run under the other's proof. +// Why: the cutover left one arm per relay provider. A leftover `stablyai/orca-cloud` claim or +// workflow ref would keep trusting a repository whose relay workflows are retired, and an unprefixed +// ref would name a file the public repository does not have. for (const environment of Object.keys(EXPECTED_CONDITIONS)) { - test(`${environment} admits both repositories through every relay provider`, async () => { + test(`${environment} admits only the public repository through every relay provider`, async () => { + const { claims, workflowHead } = ROOT_REPOSITORIES.relay const rendered = await renderAttributeConditions(environment) for (const [provider, condition] of Object.entries(rendered.relay)) { - assert.ok( - condition.startsWith("assertion.ref == 'refs/heads/main' && "), - `${provider} does not lead with the repository-independent claims` - ) + assert.ok(condition.startsWith(`${claims} && `), `${provider} does not lead with the claims`) + assert.doesNotMatch(condition, /stablyai\/orca-cloud|1273841466/, `${provider} keeps an old arm`) const refs = [...condition.matchAll(/(?:job_)?workflow_ref == '([^']+)'/g)].map( (match) => match[1] ) - const perRepository = ACCEPTED_REPOSITORIES.map((repository) => { - assert.ok(condition.includes(`(${repository.claims} && `), `${provider} misses an arm`) - return refs.filter((ref) => ref.startsWith(repository.workflowHead)).length - }) - assert.equal(refs.length, perRepository[0] + perRepository[1], `${provider} names a stray ref`) - assert.equal(perRepository[0], perRepository[1], `${provider} arms are not the same size`) - assert.ok(perRepository[0] > 0, `${provider} names no workflow`) + assert.ok(refs.length > 0, `${provider} names no workflow`) + for (const ref of refs) { + assert.ok(ref.startsWith(workflowHead), `${provider} names a stray ref ${ref}`) + } } }) } diff --git a/cloud/infra/terraform/README.md b/cloud/infra/terraform/README.md index 2e2b9b02011..8aa7aa0a95c 100644 --- a/cloud/infra/terraform/README.md +++ b/cloud/infra/terraform/README.md @@ -37,35 +37,26 @@ identity (`google_service_account.github_deploy`, its provider, and its bindings with production-only counts; staging's copies are declared by `infra/terraform-apps`. An untargeted plan is orderable again; the `Plan:` line still reflects the standing cell-template drift backlog. -### Dual-accept Workload Identity during the public extraction +### Workload Identity trusts the public repository -While the relay source moves to the public `stablyai/orca` repository, every relay Workload -Identity provider accepts the same workflows from both repositories. `github_accepted_repositories` -lists the extra repositories; `relay-github-workflow-trust.tf` renders one parenthesised OR arm per -accepted repository, each arm carrying that repository's own `repository`, `repository_id`, and -`repository_owner_id` claims plus its exact workflow refs. `ref`, `environment`, and `event_name` -stay outside the OR. Workflow files keep their names in the private repo and take the -`workflow_file_prefix` (`cloud-`) in the public one. +The cutover closed on 2026-09-03. Every relay Workload Identity provider now accepts exactly one +repository, `stablyai/orca` (`1183888342`, owner `127256420`), and every workflow ref it names is +built from `github_workflow_file_prefix` (`cloud-`), which is the rename the public repo applies to +the workflow files it carries. `github_repo`, `github_repo_id`, and that prefix are set in both +`environments/*.tfvars` as well as defaulted here, and `github_accepted_repositories` is empty. +Nothing in this root trusts `stablyai/orca-cloud` any more; the apps and foundation roots still do, +because the app workflows still live there. -Adding a repository is a tfvars edit: no provider block changes, and the rendered strings are -pinned by `dev/scripts/workload-identity-attribute-conditions.test.mjs`. An empty list renders -byte-identically to the single-repository form, which is what makes the arms reviewable against -the pre-extraction condition. +`github_accepted_repositories` stays available for the next repository move. Each entry renders its +own parenthesised OR arm in `relay-github-workflow-trust.tf`, carrying that repository's own +`repository`, `repository_id`, and `repository_owner_id` claims plus its exact workflow refs, while +`ref`, `environment`, and `event_name` stay outside the OR. An empty list renders byte-identically +to the single-repository form, so adding and removing a repository is a tfvars edit with no provider +block change. The rendered strings are pinned by +`dev/scripts/workload-identity-attribute-conditions.test.mjs`. -Closing the cutover is an owner step, in this order: - -1. Retire the private workflows, so nothing runs from `stablyai/orca-cloud` any more. -2. Point `github_owner`, `github_repo`, `github_repo_id`, and `github_owner_id` at - `stablyai/orca` (`1183888342`, owner `127256420`), and set `workflow_file_prefix` for it by - moving the surviving entry's prefix onto the primary: the public files keep the `cloud-` names, - so the primary prefix becomes `cloud-` unless the files are renamed back. -3. Empty `github_accepted_repositories` in both `environments/*.tfvars`. -4. Re-render and update the pinned conditions, then apply. Each provider goes back to a single - arm, and `google_service_account_iam_member.github_accepted_repository_workload_identity_user` - is destroyed as the primary `attribute.repository` binding takes over. - -Step 2 and step 3 must land in the same apply: dropping the accepted entry before repointing the -primary would revoke the public repository mid-flight. +Repointing the primary and emptying the list must land in the same apply: dropping the accepted +entry before repointing the primary would revoke the surviving repository mid-flight. ### `ORCA_RELAY_IMAGE_DIGEST` is not Terraform-owned diff --git a/cloud/infra/terraform/environments/production.tfvars b/cloud/infra/terraform/environments/production.tfvars index 60d36e3d832..8e442c75900 100644 --- a/cloud/infra/terraform/environments/production.tfvars +++ b/cloud/infra/terraform/environments/production.tfvars @@ -5,19 +5,11 @@ region = "us-central1" artifact_repository_id = "orca-cloud" -# Dual accept while the relay source moves to the public stablyai/orca repository: the same -# workflows are trusted from both repos, and the public copies carry a `cloud-` file prefix. -# Remove this entry once the private workflows are retired and point github_owner/github_repo, -# github_repo_id, and github_owner_id at the surviving repository. -github_accepted_repositories = [ - { - owner = "stablyai" - repo = "orca" - repo_id = "1183888342" - owner_id = "127256420" - workflow_file_prefix = "cloud-" - } -] +# The relay source lives in the public stablyai/orca repository, where the workflows carry a +# `cloud-` file prefix. github_owner and github_owner_id keep their defaults. +github_repo = "orca" +github_repo_id = "1183888342" +github_workflow_file_prefix = "cloud-" # Our first-party auth service. auth.onorca.dev is PropelAuth's prod domain, so # our service lives at login.onorca.dev (desktop points ORCA_CLOUD_API_URL here). diff --git a/cloud/infra/terraform/environments/staging.tfvars b/cloud/infra/terraform/environments/staging.tfvars index bb894744612..4a32458fcd5 100644 --- a/cloud/infra/terraform/environments/staging.tfvars +++ b/cloud/infra/terraform/environments/staging.tfvars @@ -5,19 +5,11 @@ region = "us-central1" artifact_repository_id = "orca-cloud" -# Dual accept while the relay source moves to the public stablyai/orca repository: the same -# workflows are trusted from both repos, and the public copies carry a `cloud-` file prefix. -# Remove this entry once the private workflows are retired and point github_owner/github_repo, -# github_repo_id, and github_owner_id at the surviving repository. -github_accepted_repositories = [ - { - owner = "stablyai" - repo = "orca" - repo_id = "1183888342" - owner_id = "127256420" - workflow_file_prefix = "cloud-" - } -] +# The relay source lives in the public stablyai/orca repository, where the workflows carry a +# `cloud-` file prefix. github_owner and github_owner_id keep their defaults. +github_repo = "orca" +github_repo_id = "1183888342" +github_workflow_file_prefix = "cloud-" auth_base_url = "https://auth-staging.onorca.dev" diff --git a/cloud/infra/terraform/relay-shared.tf b/cloud/infra/terraform/relay-shared.tf index d4f0e5dac15..b1afc4d1890 100644 --- a/cloud/infra/terraform/relay-shared.tf +++ b/cloud/infra/terraform/relay-shared.tf @@ -14,16 +14,16 @@ locals { "assertion.repository_owner_id == '${var.github_owner_id}'", ] - # Dual accept during the public extraction: the primary repository first, then every repository - # var.github_accepted_repositories adds. Each one renders its own OR arm in every provider - # condition, so both repos can run the same workflows through the same identities. A repository - # that imports these workflows may rename the files, hence the per-repository prefix. + # The primary repository first, then every repository var.github_accepted_repositories adds. + # Each one renders its own OR arm in every provider condition, so a repository move can trust + # both repos at once. A repository that imports these workflows may rename the files, hence the + # per-repository prefix; the primary's is var.github_workflow_file_prefix. relay_github_accepted_repositories = concat([{ owner = var.github_owner repo = var.github_repo repo_id = var.github_repo_id owner_id = var.github_owner_id - workflow_file_prefix = "" + workflow_file_prefix = var.github_workflow_file_prefix }], var.github_accepted_repositories) relay_github_single_repository = length(local.relay_github_accepted_repositories) == 1 diff --git a/cloud/infra/terraform/variables.tf b/cloud/infra/terraform/variables.tf index c3e06ee280f..68f6c555bd3 100644 --- a/cloud/infra/terraform/variables.tf +++ b/cloud/infra/terraform/variables.tf @@ -22,14 +22,14 @@ variable "github_owner" { variable "github_repo" { type = string description = "GitHub repo allowed to deploy through Workload Identity Federation." - default = "orca-cloud" + default = "orca" } # Numeric IDs survive a rename or transfer of the repository; every provider pins them next to the name. variable "github_repo_id" { type = string description = "Numeric GitHub repository ID of github_owner/github_repo." - default = "1273841466" + default = "1183888342" validation { condition = can(regex("^[0-9]+$", var.github_repo_id)) @@ -48,12 +48,24 @@ variable "github_owner_id" { } } -# Additional repositories whose identical workflows the same identities must accept while the -# public extraction runs. Each entry renders its own OR arm in every provider condition, so the -# private repo keeps working while the public one takes over. `workflow_file_prefix` is the rename -# the importing repository applies to the workflow files it copies. Empty is the steady state: -# the final step of the cutover is to empty this list again and point github_owner/github_repo, -# github_repo_id, and github_owner_id at the surviving repository. +# The rename the relay repository applies to the workflow files it carries. The public repo keeps +# the workflows under `cloud-` names, so every relay workflow_ref is built from this head. +variable "github_workflow_file_prefix" { + type = string + description = "Filename prefix on github_owner/github_repo's copies of the relay workflows." + default = "cloud-" + + validation { + condition = can(regex("^[a-z0-9-]*$", var.github_workflow_file_prefix)) + error_message = "github_workflow_file_prefix must be lowercase letters, digits, or hyphens." + } +} + +# Additional repositories whose identical workflows the same identities must accept during a +# repository move. Each entry renders its own OR arm in every provider condition, so both repos +# can run the same workflows through the same identities. `workflow_file_prefix` is the rename the +# importing repository applies to the workflow files it copies. Empty is the steady state, and is +# where the public extraction left it: stablyai/orca is now the primary and only repository. variable "github_accepted_repositories" { type = list(object({ owner = string From 16e26245783744e232fb06695b2b8bb8844312b9 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:02:46 -0400 Subject: [PATCH 05/76] fix(terminal): flush xterm's parked renderer resize when releasing the pause latch (#18510) --- ...render-pause-release-parked-resize.test.ts | 113 ++++++++++++++++++ .../terminal-render-pause-release.ts | 36 ++++-- 2 files changed, 140 insertions(+), 9 deletions(-) create mode 100644 src/renderer/src/lib/pane-manager/terminal-render-pause-release-parked-resize.test.ts diff --git a/src/renderer/src/lib/pane-manager/terminal-render-pause-release-parked-resize.test.ts b/src/renderer/src/lib/pane-manager/terminal-render-pause-release-parked-resize.test.ts new file mode 100644 index 00000000000..7ad57fa7126 --- /dev/null +++ b/src/renderer/src/lib/pane-manager/terminal-render-pause-release-parked-resize.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it, vi } from 'vitest' +import { + forceFullViewportPresent, + forceRepaintThroughRenderPause, + requestFullViewportPresent +} from './terminal-render-pause-release' + +// Why a separate file: the parked-resize contract is one hazard shared by all +// three helpers, and the main spec is already at the max-lines budget. + +type FakeRenderService = { + _isPaused: boolean + _needsFullRefresh: boolean + _pausedResizeTask?: { flush: ReturnType } | null + refreshRows: ReturnType + _renderer?: { value?: { renderRows?: ReturnType } } +} + +function createPausedTerminal(options: { + synchronizedOutput?: boolean + withoutTask?: boolean + flushThrows?: boolean +}): { terminal: unknown; service: FakeRenderService; order: string[] } { + const order: string[] = [] + const flush = vi.fn(() => { + order.push('flush') + if (options.flushThrows) { + throw new Error('renderer disposed') + } + }) + const service: FakeRenderService = { + _isPaused: true, + _needsFullRefresh: true, + _pausedResizeTask: options.withoutTask ? null : { flush }, + refreshRows: vi.fn(() => order.push('refreshRows')), + _renderer: { value: { renderRows: vi.fn(() => order.push('renderRows')) } } + } + const terminal = { + rows: 24, + _core: { + _renderService: service, + coreService: { decPrivateModes: { synchronizedOutput: options.synchronizedOutput === true } } + } + } + return { terminal, service, order } +} + +const helpers = [ + ['forceRepaintThroughRenderPause', forceRepaintThroughRenderPause], + ['requestFullViewportPresent', requestFullViewportPresent], + ['forceFullViewportPresent', forceFullViewportPresent] +] as const + +describe.each(helpers)('%s parked renderer resize', (_name, present) => { + it('flushes the resize xterm parked while paused before presenting', () => { + // A resize that lands under _isPaused only parks WebglRenderer.handleResize; + // xterm flushes it solely from the observer callback we are pre-empting. + const { terminal, service, order } = createPausedTerminal({}) + + expect(present(terminal)).toBe(true) + expect(service._pausedResizeTask?.flush).toHaveBeenCalledTimes(1) + expect(order[0]).toBe('flush') + expect(order).toHaveLength(2) + expect(service._isPaused).toBe(false) + expect(service._needsFullRefresh).toBe(false) + }) + + it('flushes before a DEC 2026 present too', () => { + const { terminal, service, order } = createPausedTerminal({ synchronizedOutput: true }) + + expect(present(terminal)).toBe(true) + expect(service._pausedResizeTask?.flush).toHaveBeenCalledTimes(1) + expect(order[0]).toBe('flush') + }) + + it('still presents when the parked-task internal is unavailable', () => { + const { terminal, service } = createPausedTerminal({ withoutTask: true }) + + expect(present(terminal)).toBe(true) + expect(service._isPaused).toBe(false) + }) + + it('still presents when the parked resize throws', () => { + const { terminal, order } = createPausedTerminal({ flushThrows: true }) + + expect(present(terminal)).toBe(true) + expect(order).toEqual(['flush', expect.any(String)]) + }) +}) + +describe('parked renderer resize on an unpaused terminal', () => { + it('is left to xterm when the pause latch is not set', () => { + const flush = vi.fn() + const service = { + _isPaused: false, + _needsFullRefresh: false, + _pausedResizeTask: { flush }, + refreshRows: vi.fn() + } + const terminal = { + rows: 24, + _core: { + _renderService: service, + coreService: { decPrivateModes: { synchronizedOutput: true } } + } + } + + expect(requestFullViewportPresent(terminal)).toBe(true) + expect(forceFullViewportPresent(terminal)).toBe(true) + expect(forceRepaintThroughRenderPause(terminal)).toBe(false) + expect(flush).not.toHaveBeenCalled() + }) +}) diff --git a/src/renderer/src/lib/pane-manager/terminal-render-pause-release.ts b/src/renderer/src/lib/pane-manager/terminal-render-pause-release.ts index 552939e6cd8..9f823e84630 100644 --- a/src/renderer/src/lib/pane-manager/terminal-render-pause-release.ts +++ b/src/renderer/src/lib/pane-manager/terminal-render-pause-release.ts @@ -24,6 +24,7 @@ type MaybeWebglRenderer = { type MaybePausableRenderService = { _isPaused?: boolean _needsFullRefresh?: boolean + _pausedResizeTask?: { flush?: () => void } | null refreshRows?: (start: number, end: number, sync?: boolean) => void _renderer?: { value?: MaybeWebglRenderer | null } | MaybeWebglRenderer | null } @@ -41,6 +42,29 @@ type TerminalWithRenderService = { } } +/** + * Clears xterm's observer-pause latches and runs the renderer resize xterm parked + * while paused. + * + * Why the flush: `RenderService.handleResize` under `_isPaused` only parks the + * WebGL renderer's own resize on an idle task, and xterm flushes that task solely + * from the observer callback gated on `_needsFullRefresh`. Clearing the latch + * without flushing lets the present below paint the new grid through the old + * canvas/model geometry (misplaced fragments, stray bars until a user resize). + */ +function releaseRenderPause(service: PausableRenderService): void { + // Why: leave the latch as if the pending full refresh was serviced — we are + // about to service it — so the observer's next callback doesn't queue a + // redundant second full repaint. + service._isPaused = false + service._needsFullRefresh = false + try { + service._pausedResizeTask?.flush?.() + } catch { + // Why: a resize that throws mid-dispose must not block the present. + } +} + function getRenderService(terminal: unknown): PausableRenderService | null { const service = (terminal as TerminalWithRenderService | null)?._core?._renderService return service && typeof service.refreshRows === 'function' @@ -66,11 +90,7 @@ export function forceRepaintThroughRenderPause(terminal: unknown): boolean { return false } - // Why: leave the latch as if the pending full refresh was serviced — we are - // about to service it — so the observer's next callback doesn't queue a - // redundant second full repaint. - service._isPaused = false - service._needsFullRefresh = false + releaseRenderPause(service) try { service.refreshRows(0, rows - 1, true) return true @@ -102,8 +122,7 @@ export function requestFullViewportPresent(terminal: unknown): boolean { } if (paused) { - service._isPaused = false - service._needsFullRefresh = false + releaseRenderPause(service) } try { @@ -160,8 +179,7 @@ export function forceFullViewportPresent(terminal: unknown): boolean { } if (paused) { - service._isPaused = false - service._needsFullRefresh = false + releaseRenderPause(service) } const renderer = getRenderer(service) From 91c5a615c5dd46568d232d32fa72c1dc13bfb220 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 3 Sep 2026 13:22:42 -0700 Subject: [PATCH 06/76] fix(settings): indent the Agent sleep "Sleep after" sub-setting (#18379) "Sleep after" rendered flush with its parent toggle, unlike the Agent Dashboard and Chat UI sub-settings which sit inside the indented, left-bordered group. Reuse that same wrapper and drop the row's extra vertical padding so the block matches its siblings. Co-authored-by: Merge Sim --- .../components/settings/ExperimentalPane.tsx | 51 ++++++++++--------- .../settings/SettingsFormControls.tsx | 5 +- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/src/renderer/src/components/settings/ExperimentalPane.tsx b/src/renderer/src/components/settings/ExperimentalPane.tsx index 2ef34d943c8..e755db7e41b 100644 --- a/src/renderer/src/components/settings/ExperimentalPane.tsx +++ b/src/renderer/src/components/settings/ExperimentalPane.tsx @@ -190,30 +190,33 @@ export function ExperimentalPane({ /> {agentHibernationEnabled ? ( - - updateSettings({ - // Why: settings persist the planner contract, not the display unit. - agentHibernationIdleMs: minutes * MS_PER_MINUTE - }) - } - /> +
+ + updateSettings({ + // Why: settings persist the planner contract, not the display unit. + agentHibernationIdleMs: minutes * MS_PER_MINUTE + }) + } + /> +
) : null} ) : null} diff --git a/src/renderer/src/components/settings/SettingsFormControls.tsx b/src/renderer/src/components/settings/SettingsFormControls.tsx index bb38bcc8d4e..9a0993849f6 100644 --- a/src/renderer/src/components/settings/SettingsFormControls.tsx +++ b/src/renderer/src/components/settings/SettingsFormControls.tsx @@ -270,6 +270,7 @@ type NumberFieldProps = { integer?: boolean onChange: (value: number) => void suffix?: string + className?: string } export function ColorField({ @@ -315,7 +316,8 @@ export function NumberField({ step = 1, integer = false, onChange, - suffix + suffix, + className }: NumberFieldProps): React.JSX.Element { const [draft, setDraft] = useState(Number.isFinite(value) ? String(value) : '') const [prevValue, setPrevValue] = useState(value) @@ -346,6 +348,7 @@ export function NumberField({ return ( From a35451f5b91e693ef05cbc80dcc94c75518c6e85 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:37:42 -0400 Subject: [PATCH 07/76] fix(relay): stop self-closing the control socket on unknown messages (#18400) * fix(relay): stop self-closing the control socket on unknown messages The desktop control client tore its own relay control WebSocket down with code 4401 "unknown control message" for any well-formed control frame it did not recognize. handleMessage() funneled everything that was not ping / conn-open / drain / a tracked request reply into failProtocol('unknown control message'), which closes the socket and orphans the origin. Three real frames hit that branch: - A relay reply that arrives after the desktop's 10s request deadline already deleted the pending entry. Relay control operations run DB transactions that can exceed 10s under load, so resolveMessage() finds no waiter and returns false. - A control-error carrying no reqId (or an unknown one), including the relay's own 'unknown_control_message' reply to a host command it could not route. - A newer relay's opcode that this build predates. Fleet telemetry shows ~15 of these closes per day across app versions 1.4.175..1.4.197, so it is version-agnostic. The self-close was also far more costly than the message that caused it: the relay session dropped to 'orphaned' and answered the phone with HOST_OFFLINE (4404) for the orphan grace window, then the desktop had to re-register through the director's 503 reconnect throttle, stretching a single stray frame into minutes of mobile downtime. Per docs/reference/remote-wire-compatibility.md Rule 2, an unknown but well-formed control frame must be dropped, not treated as fatal. Log and ignore it; malformed JSON, binary frames, and messages before activation still close as protocol violations. Adds unit tests for the unknown-opcode drop, the timed-out-reply drop, and the preserved malformed-frame teardown. * docs(relay): correct the ignore rationale, drop the Rule 2 misattribution Rule 2 of remote-wire-compatibility governs the SENDER of a new terminal- stream opcode and treats the receiver's silent drop as a hazard, not a mandate. Reframe the comment around the actual justification: the decoder convention of dropping unknown frames, the control channel's lack of an opcode negotiation step, and the incident cost asymmetry. --- .../relay/relay-control-client.test.ts | 45 +++++++++++++++++++ .../runtime/relay/relay-control-client.ts | 13 +++++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/main/runtime/relay/relay-control-client.test.ts b/src/main/runtime/relay/relay-control-client.test.ts index 2975b67e631..d235f0ebacd 100644 --- a/src/main/runtime/relay/relay-control-client.test.ts +++ b/src/main/runtime/relay/relay-control-client.test.ts @@ -4,6 +4,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import nacl from 'tweetnacl' import { WebSocketServer, type WebSocket } from 'ws' import type { E2EEKeypair } from '../e2ee-keypair' +import { MOBILE_RELAY_CLOSE_CODE } from '../../../shared/mobile-relay-close-codes' import { RelayControlClient } from './relay-control-client' const encoder = new TextEncoder() @@ -550,4 +551,48 @@ describe('RelayControlClient scripted-socket lifecycle', () => { vi.advanceTimersByTime(91_000) expect(client.isLive()).toBe(false) }) + + it('ignores an unrecognized control message without closing the active control', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const { client, socket, onClose } = scriptedControl() + await client.connect() + expect(client.isLive()).toBe(true) + + // A newer relay opcode the desktop schema does not know. Rule 2 of + // remote-wire-compatibility: an unknown-but-well-formed frame is dropped, + // never fatal to a live control. + socket.deliver({ type: 'relay-hint', v: 2, hint: 'future-feature' }) + + expect(client.isLive()).toBe(true) + expect(socket.readyState).toBe(1) + expect(onClose).not.toHaveBeenCalled() + warn.mockRestore() + }) + + it('ignores a reply whose request already timed out instead of self-closing', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const { client, socket, onClose } = scriptedControl() + await client.connect() + + // A relay control-error carrying a reqId with no live waiter — e.g. a late + // reply that arrived after the desktop's request deadline deleted it, or the + // relay's no-op error for a command it could not route. Must not be fatal. + socket.deliver({ type: 'control-error', reqId: 'expired-req', code: 'unknown_control_message' }) + + expect(client.isLive()).toBe(true) + expect(socket.readyState).toBe(1) + expect(onClose).not.toHaveBeenCalled() + warn.mockRestore() + }) + + it('still tears down a malformed (non-JSON) control frame', async () => { + const { client, socket, onClose } = scriptedControl() + await client.connect() + + socket.emit('message', 'not-json{', false) + + expect(client.isLive()).toBe(false) + expect(socket.readyState).toBe(3) + expect(onClose).toHaveBeenCalledWith(MOBILE_RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL) + }) }) diff --git a/src/main/runtime/relay/relay-control-client.ts b/src/main/runtime/relay/relay-control-client.ts index 76816c81a3a..7e742173f72 100644 --- a/src/main/runtime/relay/relay-control-client.ts +++ b/src/main/runtime/relay/relay-control-client.ts @@ -234,7 +234,18 @@ export class RelayControlClient { if (this.requests.resolveMessage(message)) { return } - this.failProtocol('unknown control message') + // Drop a well-formed control message we do not recognize, matching how every + // other Orca decoder treats an unknown frame (see the silent-drop convention + // in docs/reference/remote-wire-compatibility.md). The control channel has no + // opcode negotiation step, so this reaches either a newer relay's message + // this build predates, or a reply whose request already timed out and has no + // waiter (relay control ops run DB transactions that can exceed the request + // deadline under load). Self-closing here was strictly worse than ignoring: + // it orphaned the relay session, which answered the phone with HOST_OFFLINE + // for the orphan-grace window plus the director's reconnect throttle — minutes + // of outage from a single stray frame. + const messageType = typeof message.type === 'string' ? message.type : 'unknown' + console.warn(`[relay] ignoring unrecognized control message type=${messageType}`) } private handleProofMessage(message: Record): void { From aa78d4af17c19549ab003738c2f5fae117e7e6a2 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:53:32 -0400 Subject: [PATCH 08/76] fix(release): restore version and harden staging confirmation Resolves release scan blockers STA-6611 and STA-6612. --- .../cloud-prove-relay-asia-staging.yml | 4 ++- config/scripts/release-blocker-fixes.test.mjs | 35 +++++++++++++++++++ package.json | 2 +- 3 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 config/scripts/release-blocker-fixes.test.mjs diff --git a/.github/workflows/cloud-prove-relay-asia-staging.yml b/.github/workflows/cloud-prove-relay-asia-staging.yml index 9a66e967b57..56677a98600 100644 --- a/.github/workflows/cloud-prove-relay-asia-staging.yml +++ b/.github/workflows/cloud-prove-relay-asia-staging.yml @@ -55,9 +55,11 @@ jobs: - name: Validate the exact staging proof request shell: bash + env: + CONFIRMATION: ${{ inputs.confirmation }} run: | set -euo pipefail - test "${{ inputs.confirmation }}" = PROVE_ASIA_STAGING + test "${CONFIRMATION}" = PROVE_ASIA_STAGING [[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]] [[ "${INITIAL_SELECTOR_GENERATION}" =~ ^[1-9][0-9]*$ ]] [[ "${PROMOTE_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]] diff --git a/config/scripts/release-blocker-fixes.test.mjs b/config/scripts/release-blocker-fixes.test.mjs new file mode 100644 index 00000000000..bccd631a266 --- /dev/null +++ b/config/scripts/release-blocker-fixes.test.mjs @@ -0,0 +1,35 @@ +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { describe, expect, it } from 'vitest' +import { parse } from 'yaml' + +const projectDir = resolve(import.meta.dirname, '../..') + +describe('release blocker safeguards', () => { + it('keeps the root package version on the current stable release line', () => { + const packageJson = JSON.parse(readFileSync(resolve(projectDir, 'package.json'), 'utf8')) + const match = /^(\d+)\.(\d+)\.(\d+)(?:-[0-9A-Za-z.-]+)?$/.exec(packageJson.version) + expect(match).not.toBeNull() + const version = match.slice(1, 4).map(Number) + const isAtLeastStable = + version[0] > 1 || + (version[0] === 1 && (version[1] > 4 || (version[1] === 4 && version[2] >= 196))) + expect(isAtLeastStable).toBe(true) + }) + + it('passes the staging confirmation through the step environment', () => { + const workflow = parse( + readFileSync( + resolve(projectDir, '.github/workflows/cloud-prove-relay-asia-staging.yml'), + 'utf8' + ) + ) + const step = workflow.jobs.prove.steps.find( + ({ name }) => name === 'Validate the exact staging proof request' + ) + + expect(step.env.CONFIRMATION).toBe('${{ inputs.confirmation }}') + expect(step.run).toContain('test "${CONFIRMATION}" = PROVE_ASIA_STAGING') + expect(step.run).not.toContain('${{ inputs.confirmation }}') + }) +}) diff --git a/package.json b/package.json index 179ffd1a82a..58f4805d937 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "orca", - "version": "1.4.178-rc.2", + "version": "1.4.197", "description": "Next-gen IDE for parallel agentic development", "homepage": "https://github.com/stablyai/orca", "author": "stablyai", From 4d24fb340b6fd6596fac5ed37d25fe875f4d77bd Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:16:35 -0400 Subject: [PATCH 09/76] fix(mobile): stage-aware relay dial bound so a slow cell is not hung up on (#18518) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A phone returning to foreground on 2026-09-03 logged "replacement session authentication timed out" five dials in a row while the desktop's relay control was live. The cell (production-gce-c27) had taken relay-auth but its assignment/reservation transactions were lock-contended (55P03 retries, 14–16s per accept); the phone's flat 12s migrateTo bound closed the socket 2–4s before the cell finished (cell logged host_data_reservation_already_bound), and because the timeout counted as a director-class failure the phone re-resolved the same cell and waited 12s again before logging — every retry landed in the same contended window. - MobileRelayE2eeLink reports onOpen once relay-auth is on the wire; MobileRelayRpcSession exposes a dial stage (opening → awaiting-hello → handshaking → confirming). - waitForAuthenticated keeps the caller's bound until the socket opens, then re-arms a per-stage budget (30s awaiting-hello, 12s handshaking, 35s confirming) so a reachable, slow cell is not treated as a black hole. - The timeout error carries the stalled stage and shows up in the "relay dial failed" log line; a stall past the open socket no longer triggers the director re-resolve round. Phone-local only: no wire change. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb --- ...e-endpoint-supervisor-stalled-cell.test.ts | 80 +++++++++++ .../mobile-endpoint-supervisor-support.ts | 6 + .../mobile-endpoint-supervisor-test-fakes.ts | 5 + .../transport/mobile-relay-e2ee-link.test.ts | 55 ++++++++ .../src/transport/mobile-relay-e2ee-link.ts | 4 + .../mobile-relay-rpc-session.test.ts | 30 +++++ .../src/transport/mobile-relay-rpc-session.ts | 24 ++-- .../mobile-relay-runtime-failover.test.ts | 5 + mobile/src/transport/relay-dial-stage.ts | 64 +++++++++ ...replacement-session-authentication.test.ts | 127 ++++++++++++++++++ .../replacement-session-authentication.ts | 58 +++++++- .../stable-logical-rpc-client.test.ts | 29 ++++ 12 files changed, 472 insertions(+), 15 deletions(-) create mode 100644 mobile/src/transport/mobile-endpoint-supervisor-stalled-cell.test.ts create mode 100644 mobile/src/transport/relay-dial-stage.ts create mode 100644 mobile/src/transport/replacement-session-authentication.test.ts diff --git a/mobile/src/transport/mobile-endpoint-supervisor-stalled-cell.test.ts b/mobile/src/transport/mobile-endpoint-supervisor-stalled-cell.test.ts new file mode 100644 index 00000000000..be4050cee5b --- /dev/null +++ b/mobile/src/transport/mobile-endpoint-supervisor-stalled-cell.test.ts @@ -0,0 +1,80 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { MobileEndpointSupervisor } from './mobile-endpoint-supervisor' +import { + dependencies, + FakeLogicalClient, + FakeRelaySession, + host, + relay +} from './mobile-endpoint-supervisor-test-fakes' +import { ReplacementAuthenticationTimeoutError } from './replacement-session-authentication' +import type { RpcClient } from './rpc-client' + +vi.mock('react-native', () => ({ Platform: { OS: 'ios' } })) +vi.mock('expo-secure-store', () => ({ WHEN_UNLOCKED_THIS_DEVICE_ONLY: 'when-unlocked' })) +vi.mock('expo-crypto', () => ({ getRandomBytes: (length: number) => new Uint8Array(length) })) + +// The 2026-09-03 incident: five consecutive "authentication timed out" dials against a +// live desktop while the cell's assignment tables were lock-contended. Each logged +// failure was two dials — the timeout counted as a director-class failure, so the phone +// re-resolved the same cell and waited the full bound again. +describe('relay dial against a cell that took the dial and stalled', () => { + beforeEach(() => { + vi.useFakeTimers() + vi.spyOn(console, 'log').mockImplementation(() => {}) + }) + afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() + }) + + function timingOut(logical: FakeLogicalClient, error: Error): void { + logical.migrateTo.mockImplementation(async (session: RpcClient) => { + session.close() + throw error + }) + } + + it('does not re-resolve the director and names the stalled stage', async () => { + const logical = new FakeLogicalClient('disconnected', 'lan') + timingOut(logical, new ReplacementAuthenticationTimeoutError('awaiting-hello', 30_000)) + const openRelay = vi.fn(() => new FakeRelaySession('connecting')) + const resolveRelay = vi.fn(async () => relay) + const onLog = vi.fn() + const supervisor = new MobileEndpointSupervisor( + logical, + host, + dependencies({ openRelay, resolveRelay, onLog }) + ) + + await supervisor.start() + + expect(openRelay).toHaveBeenCalledOnce() + expect(resolveRelay).not.toHaveBeenCalled() + expect(onLog).toHaveBeenCalledWith( + expect.objectContaining({ + code: 'relay-dial-failed', + detail: expect.stringContaining('timed out (awaiting-hello, 30s)') + }) + ) + supervisor.stop() + }) + + it('still re-resolves the director when the cell socket never opened', async () => { + const logical = new FakeLogicalClient('disconnected', 'lan') + timingOut(logical, new ReplacementAuthenticationTimeoutError('opening', 12_000)) + const openRelay = vi.fn(() => new FakeRelaySession('connecting')) + const resolveRelay = vi.fn(async () => relay) + const supervisor = new MobileEndpointSupervisor( + logical, + host, + dependencies({ openRelay, resolveRelay }) + ) + + await supervisor.start() + + expect(resolveRelay).toHaveBeenCalledOnce() + expect(openRelay).toHaveBeenCalledTimes(2) + supervisor.stop() + }) +}) diff --git a/mobile/src/transport/mobile-endpoint-supervisor-support.ts b/mobile/src/transport/mobile-endpoint-supervisor-support.ts index 6ee0a6b42cb..1a2c00f12de 100644 --- a/mobile/src/transport/mobile-endpoint-supervisor-support.ts +++ b/mobile/src/transport/mobile-endpoint-supervisor-support.ts @@ -1,5 +1,6 @@ import { RelayOuterError } from './mobile-relay-e2ee-link' import { MobileE2EEAuthenticationError } from './mobile-e2ee-v2-physical-channel' +import { ReplacementAuthenticationTimeoutError } from './replacement-session-authentication' import type { RelayReconnectController } from './mobile-relay-reconnect-controller' import type { StableLogicalRpcClient } from './stable-logical-rpc-client' import type { HostProfile } from './types' @@ -68,6 +69,11 @@ export async function dialRelayThroughDirectorFallback(args: { } export function isDirectorResolutionFailure(error: Error): boolean { + // Why: a cell that took relay-auth and went quiet is the right cell working slowly; + // re-resolving it just doubles the wait against the same contended window. + if (error instanceof ReplacementAuthenticationTimeoutError) { + return error.stage === null || error.stage === 'opening' + } return ( !(error instanceof MobileE2EEAuthenticationError) && (!(error instanceof RelayOuterError) || [4409, 4503, 1006].includes(error.code)) diff --git a/mobile/src/transport/mobile-endpoint-supervisor-test-fakes.ts b/mobile/src/transport/mobile-endpoint-supervisor-test-fakes.ts index 4023a1a8e39..1dc1473d9db 100644 --- a/mobile/src/transport/mobile-endpoint-supervisor-test-fakes.ts +++ b/mobile/src/transport/mobile-endpoint-supervisor-test-fakes.ts @@ -1,6 +1,7 @@ import { vi } from 'vitest' import type { MobileRelayCredentialBundle } from './mobile-relay-credential-bundle' import type { MobileRelayRpcSession } from './mobile-relay-rpc-session' +import { RelayDialStageTracker, type RelayDialStage } from './relay-dial-stage' import type { MobileEndpointSupervisorDependencies } from './mobile-endpoint-supervisor' import type { RpcClient } from './rpc-client' import type { MobileConnectionPath, StableLogicalRpcClient } from './stable-logical-rpc-client' @@ -51,6 +52,10 @@ export class FakeRelaySession extends FakeSession implements MobileRelayRpcSessi // Why: production-realistic defaults — fictional fake values hid three // live defects in this subsystem (latch, churn, int32 timer overflow). getAttachDeadlineAt = () => Date.now() + 10_000 + readonly dialStage = new RelayDialStageTracker() + getDialStage = () => this.dialStage.getDialStage() + onDialStageChange = (listener: (stage: RelayDialStage) => void) => + this.dialStage.onDialStageChange(listener) getResumeExpiresAt = () => this.resumeExpiry getResumeConfirmation = () => ({ v: 1 as const, diff --git a/mobile/src/transport/mobile-relay-e2ee-link.test.ts b/mobile/src/transport/mobile-relay-e2ee-link.test.ts index aa2d42b13f0..965135511eb 100644 --- a/mobile/src/transport/mobile-relay-e2ee-link.test.ts +++ b/mobile/src/transport/mobile-relay-e2ee-link.test.ts @@ -60,6 +60,61 @@ describe('MobileRelayE2eeLink', () => { expect(socket.close).toHaveBeenCalledOnce() }) + it('reports open only once relay-auth is on the wire', () => { + const socket = new ThrowingSocket() + const onOpen = vi.fn() + const sent: string[] = [] + socket.send.mockImplementation((frame: string) => { + sent.push(frame) + }) + new MobileRelayE2eeLink({ + endpoint: { + cellUrl: 'https://relay-c1.onorca.dev', + relayHostId: 'AbCdEf0123_-xyZ9' + }, + credential: 'credential', + expectedCredentialKind: 'resume', + deviceToken: 'device-token', + desktopPublicKeyB64: 'desktop-key', + onAuthenticated: vi.fn(), + onText: vi.fn(), + onBinary: vi.fn(), + onOpen, + onError: vi.fn(), + createSocket: () => socket as unknown as WebSocket + }) + + expect(onOpen).not.toHaveBeenCalled() + socket.onopen?.() + expect(sent).toHaveLength(1) + expect(JSON.parse(sent[0]!)).toMatchObject({ type: 'relay-auth', mode: 'connect' }) + expect(onOpen).toHaveBeenCalledOnce() + }) + + it('does not report open when the relay-auth write fails', () => { + const socket = new ThrowingSocket() + const onOpen = vi.fn() + new MobileRelayE2eeLink({ + endpoint: { + cellUrl: 'https://relay-c1.onorca.dev', + relayHostId: 'AbCdEf0123_-xyZ9' + }, + credential: 'credential', + expectedCredentialKind: 'resume', + deviceToken: 'device-token', + desktopPublicKeyB64: 'desktop-key', + onAuthenticated: vi.fn(), + onText: vi.fn(), + onBinary: vi.fn(), + onOpen, + onError: vi.fn(), + createSocket: () => socket as unknown as WebSocket + }) + + socket.onopen?.() + expect(onOpen).not.toHaveBeenCalled() + }) + it('keeps a typed close code when transport error precedes close', () => { const socket = new ThrowingSocket() const onError = vi.fn() diff --git a/mobile/src/transport/mobile-relay-e2ee-link.ts b/mobile/src/transport/mobile-relay-e2ee-link.ts index f19417a60f1..7743deb23dd 100644 --- a/mobile/src/transport/mobile-relay-e2ee-link.ts +++ b/mobile/src/transport/mobile-relay-e2ee-link.ts @@ -26,6 +26,8 @@ type MobileRelayE2eeLinkOptions = { onText: (plaintext: string) => void onBinary: (plaintext: Uint8Array) => void onHello?: (hello: Extract) => void + // Fired once relay-auth is on the wire: from here the cell owns the wait. + onOpen?: () => void onError: (error: Error) => void createSocket?: (url: string) => WebSocket } @@ -96,7 +98,9 @@ export class MobileRelayE2eeLink { ) } catch (error) { this.fail(asError(error)) + return } + this.options.onOpen?.() } this.socket.onmessage = (event) => { this.inboundChain = this.inboundChain diff --git a/mobile/src/transport/mobile-relay-rpc-session.test.ts b/mobile/src/transport/mobile-relay-rpc-session.test.ts index d5b547885cf..7f98436c63b 100644 --- a/mobile/src/transport/mobile-relay-rpc-session.test.ts +++ b/mobile/src/transport/mobile-relay-rpc-session.test.ts @@ -11,6 +11,7 @@ const fakes = vi.hoisted(() => ({ endpoint: { cellUrl: string; relayHostId: string } credential: string expectedCredentialKind: string + onOpen(): void onHello(value: unknown): void onAuthenticated(): void onText(value: string): void @@ -123,6 +124,35 @@ describe('mobile relay RPC session', () => { expect(session.getAttachDeadlineAt()).toEqual(expect.any(Number)) }) + // Why: ConnectionState stays 'connecting' until relay-hello, so the migration bound + // needs a separate signal to tell "cell never answered the upgrade" from "cell took + // relay-auth and is still resolving the assignment". + it('reports the dial stage as the link opens, receives hello, and authenticates', async () => { + const session = openSession() + const stages: string[] = [] + session.onDialStageChange((stage) => stages.push(stage)) + expect(session.getDialStage()).toBe('opening') + + fakes.linkOptions!.onOpen() + expect(session.getDialStage()).toBe('awaiting-hello') + expect(session.getState()).toBe('connecting') + fakes.linkOptions!.onHello({ + type: 'relay-hello', + ok: true, + credentialKind: 'resume', + leaseExpiresAt: Date.now() + 10_000, + acceptedCredentialVersion: 3, + acceptedAs: 'current', + resumeExpiresAt: Date.now() + 300_000 + }) + expect(session.getDialStage()).toBe('handshaking') + fakes.linkOptions!.onAuthenticated() + expect(session.getDialStage()).toBe('confirming') + await vi.waitFor(() => expect(fakes.sendText).toHaveBeenCalledOnce()) + expect(stages).toEqual(['awaiting-hello', 'handshaking', 'confirming']) + session.close() + }) + it('rejects a mismatched outer credential version and closes the physical link', () => { const session = openSession() fakes.linkOptions!.onHello({ diff --git a/mobile/src/transport/mobile-relay-rpc-session.ts b/mobile/src/transport/mobile-relay-rpc-session.ts index f242fce07ba..40b93927139 100644 --- a/mobile/src/transport/mobile-relay-rpc-session.ts +++ b/mobile/src/transport/mobile-relay-rpc-session.ts @@ -9,6 +9,7 @@ import { MobileE2EEAuthenticationError } from './mobile-e2ee-v2-physical-channel import { markRpcDeliveryUnknown } from './rpc-delivery-ambiguity' import { openRpcRequestBudget, resolvePostConnectRequestTimeout } from './rpc-request-budget' import { isRpcResponse } from './rpc-response-shape' +import { RelayDialStageTracker, type RelayDialStageSource } from './relay-dial-stage' import { RpcSessionLivenessWatchdog } from './rpc-session-liveness-watchdog' import type { RpcClient } from './rpc-client' import type { ConnectionLogSink, ConnectionState, RpcResponse } from './types' @@ -24,14 +25,15 @@ type PendingRequest = { timer: ReturnType } -export type MobileRelayRpcSession = RpcClient & { - // The cell's attach-reservation deadline (~10s). Diagnostics only — never - // schedule anything from it; rotation keys off getResumeExpiresAt(). - getAttachDeadlineAt(): number | null - getResumeExpiresAt(): number | null - getResumeConfirmation(): DeviceResumeConfirmed | null - getFailure(): Error | null -} +export type MobileRelayRpcSession = RpcClient & + RelayDialStageSource & { + // The cell's attach-reservation deadline (~10s). Diagnostics only — never + // schedule anything from it; rotation keys off getResumeExpiresAt(). + getAttachDeadlineAt(): number | null + getResumeExpiresAt(): number | null + getResumeConfirmation(): DeviceResumeConfirmed | null + getFailure(): Error | null + } export function connectMobileRelayRpcSession(args: { relay: MobileRelayEndpoint @@ -58,6 +60,7 @@ export function connectMobileRelayRpcSession(args: { let logSequence = 0 const logSessionId = `${Date.now().toString(36)}-${(++relayRpcSessionSequence).toString(36)}` const livenessIdentity = {} + const dialStage = new RelayDialStageTracker() const streams = new MobileRelayRpcStreams({ nextId, sendFrame, @@ -71,6 +74,7 @@ export function connectMobileRelayRpcSession(args: { deviceToken: args.deviceToken, desktopPublicKeyB64: args.desktopPublicKeyB64, createSocket: args.createSocket, + onOpen: () => dialStage.advance('awaiting-hello'), onHello: (hello) => { if ( hello.credentialKind !== 'resume' || @@ -81,6 +85,7 @@ export function connectMobileRelayRpcSession(args: { } attachDeadlineAt = hello.leaseExpiresAt resumeExpiresAt = hello.resumeExpiresAt + dialStage.advance('handshaking') publishState('handshaking') }, onAuthenticated: () => void confirmResume(), @@ -136,6 +141,8 @@ export function connectMobileRelayRpcSession(args: { streams.clear() publishState('disconnected') }, + getDialStage: () => dialStage.getDialStage(), + onDialStageChange: (listener) => dialStage.onDialStageChange(listener), getAttachDeadlineAt: () => attachDeadlineAt, getResumeExpiresAt: () => resumeExpiresAt, getResumeConfirmation: () => resumeConfirmation, @@ -165,6 +172,7 @@ export function connectMobileRelayRpcSession(args: { return client async function confirmResume(): Promise { + dialStage.advance('confirming') try { const response = await sendRpc( 'pairing.getEndpoints', diff --git a/mobile/src/transport/mobile-relay-runtime-failover.test.ts b/mobile/src/transport/mobile-relay-runtime-failover.test.ts index 795f2618dfb..01f4d45feb0 100644 --- a/mobile/src/transport/mobile-relay-runtime-failover.test.ts +++ b/mobile/src/transport/mobile-relay-runtime-failover.test.ts @@ -9,6 +9,7 @@ import { MobileE2EEAuthenticationError } from './mobile-e2ee-v2-physical-channel import { RelayOuterError } from './mobile-relay-e2ee-link' import type { MobileRelayCredentialBundle } from './mobile-relay-credential-bundle' import type { MobileRelayRpcSession } from './mobile-relay-rpc-session' +import { RelayDialStageTracker, type RelayDialStage } from './relay-dial-stage' import { MobileEndpointSupervisor, type MobileEndpointSupervisorDependencies @@ -81,6 +82,10 @@ class FakeRelaySession extends FakeSession implements MobileRelayRpcSession { // Why: production-realistic constants — fictional fake values hid three // live defects in this subsystem (latch, churn, int32 timer overflow). getAttachDeadlineAt = () => Date.now() + 10_000 + readonly dialStage = new RelayDialStageTracker() + getDialStage = () => this.dialStage.getDialStage() + onDialStageChange = (listener: (stage: RelayDialStage) => void) => + this.dialStage.onDialStageChange(listener) getResumeExpiresAt = () => Date.now() + 30 * 24 * 3_600_000 getResumeConfirmation = () => null getFailure = () => this.failure diff --git a/mobile/src/transport/relay-dial-stage.ts b/mobile/src/transport/relay-dial-stage.ts new file mode 100644 index 00000000000..c4a743f84f4 --- /dev/null +++ b/mobile/src/transport/relay-dial-stage.ts @@ -0,0 +1,64 @@ +// Where a relay dial is waiting, so a bound can tell "the cell never answered the +// upgrade" from "the cell took the dial and is slow" — the two look identical from +// ConnectionState, which stays 'connecting' until relay-hello arrives. +export type RelayDialStage = + // WebSocket upgrade not yet open. + | 'opening' + // Socket open and relay-auth sent; the cell is resolving/reserving and asking the + // desktop to attach before it can answer with relay-hello. + | 'awaiting-hello' + // relay-hello accepted; E2EE handshake with the desktop in flight. + | 'handshaking' + // E2EE authenticated; waiting on the desktop's resume confirmation. + | 'confirming' + +export type RelayDialStageSource = { + getDialStage(): RelayDialStage + onDialStageChange(listener: (stage: RelayDialStage) => void): () => void +} + +export function relayDialStageSource(session: object): RelayDialStageSource | null { + const candidate = session as Partial + return typeof candidate.getDialStage === 'function' && + typeof candidate.onDialStageChange === 'function' + ? (candidate as RelayDialStageSource) + : null +} + +export class RelayDialStageTracker implements RelayDialStageSource { + private stage: RelayDialStage = 'opening' + private readonly listeners = new Set<(stage: RelayDialStage) => void>() + + getDialStage(): RelayDialStage { + return this.stage + } + + onDialStageChange(listener: (stage: RelayDialStage) => void): () => void { + this.listeners.add(listener) + return () => this.listeners.delete(listener) + } + + advance(stage: RelayDialStage): void { + if (this.stage === stage) { + return + } + this.stage = stage + for (const listener of this.listeners) { + listener(stage) + } + } +} + +// Budget per stage once the cell holds the dial. awaiting-hello covers the cell's +// assignment/reservation transactions (observed 14–16s under lock contention) plus its +// 10s host-attach deadline; handshaking is two E2EE round trips; confirming is bounded +// by the session's own 30s resume-confirmation request, with slack so that error wins. +const RELAY_DIAL_STAGE_BUDGET_MS: Record, number> = { + 'awaiting-hello': 30_000, + handshaking: 12_000, + confirming: 35_000 +} + +export function relayDialStageBudgetMs(stage: Exclude): number { + return RELAY_DIAL_STAGE_BUDGET_MS[stage] +} diff --git a/mobile/src/transport/replacement-session-authentication.test.ts b/mobile/src/transport/replacement-session-authentication.test.ts new file mode 100644 index 00000000000..096721fa02d --- /dev/null +++ b/mobile/src/transport/replacement-session-authentication.test.ts @@ -0,0 +1,127 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { RelayDialStageTracker } from './relay-dial-stage' +import { + ReplacementAuthenticationTimeoutError, + waitForAuthenticated +} from './replacement-session-authentication' +import type { RpcClient } from './rpc-client' +import type { ConnectionState } from './types' + +class FakeSession implements RpcClient { + readonly sendRequest = vi.fn() + readonly subscribe = vi.fn(() => () => {}) + readonly updateTerminalSubscriptionViewport = vi.fn() + readonly notifyForeground = vi.fn() + readonly close = vi.fn() + private readonly listeners = new Set<(state: ConnectionState) => void>() + constructor(private state: ConnectionState = 'connecting') {} + getState = () => this.state + getReconnectAttempt = () => 0 + getLastConnectedAt = () => null + onStateChange = (listener: (state: ConnectionState) => void) => { + this.listeners.add(listener) + return () => this.listeners.delete(listener) + } + setState(state: ConnectionState): void { + this.state = state + for (const listener of this.listeners) { + listener(state) + } + } +} + +class FakeRelaySession extends FakeSession { + readonly dialStage = new RelayDialStageTracker() + getDialStage = () => this.dialStage.getDialStage() + onDialStageChange = this.dialStage.onDialStageChange.bind(this.dialStage) +} + +// Why: fake timers are active, so "still pending" is decided on the microtask queue. +async function settle( + promise: Promise +): Promise<{ status: 'pending' | 'settled'; error?: Error }> { + let outcome: { status: 'pending' | 'settled'; error?: Error } = { status: 'pending' } + void promise.then( + () => (outcome = { status: 'settled' }), + (error: Error) => (outcome = { status: 'settled', error }) + ) + await Promise.resolve() + await Promise.resolve() + return outcome +} + +describe('waitForAuthenticated', () => { + beforeEach(() => vi.useFakeTimers()) + afterEach(() => vi.useRealTimers()) + + it('keeps the flat bound for a session that reports no dial stages', async () => { + const session = new FakeSession() + const waiting = waitForAuthenticated(session, 12_000) + waiting.catch(() => {}) + await vi.advanceTimersByTimeAsync(11_999) + expect((await settle(waiting)).status).toBe('pending') + await vi.advanceTimersByTimeAsync(1) + const outcome = await settle(waiting) + expect(outcome.error).toBeInstanceOf(ReplacementAuthenticationTimeoutError) + expect(outcome.error?.message).toBe('replacement session authentication timed out') + }) + + // The 2026-09-03 incident: the cell accepted relay-auth and spent 14–16s in its + // lock-contended assignment transactions. The flat 12s bound hung up 2–4s before + // the cell finished, five dials in a row, while the desktop was live the whole time. + it('re-arms the bound per stage once the cell holds the dial', async () => { + const session = new FakeRelaySession() + const waiting = waitForAuthenticated(session, 12_000) + waiting.catch(() => {}) + await vi.advanceTimersByTimeAsync(11_000) + session.dialStage.advance('awaiting-hello') + await vi.advanceTimersByTimeAsync(5_000) + expect((await settle(waiting)).status).toBe('pending') + session.dialStage.advance('handshaking') + session.setState('handshaking') + await vi.advanceTimersByTimeAsync(11_000) + expect((await settle(waiting)).status).toBe('pending') + session.dialStage.advance('confirming') + await vi.advanceTimersByTimeAsync(20_000) + session.setState('connected') + await expect(waiting).resolves.toBeUndefined() + }) + + it('bounds a cell that took the dial and never answers, naming the stage', async () => { + const session = new FakeRelaySession() + const waiting = waitForAuthenticated(session, 12_000) + waiting.catch(() => {}) + await vi.advanceTimersByTimeAsync(2_000) + session.dialStage.advance('awaiting-hello') + await vi.advanceTimersByTimeAsync(29_999) + expect((await settle(waiting)).status).toBe('pending') + await vi.advanceTimersByTimeAsync(1) + const outcome = await settle(waiting) + expect(outcome.error).toBeInstanceOf(ReplacementAuthenticationTimeoutError) + expect((outcome.error as ReplacementAuthenticationTimeoutError).stage).toBe('awaiting-hello') + expect(outcome.error?.message).toBe( + 'replacement session authentication timed out (awaiting-hello, 30s)' + ) + }) + + it('keeps the caller bound while the socket never opens', async () => { + const session = new FakeRelaySession() + const waiting = waitForAuthenticated(session, 12_000) + waiting.catch(() => {}) + await vi.advanceTimersByTimeAsync(12_000) + const outcome = await settle(waiting) + expect((outcome.error as ReplacementAuthenticationTimeoutError).stage).toBe('opening') + expect(outcome.error?.message).toBe( + 'replacement session authentication timed out (opening, 12s)' + ) + }) + + it('ignores stage advances after the wait has settled', async () => { + const session = new FakeRelaySession() + const waiting = waitForAuthenticated(session, 12_000) + session.setState('disconnected') + await expect(waiting).rejects.toThrow('replacement session disconnected') + session.dialStage.advance('awaiting-hello') + expect(vi.getTimerCount()).toBe(0) + }) +}) diff --git a/mobile/src/transport/replacement-session-authentication.ts b/mobile/src/transport/replacement-session-authentication.ts index 0ba54a9d611..5ef9a5f1f46 100644 --- a/mobile/src/transport/replacement-session-authentication.ts +++ b/mobile/src/transport/replacement-session-authentication.ts @@ -1,21 +1,43 @@ import type { RpcClient } from './rpc-client' +import { + relayDialStageBudgetMs, + relayDialStageSource, + type RelayDialStage +} from './relay-dial-stage' + +export class ReplacementAuthenticationTimeoutError extends Error { + constructor( + readonly stage: RelayDialStage | null, + budgetMs: number + ) { + super( + stage + ? `replacement session authentication timed out (${stage}, ${Math.round(budgetMs / 1000)}s)` + : 'replacement session authentication timed out' + ) + this.name = 'ReplacementAuthenticationTimeoutError' + } +} // Why: a migration must not cut over to a session that has only opened a socket — the -// replacement has to reach 'connected' (E2EE authenticated) first, and a relay dial can -// sit in handshaking for seconds, so the wait is bounded by the caller's timeout. +// replacement has to reach 'connected' (E2EE authenticated) first. The caller's bound +// covers reaching an open socket; a relay session that reports dial stages re-arms a +// per-stage budget on every advance, so a cell that accepted the dial and is working +// slowly (lock-contended assignment tables) is not hung up on like a black hole — the +// retry would land in the same window and burn a director round on the way. export function waitForAuthenticated(session: RpcClient, timeoutMs: number): Promise { if (session.getState() === 'connected') { return Promise.resolve() } + const stages = relayDialStageSource(session) return new Promise((resolve, reject) => { let settled = false let unsubscribe: (() => void) | null = null + let unsubscribeStage: (() => void) | null = null + let timer: ReturnType | null = null // Why: armed before subscribing — a synchronous notification during registration // must find a timer to clear, or a settled wait leaves it running for 12s. - const timer = setTimeout(() => { - finish() - reject(new Error('replacement session authentication timed out')) - }, timeoutMs) + arm(stages?.getDialStage() ?? null) unsubscribe = session.onStateChange((state) => { if (state === 'connected') { finish() @@ -29,6 +51,23 @@ export function waitForAuthenticated(session: RpcClient, timeoutMs: number): Pro // Why: the notification fired inside onStateChange, before we held the handle. unsubscribe() unsubscribe = null + } else if (stages) { + unsubscribeStage = stages.onDialStageChange((stage) => arm(stage)) + } + + function arm(stage: RelayDialStage | null): void { + if (settled) { + return + } + if (timer) { + clearTimeout(timer) + } + const budgetMs = + stage === null || stage === 'opening' ? timeoutMs : relayDialStageBudgetMs(stage) + timer = setTimeout(() => { + finish() + reject(new ReplacementAuthenticationTimeoutError(stage, budgetMs)) + }, budgetMs) } function finish(): void { @@ -36,9 +75,14 @@ export function waitForAuthenticated(session: RpcClient, timeoutMs: number): Pro return } settled = true - clearTimeout(timer) + if (timer) { + clearTimeout(timer) + timer = null + } unsubscribe?.() unsubscribe = null + unsubscribeStage?.() + unsubscribeStage = null } }) } diff --git a/mobile/src/transport/stable-logical-rpc-client.test.ts b/mobile/src/transport/stable-logical-rpc-client.test.ts index 0ba7a1f2ea7..faa236a88ca 100644 --- a/mobile/src/transport/stable-logical-rpc-client.test.ts +++ b/mobile/src/transport/stable-logical-rpc-client.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from 'vitest' +import { RelayDialStageTracker, type RelayDialStage } from './relay-dial-stage' import type { ConnectionState, RpcResponse } from './types' import type { RpcClient } from './rpc-client' import { isRpcDeliveryUnknown, markRpcDeliveryUnknown } from './rpc-delivery-ambiguity' @@ -399,6 +400,34 @@ describe('stable logical RPC client', () => { expect(client.getPendingPath()).toBeNull() }) + // Pins the shipping wiring: migrateTo's bound honors the replacement's dial stages. + it('outlives the flat bound when the relay cell holds the dial', async () => { + vi.useFakeTimers() + try { + const oldSession = new FakeSession('connected') + const replacement = Object.assign(new FakeSession('connecting'), { + dialStage: new RelayDialStageTracker(), + getDialStage(): RelayDialStage { + return this.dialStage.getDialStage() + }, + onDialStageChange(listener: (stage: RelayDialStage) => void) { + return this.dialStage.onDialStageChange(listener) + } + }) + const client = createStableLogicalRpcClient(oldSession, 'lan') + const migrating = client.migrateTo(replacement, 'relay', 12_000) + await vi.advanceTimersByTimeAsync(1_000) + replacement.dialStage.advance('awaiting-hello') + await vi.advanceTimersByTimeAsync(20_000) + expect(replacement.close).not.toHaveBeenCalled() + replacement.setState('connected') + await migrating + expect(client.getActivePath()).toBe('relay') + } finally { + vi.useRealTimers() + } + }) + it('closes a replacement that fails authentication and preserves the active session', async () => { const oldSession = new FakeSession('connected') const replacement = new FakeSession('connecting') From d66386bc8278637a62c59502da459ecbdf6c93c6 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:27:57 -0400 Subject: [PATCH 10/76] fix(cloud): bound and yield the relay's global cell-inventory lock (#18521) Mirrors stablyai/orca-cloud#471 (squash c3354e8), byte-identical under cloud/. The relay's cell-inventory lock (SELECT ... FROM relay_cells FOR UPDATE over all 23 rows) is one global critical section shared by the assignment hot path and every director sweep; with the pool's 1s lock_timeout a blocked waiter held a pooled client for a full second, producing ~690 55P03 retries per 5 minutes in production. Request paths now bound the wait at 500ms with a SET LOCAL that is restored to the pool default before the next statement; director-only sweeps take the lock NOWAIT and skip the tick; sweep timers are jittered; hold time is exported as additive runtime-metrics fields so the bound can be tuned. --- .../relay-ops/src/incident-monitor.test.ts | 13 + cloud/apps/relay-ops/src/incident-monitor.ts | 14 + cloud/apps/relay/src/assignment-store.ts | 196 +++++-- .../relay/src/cell-inventory-hold-samples.ts | 46 ++ .../src/cell-inventory-lock-census.test.ts | 164 ++++++ .../cell-inventory-lock-contention.test.ts | 541 ++++++++++++++++++ cloud/apps/relay/src/database.ts | 103 +++- cloud/apps/relay/src/index.ts | 7 +- .../relay/src/regional-rehome-store.test.ts | 337 ++++++++++- .../apps/relay/src/regional-rehome-worker.ts | 7 +- cloud/apps/relay/src/relay-observability.ts | 5 +- .../relay/src/relay-sweep-schedule.test.ts | 55 ++ cloud/apps/relay/src/relay-sweep-schedule.ts | 13 + 13 files changed, 1439 insertions(+), 62 deletions(-) create mode 100644 cloud/apps/relay/src/cell-inventory-hold-samples.ts create mode 100644 cloud/apps/relay/src/cell-inventory-lock-census.test.ts create mode 100644 cloud/apps/relay/src/cell-inventory-lock-contention.test.ts create mode 100644 cloud/apps/relay/src/relay-sweep-schedule.test.ts create mode 100644 cloud/apps/relay/src/relay-sweep-schedule.ts diff --git a/cloud/apps/relay-ops/src/incident-monitor.test.ts b/cloud/apps/relay-ops/src/incident-monitor.test.ts index 51153bb63e1..ea5ad55b645 100644 --- a/cloud/apps/relay-ops/src/incident-monitor.test.ts +++ b/cloud/apps/relay-ops/src/incident-monitor.test.ts @@ -123,6 +123,19 @@ describe('incident monitor evaluator', () => { }) }) + // Why: sweeps no longer reach the retry wrapper, so any exhaustion left in this + // counter is a request path that terminally failed. It must still freeze. + it('freezes on a single exhausted request-path transaction', () => { + const sample = healthySample() + sample.sources['relay-logs']!.signals['relay.postgres_retry_exhausted'] = signal(1) + expect(evaluateIncidentSample(sample, startedAt)).toMatchObject({ + status: 'freeze', + failures: [ + expect.objectContaining({ signal: 'relay.postgres_retry_exhausted', threshold: 0 }) + ] + }) + }) + it('allows missing auth readiness and legacy existing-only connections', () => { const sample = healthySample() const legacySelector = { diff --git a/cloud/apps/relay-ops/src/incident-monitor.ts b/cloud/apps/relay-ops/src/incident-monitor.ts index 6073e351511..a1936f5df6c 100644 --- a/cloud/apps/relay-ops/src/incident-monitor.ts +++ b/cloud/apps/relay-ops/src/incident-monitor.ts @@ -38,6 +38,20 @@ export const INCIDENT_MONITOR_THRESHOLDS = { // margin; relayPostgresRetryExhausted below stays at zero tolerance, so any // transaction that terminally fails still freezes the gate. relayPostgresRetries: 300, + // Why: this bar stays at zero. Cell-inventory contention reaches the retry + // wrapper from exactly two kinds of caller, and neither is a sweep tick that + // can shrug the failure off: + // - request paths, which take a wait bounded at CELL_INVENTORY_LOCK_TIMEOUT_MS + // (assignment, control activation, activity, admin drain/evacuate/supersede); + // - sweep-reachable code that a request also enters, which keeps the pool + // lock_timeout so it cannot fail faster than before this change: the + // completeEvacuation site that waits, reconcileReservationAccounting, and + // placement re-entered from evacuateDeadCells. + // Sweep-only sites take the inventory NOWAIT, so their contention becomes + // database_lock_unavailable, which is not a retryable abort and never reaches + // this counter. The relay's cell-inventory-lock census test holds that split. + // Splitting the metric by the phase label PR #423 put on the log payload would + // need a labelled log-based metric, which this signal's counter does not carry. relayPostgresRetryExhausted: 0, // Why: public admission is a per-instance semaphore, so fleet assignment capacity is // concurrency x instances. A floor of 1 let the 2026-08-04 collapse from five instances diff --git a/cloud/apps/relay/src/assignment-store.ts b/cloud/apps/relay/src/assignment-store.ts index 0b2b1ef72a9..3571b57cd22 100644 --- a/cloud/apps/relay/src/assignment-store.ts +++ b/cloud/apps/relay/src/assignment-store.ts @@ -31,7 +31,12 @@ import { } from './assignment-connection-headroom-query.js' import { AssignmentIdentityQueue } from './assignment-identity-queue.js' import type { RelayCellConfig } from './config.js' -import type { RelayDatabase, RelayTransactionOptions, SqlRow } from './database.js' +import type { + RelayDatabase, + RelayLockOptions, + RelayTransactionOptions, + SqlRow +} from './database.js' import type { RegionalRehomeSafetySnapshot } from './relay-observability.js' import { combineRegionalRehomeSafety, @@ -316,6 +321,25 @@ const ACTIVITY_REQUEST_UNITS: Record = { } const ASSIGNMENT_LOCK_RETRY_DEADLINE_MS = 15_000 +// Why: one global FOR UPDATE over a 23-row table serialises every director and +// cell. At the 1s pool lock_timeout each blocked waiter also holds a pooled +// client for a full second, so the queue converts contention into pool +// exhaustion. The lock is held to COMMIT and the assignment path runs many +// statements after taking it, and no hold-time telemetry existed before this +// change, so 500ms is a first value to tune once cellInventoryHoldMsMax lands. +export const CELL_INVENTORY_LOCK_TIMEOUT_MS = 500 + +// The same inventory lock is taken by live requests and by background sweeps, +// and the right failure mode differs per caller. +export type CellInventoryLockMode = + // Bound the wait so a blocked request stops occupying a pooled client. + | 'request' + // Never queue: the caller handles database_lock_unavailable and moves on. + | 'nowait' + // A sweep can enter here, so keep the pool default. Failing sooner would turn + // ordinary contention into a 55P03 the retry wrapper reports as terminal, and + // one terminal failure freezes the incident gate. + | 'pool-default' // Why: stranded detection (issue #225) needs a grant old enough that a real // attach would have registered (the 90s activity lease covers dial + // activation), yet recent enough to prove an active retry loop rather than @@ -536,29 +560,35 @@ export class RelayAssignmentStore { async assign( identity: AssignmentIdentity, preferredRegion?: RelayRegion, - placementRegion: RelayRegion = preferredRegion ?? RELAY_DEFAULT_REGION + placementRegion: RelayRegion = preferredRegion ?? RELAY_DEFAULT_REGION, + // evacuateDeadCells re-enters placement from a sweep; it must not take the + // bounded wait, whose 55P03 would surface as a terminal sweep failure. + lockMode: CellInventoryLockMode = 'request' ): Promise { - const sticky = await this.assignStickyWithLockRetry(identity, preferredRegion) + const sticky = await this.assignStickyWithLockRetry(identity, lockMode, preferredRegion) if (sticky) return sticky // Only placement needs the global inventory critical section; queueing those // attempts locally avoids turning true placement bursts into NOWAIT storms. return await this.serializeAssignment( - async () => await this.assignWithLockRetry(identity, preferredRegion, placementRegion) + async () => + await this.assignWithLockRetry(identity, lockMode, preferredRegion, placementRegion) ) } private async assignStickyWithLockRetry( identity: AssignmentIdentity, + lockMode: CellInventoryLockMode, preferredRegion?: RelayRegion ): Promise { return await this.withAssignmentLockRetry( async (inventoryFirst) => - await this.assignStickyOnce(identity, inventoryFirst, preferredRegion) + await this.assignStickyOnce(identity, inventoryFirst, lockMode, preferredRegion) ) } private async assignWithLockRetry( identity: AssignmentIdentity, + lockMode: CellInventoryLockMode, preferredRegion?: RelayRegion, placementRegion: RelayRegion = preferredRegion ?? RELAY_DEFAULT_REGION ): Promise { @@ -566,7 +596,13 @@ export class RelayAssignmentStore { let inventoryScope: AssignmentInventoryScope = 'none' while (true) { try { - return await this.assignOnce(identity, inventoryScope, preferredRegion, placementRegion) + return await this.assignOnce( + identity, + inventoryScope, + lockMode, + preferredRegion, + placementRegion + ) } catch (error) { if (error instanceof AssignmentInventoryScopeChanged) { inventoryScope = 'all' @@ -604,12 +640,13 @@ export class RelayAssignmentStore { private async assignStickyOnce( identity: AssignmentIdentity, inventoryFirst: boolean, + lockMode: CellInventoryLockMode, preferredRegion?: RelayRegion ): Promise { const now = this.now() return await this.database.transaction(async (transaction) => { const lockedCells = inventoryFirst - ? await this.lockCellInventory(transaction) + ? await this.lockCellInventory(transaction, lockMode) : undefined const existing = await this.assignmentRow(transaction, identity, inventoryFirst) if (!existing) return null @@ -751,6 +788,7 @@ export class RelayAssignmentStore { private async assignOnce( identity: AssignmentIdentity, inventoryScope: AssignmentInventoryScope, + lockMode: CellInventoryLockMode, preferredRegion?: RelayRegion, placementRegion: RelayRegion = preferredRegion ?? RELAY_DEFAULT_REGION ): Promise { @@ -760,9 +798,9 @@ export class RelayAssignmentStore { return await this.database.transaction(async (transaction) => { let lockedCells = inventoryScope === 'all' - ? await this.lockCellInventory(transaction) + ? await this.lockCellInventory(transaction, lockMode) : inventoryScope === 'general' - ? await this.lockGeneralCellInventory(transaction) + ? await this.lockGeneralCellInventory(transaction, lockMode) : undefined const existing = await this.assignmentRow( transaction, @@ -779,7 +817,7 @@ export class RelayAssignmentStore { let connectionHeadroomReassignment = false let strandedReassignment = false if (existing && !mayNormallyReassign(activity(existing), now)) { - lockedCells ??= await this.lockCellInventory(transaction, true) + lockedCells ??= await this.lockCellInventory(transaction, 'nowait') const admission = await cellAdmissionStates(transaction) const currentRow = lockedCells.find( (row) => text(row, 'cell_id') === text(existing, 'cell_id') @@ -859,8 +897,8 @@ export class RelayAssignmentStore { } lockedCells ??= existing - ? await this.lockCellInventory(transaction, true) - : await this.lockGeneralCellInventory(transaction, true) + ? await this.lockCellInventory(transaction, 'nowait') + : await this.lockGeneralCellInventory(transaction, 'nowait') const target = await this.leastLoadedCell( transaction, lockedCells, @@ -2114,7 +2152,7 @@ export class RelayAssignmentStore { ORDER BY migration.user_id, migration.relay_host_id`, [input.cellId] ) - const cells = await this.lockCellInventory(transaction) + const cells = await this.lockCellInventory(transaction, 'request') for (const migrationRow of migrations) { const identity = { userId: text(migrationRow, 'user_id'), @@ -2608,10 +2646,12 @@ export class RelayAssignmentStore { let moved = 0 for (const row of rows) { try { - const assignment = await this.assign({ - userId: text(row, 'user_id'), - relayHostId: text(row, 'relay_host_id') - }) + const assignment = await this.assign( + { userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') }, + undefined, + undefined, + 'pool-default' + ) if (assignment.cellId !== text(row, 'cell_id')) moved++ } catch (error) { if (!(error instanceof Error && error.message === 'relay_capacity_exhausted')) throw error @@ -3162,7 +3202,7 @@ export class RelayAssignmentStore { ) const requestDelta = ACTIVITY_REQUEST_UNITS[kind] * (after - before) if (requestDelta !== 0) { - await this.lockCellInventory(transaction) + await this.lockCellInventory(transaction, 'request') await this.adjustCellReservation(transaction, text(row, 'cell_id'), requestDelta) } }) @@ -3223,7 +3263,7 @@ export class RelayAssignmentStore { } const units = ACTIVITY_REQUEST_UNITS[input.kind] if (existing) { - await this.lockCellInventory(transaction) + await this.lockCellInventory(transaction, 'request') await this.removeActivityLease(transaction, identity, existing, now) await this.adjustCellReservation(transaction, input.cellId, units) } @@ -3540,7 +3580,7 @@ export class RelayAssignmentStore { ) await this.touchAssignment(transaction, identity, expiresAt, now) } else { - await this.lockCellInventory(transaction) + await this.lockCellInventory(transaction, 'request') await this.adjustCellReservation(transaction, input.cellId, 1) await this.adjustActivityCount(transaction, identity, 'control', 1, expiresAt, now) await transaction.query( @@ -3614,7 +3654,7 @@ export class RelayAssignmentStore { } if (sourceCellId === targetCellId) throw new Error('target_matches_source') await this.lockAssignmentActivities(transaction, identity) - const cells = await this.lockCellInventory(transaction) + const cells = await this.lockCellInventory(transaction, 'request') const target = cells.find((row) => text(row, 'cell_id') === targetCellId) if (!target || integer(target, 'enabled') !== 1) throw new Error('target_cell_unavailable') if (!(await this.cellIsLive(transaction, targetCellId, now))) { @@ -3822,7 +3862,7 @@ export class RelayAssignmentStore { let lockedCells: SqlRow[] | undefined if (inventoryFirst) { try { - lockedCells = await this.lockCellInventory(transaction) + lockedCells = await this.lockCellInventory(transaction, 'request') } catch (error) { if (isDatabaseLockTimeout(error)) { throw new Error('database_lock_unavailable') @@ -3863,7 +3903,7 @@ export class RelayAssignmentStore { if (activityUnitsForCell(activityLeases, input.sourceCellId) > 0) { throw new Error('migration_source_still_active') } - const cells = lockedCells ?? (await this.lockCellInventory(transaction, true)) + const cells = lockedCells ?? (await this.lockCellInventory(transaction, 'nowait')) const source = cells.find((cell) => text(cell, 'cell_id') === input.sourceCellId) const target = cells.find((cell) => text(cell, 'cell_id') === input.targetCellId) if (!source || integer(source, 'enabled') !== 0) { @@ -3967,7 +4007,7 @@ export class RelayAssignmentStore { const now = this.now() return await this.database.transaction(async (transaction) => { const lockedCells = inventoryFirst - ? await this.lockCellInventory(transaction) + ? await this.lockCellInventory(transaction, 'request') : undefined const assignment = await this.assignmentRow(transaction, identity, inventoryFirst) const existing = ( @@ -4041,7 +4081,7 @@ export class RelayAssignmentStore { ) { throw new Error('migration_activity_topology_mismatch') } - const cells = lockedCells ?? (await this.lockCellInventory(transaction, true)) + const cells = lockedCells ?? (await this.lockCellInventory(transaction, 'nowait')) const source = cells.find((cell) => text(cell, 'cell_id') === input.sourceCellId) const currentTarget = cells.find( (cell) => text(cell, 'cell_id') === input.currentTargetCellId @@ -4458,7 +4498,7 @@ export class RelayAssignmentStore { throw new Error('migration_activity_topology_mismatch') } } - if (obsoleteLeases.length > 0) await this.lockCellInventory(transaction) + if (obsoleteLeases.length > 0) await this.lockCellInventory(transaction, 'request') for (const lease of obsoleteLeases) { await this.removeActivityLease(transaction, identity, lease, now) } @@ -4634,7 +4674,7 @@ export class RelayAssignmentStore { ) { throw new Error('migration_activity_lease_shape_mismatch') } - await this.lockCellInventory(transaction) + await this.lockCellInventory(transaction, 'request') await this.adjustCellReservation( transaction, input.currentTargetCellId, @@ -4723,7 +4763,7 @@ export class RelayAssignmentStore { if (this.requireLiveCells) { let cells: SqlRow[] try { - cells = await this.lockCellInventory(transaction, true) + cells = await this.lockCellInventory(transaction, 'nowait') } catch (error) { if (isDatabaseLockUnavailable(error)) { // Mixed-version workers may still hold a cell-first lock; defer @@ -4779,7 +4819,7 @@ export class RelayAssignmentStore { ) if (!targetIsActive) throw new Error('migration_target_not_active') const lease = activityLeaseById(activityLeases, migrationActivityId(assignmentEpoch)) - if (lease && !cellsLocked) await this.lockCellInventory(transaction) + if (lease && !cellsLocked) await this.lockCellInventory(transaction, 'pool-default') if (lease) await this.removeActivityLease(transaction, identity, lease, now) await transaction.query( `UPDATE relay_assignment_migrations SET completed_at = ?, updated_at = ? @@ -4801,7 +4841,7 @@ export class RelayAssignmentStore { const sourceCellId = text(assignment, 'cell_id') if (sourceCellId === targetCellId) throw new Error('target_matches_source') await this.lockAssignmentActivities(transaction, identity) - const cells = await this.lockCellInventory(transaction) + const cells = await this.lockCellInventory(transaction, 'request') const admission = await cellAdmissionStates(transaction) const targetRow = cells.find( (row) => @@ -5051,7 +5091,13 @@ export class RelayAssignmentStore { } this.pendingRegionalRehomeDisableLog = null const candidateSkips: RegionalRehomeCandidateSkip[] = [] + // A Postgres transaction is unusable after a NOWAIT abort, so a contended + // tick abandons the candidate it stopped on plus every one behind it. + let candidatesTotal = 0 + let candidatesFinished = 0 const claimResult = await this.database.transaction(async (transaction) => { + candidatesTotal = 0 + candidatesFinished = 0 candidateSkips.length = 0 await this.initializeRegionalRehomeControl(transaction, now) const control = ( @@ -5122,6 +5168,7 @@ export class RelayAssignmentStore { ) )[0] if (retry) { + candidatesTotal = 1 const fleetSafety = await this.lockedRegionalRehomeFleetSafety(transaction, now) if ( !(await this.regionalRehomeSafetyAllowsClaim( @@ -5190,6 +5237,7 @@ export class RelayAssignmentStore { ) )[0] if (redrain) { + candidatesTotal = 1 const fleetSafety = await this.lockedRegionalRehomeFleetSafety(transaction, now) if ( !(await this.regionalRehomeSafetyAllowsClaim( @@ -5253,6 +5301,7 @@ export class RelayAssignmentStore { LIMIT 10`, [preferenceCutoff, now - this.heartbeatTtlMs, now] ) + candidatesTotal = candidates.length for (const candidate of candidates) { const claimed = await this.startRegionalRehomeCandidate(transaction, { identity: { @@ -5268,6 +5317,7 @@ export class RelayAssignmentStore { now, skips: candidateSkips }) + candidatesFinished++ if (!claimed) continue await this.markRegionalRehomeDispatchClaimed( transaction, @@ -5283,6 +5333,21 @@ export class RelayAssignmentStore { await this.markRegionalRehomeTickSkipped(transaction, now, intervalMs) } return null + }).catch((error: unknown): RegionalRehomeAttempt | null => { + // Only inventory contention is swallowed here; every other failure keeps + // its existing propagation and its dispatch-failure accounting. + if (!isDatabaseLockUnavailable(error)) throw error + // The dispatch tick runs every second; losing one to inventory contention + // costs a second of latency and never loses durable rehome state. The + // rolled-back transaction never disabled anything, so its pending disable + // log would describe a decision that did not happen. + candidateSkips.length = 0 + this.pendingRegionalRehomeDisableLog = null + warnSweepCellInventoryBusy( + 'claim-regional-rehome', + Math.max(1, candidatesTotal - candidatesFinished) + ) + return null }) const pendingDisableLog = this.pendingRegionalRehomeDisableLog this.pendingRegionalRehomeDisableLog = null @@ -5343,7 +5408,7 @@ export class RelayAssignmentStore { } const activityLeases = await this.lockAssignmentActivities(transaction, input.identity) assertAssignmentActivityCounts(assignment, activityLeases, 0) - const cells = await this.lockCellInventory(transaction) + const cells = await this.lockCellInventory(transaction, 'nowait') const admission = await cellAdmissionStates(transaction) const regions = new Map( (await transaction.query(`SELECT cell_id, region FROM relay_cell_regions`)).map((row) => [ @@ -5630,7 +5695,7 @@ export class RelayAssignmentStore { transaction: RelayDatabase, now: number ): Promise { - const cells = await this.lockCellInventory(transaction) + const cells = await this.lockCellInventory(transaction, 'nowait') const admission = await cellAdmissionStates(transaction) const regions = new Map( (await transaction.query(`SELECT cell_id, region FROM relay_cell_regions`)).map((row) => [ @@ -5874,6 +5939,7 @@ export class RelayAssignmentStore { [...quarantined, limit] ) let completed = 0 + let inventoryBusy = 0 for (const candidate of candidates) { // One poisoned row must not stall every later candidate: an invariant // throw here blocked fleet completions head-of-line in production. @@ -5890,9 +5956,14 @@ export class RelayAssignmentStore { if (changed) completed++ this.regionalRehomeCandidateQuarantine.delete(attemptId) } catch (error) { + if (isDatabaseLockUnavailable(error)) { + inventoryBusy++ + continue + } this.recordRegionalRehomeCandidateFailure('complete', attemptId, now, error) } } + warnSweepCellInventoryBusy('complete-ready-regional-rehomes', inventoryBusy) return completed } @@ -6112,7 +6183,7 @@ export class RelayAssignmentStore { leases, migration ) - const cells = await this.lockCellInventory(transaction) + const cells = await this.lockCellInventory(transaction, 'nowait') const target = cells.find((cell) => text(cell, 'cell_id') === targetCellId) const admission = await cellAdmissionStates(transaction) if ( @@ -6261,6 +6332,7 @@ export class RelayAssignmentStore { [now - REGIONAL_REHOME_MAX_REFRESH_MS, ...quarantined, limit] ) let aborted = 0 + let inventoryBusy = 0 for (const candidate of candidates) { const identity = { userId: text(candidate, 'user_id'), @@ -6318,7 +6390,7 @@ export class RelayAssignmentStore { integer(lease, 'expires_at') > now ) if (targetActive) return false - const cells = await this.lockCellInventory(transaction) + const cells = await this.lockCellInventory(transaction, 'nowait') const source = cells.find((cell) => text(cell, 'cell_id') === sourceCellId) const admission = await cellAdmissionStates(transaction) if ( @@ -6376,10 +6448,12 @@ export class RelayAssignmentStore { }) this.regionalRehomeCandidateQuarantine.delete(attemptId) } catch (error) { - this.recordRegionalRehomeCandidateFailure('abort', attemptId, now, error) + if (isDatabaseLockUnavailable(error)) inventoryBusy++ + else this.recordRegionalRehomeCandidateFailure('abort', attemptId, now, error) } if (changed) aborted++ } + warnSweepCellInventoryBusy('abort-expired-regional-rehomes', inventoryBusy) return aborted } @@ -6396,6 +6470,7 @@ export class RelayAssignmentStore { [now, now, abandonedBefore, abandonedBefore] ) let aborted = 0 + let inventoryBusy = 0 for (const candidate of candidates) { const didAbort = await this.database.transaction(async (transaction) => { const identity = { @@ -6480,7 +6555,7 @@ export class RelayAssignmentStore { ] .map((activityId) => activityLeaseById(activityLeases, activityId)) .filter((lease): lease is SqlRow => lease !== undefined) - if (obsoleteLeases.length > 0) await this.lockCellInventory(transaction) + if (obsoleteLeases.length > 0) await this.lockCellInventory(transaction, 'nowait') for (const lease of obsoleteLeases) { await this.removeActivityLease(transaction, identity, lease, now) } @@ -6498,7 +6573,7 @@ export class RelayAssignmentStore { ) return true } - const cells = await this.lockCellInventory(transaction) + const cells = await this.lockCellInventory(transaction, 'nowait') const sourceCellId = text(row, 'source_cell_id') const admissionRows = await transaction.query( `SELECT cell_id, admission_state, updated_at FROM relay_cell_admission @@ -6595,9 +6670,15 @@ export class RelayAssignmentStore { [now, now, identity.userId, identity.relayHostId, assignmentEpoch] ) return true + }).catch((error: unknown): boolean => { + // Expiry is durable; another director settling this row is not a failure. + if (!isDatabaseLockUnavailable(error)) throw error + inventoryBusy++ + return false }) if (didAbort) aborted++ } + warnSweepCellInventoryBusy('abort-expired-evacuations', inventoryBusy) return aborted } @@ -6665,7 +6746,7 @@ export class RelayAssignmentStore { const activityLeases = await this.lockAssignmentActivities(transaction, identity, true) const lease = activityLeaseById(activityLeases, text(candidate, 'activity_id')) if (!lease || integer(lease, 'expires_at') > now) return false - await this.lockCellInventory(transaction, true) + await this.lockCellInventory(transaction, 'nowait') await this.removeActivityLease(transaction, identity, lease, now) return true }) @@ -6709,7 +6790,7 @@ export class RelayAssignmentStore { [now], { failIfUnavailable: true } ) - if (expired.length > 0) await this.lockCellInventory(transaction, true) + if (expired.length > 0) await this.lockCellInventory(transaction, 'nowait') for (const row of expired) { await this.adjustCellReservation(transaction, text(row, 'cell_id'), -requestUnits(row)) await transaction.query( @@ -6782,7 +6863,7 @@ export class RelayAssignmentStore { targetCellId ] ) - const cells = await this.lockCellInventory(transaction) + const cells = await this.lockCellInventory(transaction, 'pool-default') const assignmentKeys = new Set( assignments.map((row) => assignmentKey(text(row, 'user_id'), text(row, 'relay_host_id')) @@ -6861,30 +6942,32 @@ export class RelayAssignmentStore { private async lockCellInventory( database: RelayDatabase, - failIfUnavailable = false + mode: CellInventoryLockMode ): Promise { // Every capacity-changing assignment takes the tiny cell inventory in one // order; dynamically locking only the selected target allowed cross-cell cycles. - return await database.queryLocked( + const rows = await database.queryLocked( `SELECT * FROM relay_cells ORDER BY cell_id ASC`, [], - { failIfUnavailable } + cellInventoryLockOptions(mode) ) + return rows } private async lockGeneralCellInventory( database: RelayDatabase, - failIfUnavailable = false + mode: CellInventoryLockMode ): Promise { - return await database.queryLocked( + const rows = await database.queryLocked( `SELECT * FROM relay_cells WHERE cell_id IN ( SELECT cell_id FROM relay_cell_admission WHERE admission_state = 'general' ) ORDER BY cell_id ASC`, [], - { failIfUnavailable } + cellInventoryLockOptions(mode) ) + return rows } private async leastLoadedCell( @@ -6892,7 +6975,7 @@ export class RelayAssignmentStore { lockedCells: SqlRow[] | undefined, preferredRegion: RelayRegion ): Promise { - const rows = lockedCells ?? (await this.lockCellInventory(database)) + const rows = lockedCells ?? (await this.lockCellInventory(database, 'pool-default')) const regions = new Map( (await database.query(`SELECT cell_id, region FROM relay_cell_regions`)).map((row) => [ text(row, 'cell_id'), @@ -7507,7 +7590,7 @@ export class RelayAssignmentStore { ) { throw new Error('activity_lease_shape_mismatch') } - const cells = await this.lockCellInventory(database) + const cells = await this.lockCellInventory(database, 'request') await database.query( `DELETE FROM relay_assignment_activity_leases WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'control' @@ -7886,6 +7969,23 @@ function isDatabaseLockUnavailable(error: unknown): boolean { return error instanceof Error && error.message === 'database_lock_unavailable' } +function cellInventoryLockOptions(mode: CellInventoryLockMode): RelayLockOptions { + if (mode === 'nowait') return { failIfUnavailable: true, measureHoldMs: true } + if (mode === 'pool-default') return { measureHoldMs: true } + return { lockTimeoutMs: CELL_INVENTORY_LOCK_TIMEOUT_MS, measureHoldMs: true } +} + +// Background sweeps take the cell inventory NOWAIT so they never queue ahead of +// assignment traffic. A skipped candidate is re-derived from durable state on +// the next tick, so it is ordinary contention, not a sweep failure: one summary +// line per tick, never an error and never a quarantine. +function warnSweepCellInventoryBusy(sweep: string, skipped: number): void { + if (skipped === 0) return + console.warn( + JSON.stringify({ event: 'orca_relay_sweep_cell_inventory_busy', sweep, skipped }) + ) +} + function isDatabaseLockTimeout(error: unknown): boolean { return String((error as { code?: unknown }).code) === '55P03' } diff --git a/cloud/apps/relay/src/cell-inventory-hold-samples.ts b/cloud/apps/relay/src/cell-inventory-hold-samples.ts new file mode 100644 index 00000000000..14941032d80 --- /dev/null +++ b/cloud/apps/relay/src/cell-inventory-hold-samples.ts @@ -0,0 +1,46 @@ +// Why: the cell inventory lock is held to COMMIT, and the assignment path runs +// many statements after taking it. Tuning the request-path wait bound needs the +// hold distribution, and no runtime metric carried it before this change. +export type CellInventoryHoldCounts = { + cellInventoryHoldMsMax: number + cellInventoryHoldMsP95: number + cellInventoryHolds: number +} + +// Bounded so a flush interval with heavy assignment traffic cannot grow the array +// without limit; the reservoir keeps the most recent holds. +const MAX_SAMPLES = 2_048 + +export function emptyCellInventoryHoldCounts(): CellInventoryHoldCounts { + return { cellInventoryHoldMsMax: 0, cellInventoryHoldMsP95: 0, cellInventoryHolds: 0 } +} + +export class CellInventoryHoldSamples { + private samples: number[] = [] + + record(holdMs: number): void { + if (!Number.isFinite(holdMs) || holdMs < 0) return + if (this.samples.length === MAX_SAMPLES) this.samples.shift() + this.samples.push(holdMs) + } + + consumeCounts(): CellInventoryHoldCounts { + const counts = this.readCounts() + this.samples = [] + return counts + } + + readCounts(): CellInventoryHoldCounts { + if (this.samples.length === 0) return emptyCellInventoryHoldCounts() + const sorted = [...this.samples].sort((left, right) => left - right) + return { + cellInventoryHoldMsMax: round(sorted[sorted.length - 1]!), + cellInventoryHoldMsP95: round(sorted[Math.ceil(0.95 * sorted.length) - 1] ?? 0), + cellInventoryHolds: sorted.length + } + } +} + +function round(value: number): number { + return Number(value.toFixed(3)) +} diff --git a/cloud/apps/relay/src/cell-inventory-lock-census.test.ts b/cloud/apps/relay/src/cell-inventory-lock-census.test.ts new file mode 100644 index 00000000000..d0527534935 --- /dev/null +++ b/cloud/apps/relay/src/cell-inventory-lock-census.test.ts @@ -0,0 +1,164 @@ +import { readFileSync } from 'node:fs' +import { describe, expect, it } from 'vitest' +import type { CellInventoryLockMode } from './assignment-store.js' + +// Which entry points can reach a call site. A site a sweep can enter must never +// take the bounded wait: its 55P03 becomes a terminal transaction failure, and +// the incident monitor freezes on a single one. +type Reachability = 'request' | 'sweep' | 'both' + +// 'caller' is not a CellInventoryLockMode: those sites take the mode threaded +// from `assign`, which is 'request' for a client and 'pool-default' for the +// evacuateDeadCells sweep. +type CensusMode = CellInventoryLockMode | 'caller' + +type CensusEntry = { method: string; mode: CensusMode; reach: Reachability } + +// Every lockCellInventory / lockGeneralCellInventory call site in +// assignment-store.ts, in source order. A new site fails this test until it is +// classified here, which is the point. +const CENSUS: CensusEntry[] = [ + { method: 'assignStickyOnce', mode: 'caller', reach: 'both' }, + { method: 'assignOnce', mode: 'caller', reach: 'both' }, + { method: 'assignOnce', mode: 'caller', reach: 'both' }, + { method: 'assignOnce', mode: 'nowait', reach: 'both' }, + { method: 'assignOnce', mode: 'nowait', reach: 'both' }, + { method: 'assignOnce', mode: 'nowait', reach: 'both' }, + { method: 'refreshDrainMigrationLeasesOnce', mode: 'request', reach: 'request' }, + { method: 'changeActivity', mode: 'request', reach: 'request' }, + { method: 'acquireActivity', mode: 'request', reach: 'request' }, + { method: 'activateControl', mode: 'request', reach: 'request' }, + { method: 'startEvacuation', mode: 'request', reach: 'request' }, + { method: 'completeEvacuationFromDeadSourceOnce', mode: 'request', reach: 'request' }, + { method: 'completeEvacuationFromDeadSourceOnce', mode: 'nowait', reach: 'request' }, + { method: 'supersedeRegisteredEvacuationOnce', mode: 'request', reach: 'request' }, + { method: 'supersedeRegisteredEvacuationOnce', mode: 'nowait', reach: 'request' }, + { method: 'prepareRegisteredCellSupersession', mode: 'request', reach: 'request' }, + { method: 'prepareRegisteredCellSupersession', mode: 'request', reach: 'request' }, + { method: 'completeEvacuation', mode: 'nowait', reach: 'both' }, + { method: 'completeEvacuation', mode: 'pool-default', reach: 'both' }, + { method: 'rebalanceDormant', mode: 'request', reach: 'request' }, + { method: 'startRegionalRehomeCandidate', mode: 'nowait', reach: 'sweep' }, + { method: 'lockedRegionalRehomeFleetSafety', mode: 'nowait', reach: 'sweep' }, + { method: 'completeRegionalRehomeCandidate', mode: 'nowait', reach: 'sweep' }, + { method: 'abortExpiredRegionalRehomes', mode: 'nowait', reach: 'sweep' }, + { method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' }, + { method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' }, + { method: 'releaseExpiredActivityLeases', mode: 'nowait', reach: 'sweep' }, + { method: 'releaseExpiredActivity', mode: 'nowait', reach: 'sweep' }, + { method: 'reconcileReservationAccounting', mode: 'pool-default', reach: 'both' }, + { method: 'leastLoadedCell', mode: 'pool-default', reach: 'both' }, + { method: 'removeSupersededSameCellControls', mode: 'request', reach: 'request' } +] + +// The background sweeps, and nothing else. A method reachable from one of these +// can be entered by a sweep tick, whatever else can also enter it. +const SWEEP_ROOTS = [ + 'refreshRegionalRehomeLeases', + 'completeReadyEvacuations', + 'completeReadyRegionalRehomes', + 'abortExpiredEvacuations', + 'abortExpiredRegionalRehomes', + 'reapRegionalRehomeAttempts', + 'releaseExpiredActivityLeases', + 'releaseExpiredActivity', + 'releaseExpiredRegionPreferences', + 'evacuateDeadCells', + 'claimRegionalRehome', + 'recordRegionalRehomeDispatchFailure' +] + +const DECLARATION = /^ {2}(?:private |public )?(?:static )?(?:async )?([A-Za-z_][\w]*)[(<]/ + +function storeSource(): string[] { + return readFileSync(new URL('./assignment-store.ts', import.meta.url), 'utf8').split('\n') +} + +// Why: a hand-written reachability column is a claim, not a check. Derive it, so +// a new sweep edge into a bounded site fails here instead of in production. +function sweepReachableMethods(lines: string[]): Set { + const bounds: { name: string; start: number }[] = [] + lines.forEach((line, index) => { + const declaration = DECLARATION.exec(line) + if (declaration) bounds.push({ name: declaration[1]!, start: index }) + }) + const callees = new Map>() + bounds.forEach((method, index) => { + const end = bounds[index + 1]?.start ?? lines.length + const names = callees.get(method.name) ?? new Set() + for (const call of lines.slice(method.start, end).join('\n').matchAll( + /this\.([A-Za-z_][\w]*)\s*\(/g + )) { + names.add(call[1]!) + } + callees.set(method.name, names) + }) + const reached = new Set() + const pending = [...SWEEP_ROOTS] + while (pending.length > 0) { + const name = pending.pop()! + if (reached.has(name)) continue + reached.add(name) + for (const callee of callees.get(name) ?? []) if (!reached.has(callee)) pending.push(callee) + } + return reached +} + +function readCallSites(): { method: string; mode: CensusMode }[] { + const sites: { method: string; mode: CensusMode }[] = [] + let method = '' + for (const line of storeSource()) { + const declaration = DECLARATION.exec(line) + if (declaration) method = declaration[1]! + if (/private async lock(General)?CellInventory\(/.test(line)) continue + const call = /lock(?:General)?CellInventory\(\s*\w+\s*,\s*(?:'([a-z-]+)'|(\w+))\s*\)/.exec(line) + if (!call) continue + sites.push({ method, mode: (call[1] ?? 'caller') as CensusMode }) + } + return sites +} + +describe('cell inventory lock call-site census', () => { + it('classifies every call site exactly as recorded', () => { + expect(readCallSites()).toEqual( + CENSUS.map(({ method, mode }) => ({ method, mode })) + ) + }) + + it('leaves no call site taking the inventory without naming a mode', () => { + const source = readFileSync(new URL('./assignment-store.ts', import.meta.url), 'utf8') + const unclassified = source + .split('\n') + .filter((line) => /lock(?:General)?CellInventory\(\s*\w+\s*\)/.test(line)) + .filter((line) => !line.includes('private async')) + + expect(unclassified).toEqual([]) + }) + + it('derives the same reachability the census claims', () => { + const reached = sweepReachableMethods(storeSource()) + const derived = readCallSites().map(({ method }) => reached.has(method)) + + expect(derived).toEqual(CENSUS.map((entry) => entry.reach !== 'request')) + }) + + // Why: this is the whole point of the classification. A shorter wait on a + // sweep-reachable site turns contention into a terminal transaction failure, + // and relayPostgresRetryExhausted freezes the incident gate at zero. + it('never puts a sweep-reachable site on the bounded wait', () => { + const reached = sweepReachableMethods(storeSource()) + const bounded = readCallSites().filter( + (site) => site.mode === 'request' && reached.has(site.method) + ) + + expect(bounded).toEqual([]) + }) + + it('routes every sweep-only site to NOWAIT so it can skip the tick', () => { + const queueing = CENSUS.filter( + (entry) => entry.reach === 'sweep' && entry.mode !== 'nowait' + ) + + expect(queueing).toEqual([]) + }) +}) diff --git a/cloud/apps/relay/src/cell-inventory-lock-contention.test.ts b/cloud/apps/relay/src/cell-inventory-lock-contention.test.ts new file mode 100644 index 00000000000..783d8a62e8b --- /dev/null +++ b/cloud/apps/relay/src/cell-inventory-lock-contention.test.ts @@ -0,0 +1,541 @@ +import { readFileSync } from 'node:fs' +import { afterEach, describe, expect, it, vi } from 'vitest' + +const fakes = vi.hoisted(() => ({ + statements: [] as string[], + query: vi.fn(async (sql: string) => { + fakes.statements.push(sql) + return { rows: [], rowCount: 0 } + }), + release: vi.fn(), + end: vi.fn(async () => undefined) +})) + +vi.mock('pg', () => ({ + default: { + Pool: class { + totalCount = 1 + idleCount = 1 + waitingCount = 0 + end = fakes.end + on = vi.fn() + connect = vi.fn(async () => ({ query: fakes.query, release: fakes.release })) + } + } +})) + +const { CELL_INVENTORY_LOCK_TIMEOUT_MS, RelayAssignmentStore } = await import( + './assignment-store.js' +) +const { consumeRelayCellInventoryHold, openInMemoryRelayDatabase, openRelayDatabase, POSTGRES_LOCK_TIMEOUT_MS } = + await import('./database.js') +const RESTORE = `SET LOCAL lock_timeout = '${POSTGRES_LOCK_TIMEOUT_MS}ms'` +type RelayDatabase = import('./database.js').RelayDatabase +type RelayLockOptions = import('./database.js').RelayLockOptions +type RelayTransactionOptions = import('./database.js').RelayTransactionOptions +type SqlRow = import('./database.js').SqlRow + +const CELL_INVENTORY_SQL = 'SELECT * FROM relay_cells ORDER BY cell_id ASC' + +// The assignment path locks the general-admission subset; both forms are the +// same ordered scan of the same 23-row table and share its lock queue. +function locksCellInventory(sql: string): boolean { + return sql.trim().startsWith('SELECT * FROM relay_cells') && sql.includes('ORDER BY cell_id ASC') +} +const CELLS = [ + { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 } +] +const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + +async function openFakePostgres(): Promise { + const database = await openRelayDatabase({ + databaseUrl: 'postgresql://relay:secret@127.0.0.1:5432/relay', + dataDir: './unused' + }) + fakes.statements.length = 0 + return database +} + +afterEach(() => { + fakes.statements.length = 0 + fakes.query.mockReset() + fakes.query.mockImplementation(async (sql: string) => { + fakes.statements.push(sql) + return { rows: [], rowCount: 0 } + }) +}) + +describe('bounded cell-inventory lock wait', () => { + // Why: a bound at or above the pool default would fence nothing, and one far + // below the hold time would convert ordinary contention into terminal failures. + it('keeps the request bound strictly inside the pool default', () => { + expect(CELL_INVENTORY_LOCK_TIMEOUT_MS).toBe(500) + expect(CELL_INVENTORY_LOCK_TIMEOUT_MS).toBeLessThan(POSTGRES_LOCK_TIMEOUT_MS) + }) + + // Why: SET LOCAL lasts to COMMIT. Left in place it would govern every later + // locked statement in the transaction and misattribute their 55P03s. + it('restores the pool default before the next statement in the transaction', async () => { + const database = await openFakePostgres() + + await database.transaction(async (transaction) => { + await transaction.queryLocked(CELL_INVENTORY_SQL, [], { lockTimeoutMs: 150 }) + await transaction.queryLocked('SELECT * FROM relay_assignments', []) + }) + + expect(fakes.statements).toEqual([ + 'BEGIN', + "SET LOCAL lock_timeout = '150ms'", + `${CELL_INVENTORY_SQL} FOR UPDATE`, + RESTORE, + 'SELECT * FROM relay_assignments FOR UPDATE', + 'COMMIT' + ]) + await database.close() + }) + + it('restores the pool default when the bounded lock itself times out', async () => { + const database = await openFakePostgres() + fakes.query.mockImplementation(async (sql: string) => { + fakes.statements.push(sql) + if (sql.includes('FOR UPDATE')) { + throw Object.assign(new Error('lock timeout'), { code: '55P03' }) + } + return { rows: [], rowCount: 0 } + }) + + await expect( + database.transaction(async (transaction) => { + await transaction.queryLocked(CELL_INVENTORY_SQL, [], { lockTimeoutMs: 150 }) + }) + ).rejects.toMatchObject({ code: '55P03' }) + + // The retry wrapper makes three attempts; each one must leave the default back. + expect(fakes.statements.filter((sql) => sql.startsWith('SET LOCAL'))).toEqual( + Array.from({ length: 3 }, () => ["SET LOCAL lock_timeout = '150ms'", RESTORE]).flat() + ) + await database.close() + }) + + it('rejects a lock bound that is not a positive whole number of milliseconds', async () => { + const database = await openFakePostgres() + + for (const lockTimeoutMs of [0, -1, 1.5, Number.NaN]) { + await expect( + database.transaction( + async (transaction) => + await transaction.queryLocked(CELL_INVENTORY_SQL, [], { lockTimeoutMs }) + ) + ).rejects.toThrow('invalid_lock_timeout') + } + await database.close() + }) + + it('skips the timeout for a NOWAIT lock, which never queues', async () => { + const database = await openFakePostgres() + + await database.transaction(async (transaction) => { + await transaction.queryLocked(CELL_INVENTORY_SQL, [], { + failIfUnavailable: true, + lockTimeoutMs: 150 + }) + }) + + expect(fakes.statements.filter((sql) => sql.startsWith('SET LOCAL'))).toEqual([]) + await database.close() + }) + + it('skips the timeout outside a transaction, where SET LOCAL cannot survive', async () => { + const database = await openFakePostgres() + + await database.queryLocked(CELL_INVENTORY_SQL, [], { lockTimeoutMs: 150 }) + + expect(fakes.statements).toEqual([`${CELL_INVENTORY_SQL} FOR UPDATE`]) + await database.close() + }) + + it('ignores the timeout on SQLite, which has no SET LOCAL', async () => { + const database = await openInMemoryRelayDatabase() + + const rows = await database.transaction( + async (transaction) => + await transaction.queryLocked(CELL_INVENTORY_SQL, [], { lockTimeoutMs: 150 }) + ) + + expect(rows).toEqual([]) + await database.close() + }) + + // Why: testing the helper alone would pass with the store still queueing for + // the pool's one-second default. + // Why: testing the helper alone would pass with the request path still queueing + // for the pool's full second. + it('never lets a request path take the unbounded wait', async () => { + const database = await openInMemoryRelayDatabase() + const probe = new InventoryLockProbe(database) + const store = new RelayAssignmentStore(probe, () => 1_000) + await store.reconcileCells(CELLS) + probe.inventoryLocks.length = 0 + + // Assignment takes the general-admission subset; evacuation takes them all. + await store.assign(identity) + const generalLocks = probe.inventoryLocks.length + await store.startEvacuation(identity, 'cell-b') + + expect(generalLocks).toBeGreaterThan(0) + expect(probe.inventoryLocks.length).toBeGreaterThan(generalLocks) + for (const options of probe.inventoryLocks) { + const bounded = options?.lockTimeoutMs === CELL_INVENTORY_LOCK_TIMEOUT_MS + expect(bounded || options?.failIfUnavailable === true).toBe(true) + } + await database.close() + }) + + // Why: evacuateDeadCells re-enters placement from a sweep. A 55P03 there would + // be reported as a terminal sweep failure and freeze the incident gate. + it('keeps the pool default when a sweep re-enters placement', async () => { + const requestModes = await recordAssignInventoryModes(async (store) => { + await store.assign(identity) + }) + const sweepModes = await recordAssignInventoryModes(async (store) => { + await store.assign(identity, undefined, undefined, 'pool-default') + }) + + // The inventory-first retry is the lane that carries the caller's mode. + expect(requestModes).toContain(CELL_INVENTORY_LOCK_TIMEOUT_MS) + expect(sweepModes).not.toContain(CELL_INVENTORY_LOCK_TIMEOUT_MS) + expect(sweepModes.filter((mode) => mode === 'nowait').length).toBe( + requestModes.filter((mode) => mode === 'nowait').length + ) + }) + + it('sends the sweep that re-enters placement down the unbounded lane', async () => { + const database = await openInMemoryRelayDatabase() + const probe = new InventoryLockProbe(database) + let now = 1_000 + const store = new RelayAssignmentStore(probe, () => now, { + requireLiveCells: true, + heartbeatTtlMs: 45_000 + }) + await store.reconcileCells(CELLS) + for (const cell of CELLS) { + await store.recordCellHeartbeat({ + cellId: cell.id, + cellUrl: cell.url, + cellIncarnation: `1111111${cell.id.slice(-1)}-1111-4111-8111-111111111111`, + startedAt: 50, + ready: true, + observedRequests: 0 + }) + } + await store.assign(identity) + // Let every heartbeat lapse so the sweep sees the assigned cell as dead. + now += 45_001 + probe.inventoryLocks.length = 0 + probe.failActivityLockOnce = true + + await store.evacuateDeadCells() + + expect(probe.inventoryLocks).not.toEqual([]) + for (const options of probe.inventoryLocks) { + expect(options?.lockTimeoutMs).toBeUndefined() + } + await database.close() + }) + + // Why: the SQLite hold test cannot reach PostgresDatabase.transaction, which is + // the only path production ever takes. + it('records the hold on the PostgreSQL transaction path', async () => { + const database = await openFakePostgres() + + await database.transaction(async (transaction) => { + await transaction.queryLocked(CELL_INVENTORY_SQL, [], { + lockTimeoutMs: 150, + measureHoldMs: true + }) + }) + + expect(consumeRelayCellInventoryHold(database).cellInventoryHolds).toBe(1) + await database.close() + }) + + it('records no hold for a PostgreSQL transaction that took no measured lock', async () => { + const database = await openFakePostgres() + + await database.transaction(async (transaction) => { + await transaction.queryLocked(CELL_INVENTORY_SQL, [], { lockTimeoutMs: 150 }) + }) + + expect(consumeRelayCellInventoryHold(database).cellInventoryHolds).toBe(0) + await database.close() + }) + + // Why: index.ts boots a server on import, so its wiring can only be read. An + // unspread hold metric is invisible: the flush simply omits the fields. + it('spreads the hold counts into the runtime metrics flush', () => { + const source = readFileSync(new URL('./index.ts', import.meta.url), 'utf8') + const flush = /observability\.start\(\(\) => \(\{([^}]*)\}\)\)/.exec(source) + + expect(flush?.[1]).toContain('...consumeRelayCellInventoryHold(database)') + }) + + // Why: 500ms is a first value, not a measurement. Tuning it needs the hold + // distribution, which no runtime metric carried. + it('reports how long the inventory lock was held to COMMIT', async () => { + const database = await openInMemoryRelayDatabase() + const store = new RelayAssignmentStore(database, () => 1_000) + await store.reconcileCells(CELLS) + consumeRelayCellInventoryHold(database) + + await store.assign(identity) + + const counts = consumeRelayCellInventoryHold(database) + expect(counts.cellInventoryHolds).toBeGreaterThan(0) + expect(counts.cellInventoryHoldMsMax).toBeGreaterThanOrEqual(counts.cellInventoryHoldMsP95) + expect(counts.cellInventoryHoldMsMax).toBeGreaterThan(0) + // Consuming resets the window so the next flush reports its own holds. + expect(consumeRelayCellInventoryHold(database).cellInventoryHolds).toBe(0) + await database.close() + }) +}) + +// Why: the incident monitor freezes at zero exhausted transactions. A sweep that +// steps aside must not spend the retry budget or report a terminal failure. +describe('sweep lock skips stay off the transaction retry counters', () => { + it('reports neither a retry nor an exhaustion when NOWAIT finds the lock held', async () => { + const database = await openFakePostgres() + fakes.query.mockImplementation(async (sql: string) => { + fakes.statements.push(sql) + if (sql.includes('FOR UPDATE NOWAIT')) { + throw Object.assign(new Error('could not obtain lock'), { code: '55P03' }) + } + return { rows: [], rowCount: 0 } + }) + const events: string[] = [] + const warn = vi.spyOn(console, 'warn').mockImplementation((line: unknown) => { + try { + events.push(String((JSON.parse(line as string) as { event?: unknown }).event)) + } catch { + // non-JSON lines are not transaction telemetry + } + }) + + try { + await expect( + database.transaction(async (transaction) => { + await transaction.queryLocked(CELL_INVENTORY_SQL, [], { failIfUnavailable: true }) + }) + ).rejects.toThrow('database_lock_unavailable') + } finally { + warn.mockRestore() + } + + expect(events).not.toContain('orca_relay_postgres_transaction_retry') + expect(events).not.toContain('orca_relay_postgres_transaction_exhausted') + expect(fakes.statements.filter((sql) => sql === 'BEGIN')).toHaveLength(1) + await database.close() + }) +}) + +describe('background sweeps skip a contended cell inventory', () => { + it('takes the inventory NOWAIT and skips the tick instead of queueing', async () => { + const database = await openInMemoryRelayDatabase() + const probe = new InventoryLockProbe(database) + let now = 1_000 + const store = new RelayAssignmentStore(probe, () => now) + await store.reconcileCells(CELLS) + const assignment = await store.assign(identity) + await store.activateControl(identity, { + cellId: assignment.cellId, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + await store.startEvacuation(identity, 'cell-b') + now += 24 * 60 * 60_000 + probe.inventoryLocks.length = 0 + probe.failNoWait = true + const warnings = collectWarnings('orca_relay_sweep_cell_inventory_busy') + + let aborted: number + try { + aborted = await store.abortExpiredEvacuations() + } finally { + warnings.restore() + } + + expect(aborted).toBe(0) + expect(probe.inventoryLocks).not.toEqual([]) + expect(probe.inventoryLocks.every((options) => options?.failIfUnavailable === true)).toBe( + true + ) + expect(warnings.entries).toEqual([ + { event: 'orca_relay_sweep_cell_inventory_busy', sweep: 'abort-expired-evacuations', skipped: 1 } + ]) + await database.close() + }) + + // Why: a summary line on every quiet tick would bury the contended ones. + it('says nothing on a tick that skipped no candidate', async () => { + const database = await openInMemoryRelayDatabase() + let now = 1_000 + const store = new RelayAssignmentStore(database, () => now) + await store.reconcileCells(CELLS) + const assignment = await store.assign(identity) + await store.activateControl(identity, { + cellId: assignment.cellId, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + await store.startEvacuation(identity, 'cell-b') + now += 24 * 60 * 60_000 + const warnings = collectWarnings('orca_relay_sweep_cell_inventory_busy') + + let aborted: number + try { + aborted = await store.abortExpiredEvacuations() + } finally { + warnings.restore() + } + + expect(aborted).toBe(1) + expect(warnings.entries).toEqual([]) + await database.close() + }) + + it('still aborts the expired evacuation once the inventory is free', async () => { + const database = await openInMemoryRelayDatabase() + const probe = new InventoryLockProbe(database) + let now = 1_000 + const store = new RelayAssignmentStore(probe, () => now) + await store.reconcileCells(CELLS) + const assignment = await store.assign(identity) + await store.activateControl(identity, { + cellId: assignment.cellId, + assignmentEpoch: assignment.assignmentEpoch, + generation: 1 + }) + await store.startEvacuation(identity, 'cell-b') + now += 24 * 60 * 60_000 + + expect(await store.abortExpiredEvacuations()).toBe(1) + await database.close() + }) +}) + +// Returns each inventory lock the run took, as its bound or 'nowait'. +async function recordAssignInventoryModes( + drive: (store: InstanceType) => Promise +): Promise<(number | 'nowait' | 'pool-default')[]> { + const database = await openInMemoryRelayDatabase() + const probe = new InventoryLockProbe(database) + const store = new RelayAssignmentStore(probe, () => 1_000) + await store.reconcileCells(CELLS) + probe.inventoryLocks.length = 0 + probe.failActivityLockOnce = true + await drive(store) + await database.close() + return probe.inventoryLocks.map((options) => + options?.failIfUnavailable ? 'nowait' : (options?.lockTimeoutMs ?? 'pool-default') + ) +} + +function collectWarnings(event: string) { + const entries: Record[] = [] + const original = console.warn + console.warn = (line: unknown, ...rest: unknown[]) => { + try { + const parsed = JSON.parse(line as string) as Record + if (parsed.event === event) return void entries.push(parsed) + } catch { + // fall through to the real console for non-JSON lines + } + original(line, ...rest) + } + return { entries, restore: () => (console.warn = original) } +} + +const ACTIVITY_LEASE_SQL = 'SELECT * FROM relay_assignment_activity_leases' + +class InventoryLockProbe implements RelayDatabase { + readonly inventoryLocks: (RelayLockOptions | undefined)[] = [] + failNoWait = false + // Forces the next assign attempt down its inventory-first retry, the only lane + // that reaches the threaded lock mode. + failActivityLockOnce = false + + constructor(private readonly delegate: RelayDatabase) {} + + async query(sql: string, params?: unknown[]): Promise { + return await this.delegate.query(sql, params) + } + + async queryLocked( + sql: string, + params?: unknown[], + options?: RelayLockOptions + ): Promise { + if (locksCellInventory(sql)) { + this.inventoryLocks.push(options) + if (this.failNoWait && options?.failIfUnavailable) { + throw new Error('database_lock_unavailable') + } + } + if (this.failActivityLockOnce && sql.trim().startsWith(ACTIVITY_LEASE_SQL) && options?.failIfUnavailable) { + this.failActivityLockOnce = false + throw new Error('database_lock_unavailable') + } + return await this.delegate.queryLocked(sql, params, options) + } + + async transaction( + operation: (transaction: RelayDatabase) => Promise, + options?: RelayTransactionOptions + ): Promise { + return await this.delegate.transaction( + async (transaction) => await operation(new InventoryLockProbeTransaction(transaction, this)), + options + ) + } + + async close(): Promise {} +} + +class InventoryLockProbeTransaction implements RelayDatabase { + constructor( + private readonly delegate: RelayDatabase, + private readonly probe: InventoryLockProbe + ) {} + + async query(sql: string, params?: unknown[]): Promise { + return await this.delegate.query(sql, params) + } + + async queryLocked( + sql: string, + params?: unknown[], + options?: RelayLockOptions + ): Promise { + if (locksCellInventory(sql)) { + this.probe.inventoryLocks.push(options) + if (this.probe.failNoWait && options?.failIfUnavailable) { + throw new Error('database_lock_unavailable') + } + } + if ( + this.probe.failActivityLockOnce && + sql.trim().startsWith(ACTIVITY_LEASE_SQL) && + options?.failIfUnavailable + ) { + this.probe.failActivityLockOnce = false + throw new Error('database_lock_unavailable') + } + return await this.delegate.queryLocked(sql, params, options) + } + + async transaction(operation: (transaction: RelayDatabase) => Promise): Promise { + return await operation(this) + } + + async close(): Promise {} +} diff --git a/cloud/apps/relay/src/database.ts b/cloud/apps/relay/src/database.ts index f7208863f42..326ab010ccb 100644 --- a/cloud/apps/relay/src/database.ts +++ b/cloud/apps/relay/src/database.ts @@ -1,4 +1,5 @@ import { mkdirSync } from 'node:fs' +import { performance } from 'node:perf_hooks' import { join } from 'node:path' import { DatabaseSync } from 'node:sqlite' import pg from 'pg' @@ -8,9 +9,37 @@ import { type PostgresPoolPressureCounts } from './postgres-pool-pressure.js' import { applyPostgresSchema } from './postgres-schema-startup.js' +import { + CellInventoryHoldSamples, + emptyCellInventoryHoldCounts, + type CellInventoryHoldCounts +} from './cell-inventory-hold-samples.js' + +export const POSTGRES_LOCK_TIMEOUT_MS = 1_000 + +function setLocalLockTimeout(milliseconds: number): string { + if (!Number.isInteger(milliseconds) || milliseconds < 1) { + throw new Error('invalid_lock_timeout') + } + return `SET LOCAL lock_timeout = '${milliseconds}ms'` +} export type SqlRow = Record -export type RelayLockOptions = { failIfUnavailable?: boolean } +export type RelayLockOptions = { + failIfUnavailable?: boolean + // Only honoured inside a transaction: SET LOCAL is a no-op in autocommit. + lockTimeoutMs?: number + // Report how long this lock is held to COMMIT. The hold, not the wait, is what + // forms the queue, and nothing measured it before. + measureHoldMs?: boolean +} + +// A transaction that can report how long it held a measured lock before COMMIT. +type HoldMeasuringTransaction = { consumeHoldMs(): number | undefined } + +function measuredHoldMs(transaction: unknown): number | undefined { + return (transaction as HoldMeasuringTransaction).consumeHoldMs?.() +} export type RelayTransactionOptions = { reportRetries?: boolean } export interface RelayDatabase { @@ -612,9 +641,23 @@ function postgresTransactionErrorPhase(error: unknown): string { class SqliteTransaction implements RelayDatabase { readonly dialect = 'sqlite' as const + private heldFromMs: number | undefined constructor(protected readonly database: DatabaseSync) {} + consumeHoldMs(): number | undefined { + if (this.heldFromMs === undefined) return undefined + const holdMs = performance.now() - this.heldFromMs + this.heldFromMs = undefined + return holdMs + } + + protected noteHeld(options: RelayLockOptions): void { + if (options.measureHoldMs && this.heldFromMs === undefined) { + this.heldFromMs = performance.now() + } + } + async query(sql: string, params: unknown[] = []): Promise { const statement = this.database.prepare(sql) const bound = params.map((value) => (value === undefined ? null : value)) as never[] @@ -626,9 +669,11 @@ class SqliteTransaction implements RelayDatabase { async queryLocked( sql: string, params: unknown[] = [], - _options: RelayLockOptions = {} + options: RelayLockOptions = {} ): Promise { - return await this.query(sql, params) + const rows = await this.query(sql, params) + this.noteHeld(options) + return rows } async transaction( @@ -643,6 +688,11 @@ class SqliteTransaction implements RelayDatabase { class SqliteDatabase extends SqliteTransaction { private tail: Promise = Promise.resolve() + private readonly holds = new CellInventoryHoldSamples() + + consumeHoldCounts(): CellInventoryHoldCounts { + return this.holds.consumeCounts() + } override async query(sql: string, params: unknown[] = []): Promise { await this.tail @@ -655,9 +705,11 @@ class SqliteDatabase extends SqliteTransaction { this.tail = new Promise((resolve) => (release = resolve)) await previous this.database.exec('BEGIN IMMEDIATE') + const transaction = new SqliteTransaction(this.database) try { - const result = await operation(new SqliteTransaction(this.database)) + const result = await operation(transaction) this.database.exec('COMMIT') + this.holds.record(measuredHoldMs(transaction) ?? Number.NaN) return result } catch (error) { this.database.exec('ROLLBACK') @@ -675,9 +727,17 @@ class SqliteDatabase extends SqliteTransaction { class PostgresTransaction implements RelayDatabase { readonly dialect = 'postgres' as const + private heldFromMs: number | undefined constructor(protected readonly client: pg.PoolClient) {} + consumeHoldMs(): number | undefined { + if (this.heldFromMs === undefined) return undefined + const holdMs = performance.now() - this.heldFromMs + this.heldFromMs = undefined + return holdMs + } + async query(sql: string, params: unknown[] = []): Promise { try { const result = await this.client.query(postgresSql(sql), params) @@ -693,11 +753,21 @@ class PostgresTransaction implements RelayDatabase { params: unknown[] = [], options: RelayLockOptions = {} ): Promise { + // SET LOCAL lasts to COMMIT, so a bound left in place would silently govern + // every later locked statement in the transaction and misattribute its 55P03s. + const bounded = options.lockTimeoutMs !== undefined && !options.failIfUnavailable try { - return await this.query( + // A blocked waiter holds its pooled client for the whole lock_timeout, so + // hot tiny-table locks bound their own wait well under the pool default. + if (bounded) await this.query(setLocalLockTimeout(options.lockTimeoutMs!)) + const rows = await this.query( `${sql} FOR UPDATE${options.failIfUnavailable ? ' NOWAIT' : ''}`, params ) + if (options.measureHoldMs && this.heldFromMs === undefined) { + this.heldFromMs = performance.now() + } + return rows } catch (error) { if ( options.failIfUnavailable && @@ -706,6 +776,10 @@ class PostgresTransaction implements RelayDatabase { throw new Error('database_lock_unavailable') } throw error + } finally { + // Restore on the error path too: the transaction may still be retried or + // continue with unrelated locks after a caught lock failure. + if (bounded) await this.query(setLocalLockTimeout(POSTGRES_LOCK_TIMEOUT_MS)).catch(() => undefined) } } @@ -723,7 +797,6 @@ const POSTGRES_TRANSACTION_ATTEMPTS = 3 const POSTGRES_RETRY_MAX_DELAY_MS = 25 const POSTGRES_CONNECTION_TIMEOUT_MS = 2_000 const POSTGRES_STATEMENT_TIMEOUT_MS = 5_000 -const POSTGRES_LOCK_TIMEOUT_MS = 1_000 const POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS = 5_000 function retryablePostgresTransactionError(error: unknown): boolean { @@ -749,6 +822,11 @@ async function waitForPostgresRetry(random: () => number = Math.random): Promise class PostgresDatabase implements RelayDatabase { readonly dialect = 'postgres' as const private readonly pressure: PostgresPoolPressure + private readonly holds = new CellInventoryHoldSamples() + + consumeHoldCounts(): CellInventoryHoldCounts { + return this.holds.consumeCounts() + } constructor(private readonly pool: pg.Pool) { this.pressure = new PostgresPoolPressure(pool) @@ -770,6 +848,8 @@ class PostgresDatabase implements RelayDatabase { options: RelayLockOptions = {} ): Promise { try { + // No transaction here, so options.lockTimeoutMs cannot apply: SET LOCAL + // would be discarded at the autocommit boundary before the lock is taken. return await this.query( `${sql} FOR UPDATE${options.failIfUnavailable ? ' NOWAIT' : ''}`, params @@ -791,10 +871,12 @@ class PostgresDatabase implements RelayDatabase { ): Promise { for (let attempt = 1; attempt <= POSTGRES_TRANSACTION_ATTEMPTS; attempt++) { const client = await this.pressure.connect() + const transaction = new PostgresTransaction(client) try { await client.query('BEGIN') - const result = await operation(new PostgresTransaction(client)) + const result = await operation(transaction) await client.query('COMMIT') + this.holds.record(measuredHoldMs(transaction) ?? Number.NaN) return result } catch (error) { await client.query('ROLLBACK').catch(() => undefined) @@ -852,6 +934,13 @@ export function consumeRelayDatabasePoolPressure( : emptyPostgresPoolPressureCounts() } +export function consumeRelayCellInventoryHold( + database: RelayDatabase +): CellInventoryHoldCounts { + const holder = database as { consumeHoldCounts?: () => CellInventoryHoldCounts } + return holder.consumeHoldCounts?.() ?? emptyCellInventoryHoldCounts() +} + export function readRelayDatabasePoolPressure( database: RelayDatabase ): PostgresPoolPressureCounts { diff --git a/cloud/apps/relay/src/index.ts b/cloud/apps/relay/src/index.ts index 635f3ae9b38..541884362c2 100644 --- a/cloud/apps/relay/src/index.ts +++ b/cloud/apps/relay/src/index.ts @@ -10,12 +10,14 @@ import { roleOwnsAssignmentMaintenance } from './cell-admission-startup.js' import { + consumeRelayCellInventoryHold, consumeRelayDatabasePoolPressure, openRelayDatabase, readRelayDatabasePoolPressure } from './database.js' import { runAssignmentCleanup } from './assignment-cleanup-steps.js' import { runRelayBackgroundOperation } from './relay-background-operation.js' +import { jitteredSweepIntervalMs } from './relay-sweep-schedule.js' import { observedRelayRequests } from './relay-observability.js' import { startRegionalRehomeWorker } from './regional-rehome-worker.js' import { createRelayServer } from './relay-server.js' @@ -54,7 +56,7 @@ const cleanupTimer = setInterval( const assignmentCleanupTimer = roleOwnsAssignmentMaintenance(config.role) ? setInterval(() => { void runAssignmentCleanup(assignments) - }, 30_000) + }, jitteredSweepIntervalMs(30_000)) : null const inventorySnapshotTimer = roleOwnsAssignmentMaintenance(config.role) ? setInterval(() => { @@ -78,7 +80,8 @@ inventorySnapshotTimer?.unref() migrationInventoryTimer?.unref() observability.start(() => ({ ...runtimeCounts(), - ...consumeRelayDatabasePoolPressure(database) + ...consumeRelayDatabasePoolPressure(database), + ...consumeRelayCellInventoryHold(database) })) const regionalRehomeWorker = startRegionalRehomeWorker(config, assignments, { safetySnapshot: () => ({ diff --git a/cloud/apps/relay/src/regional-rehome-store.test.ts b/cloud/apps/relay/src/regional-rehome-store.test.ts index 43f293b1131..6f57283396f 100644 --- a/cloud/apps/relay/src/regional-rehome-store.test.ts +++ b/cloud/apps/relay/src/regional-rehome-store.test.ts @@ -5,7 +5,12 @@ import { REGIONAL_REHOME_QUARANTINE_MS, REGIONAL_REHOME_REDRAIN_SEND_LIMIT } from './assignment-store.js' -import { openInMemoryRelayDatabase, type RelayDatabase, type SqlRow } from './database.js' +import { + openInMemoryRelayDatabase, + type RelayDatabase, + type RelayLockOptions, + type SqlRow +} from './database.js' import { REGIONAL_REHOME_SQL_FAILURES_LIMIT, REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT @@ -556,6 +561,290 @@ describe('regional rehome assignment state', () => { await context.database.close() }) + it('skips a rehome dispatch tick on a contended cell inventory', async () => { + const probe = new CellInventoryLockProbe() + const context = await setup({ wrap: (database) => probe.wrap(database) }) + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + await activatePreferredSource(context, identity) + probe.reset() + probe.failNoWait = true + const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy') + + let attempt: unknown + try { + attempt = await context.store.claimRegionalRehome() + } finally { + busy.restore() + } + + expect(attempt).toBeNull() + expect(probe.locks).not.toEqual([]) + expect(probe.locks.every((options) => options?.failIfUnavailable === true)).toBe(true) + expect(busy.entries).toEqual([ + { + event: 'orca_relay_sweep_cell_inventory_busy', + sweep: 'claim-regional-rehome', + skipped: 1 + } + ]) + + probe.failNoWait = false + expect(await context.store.claimRegionalRehome()).toMatchObject({ + sourceCellId: source.id, + targetCellId: target.id + }) + await context.database.close() + }) + + // Why: the redrain lane reaches the inventory through the fleet-safety read + // rather than through candidate selection, so it needs its own coverage. + // Why: one contended candidate must cost its own tick, not the whole page. The + // sweeps are explicitly per-candidate isolated for exactly this reason. + it('completes the candidates behind a contended one', async () => { + const probe = new CellInventoryLockProbe() + const context = await setup({ wrap: (database) => probe.wrap(database) }) + const identities = [ + { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }, + { userId: 'user-2', relayHostId: 'ponmlkjihgfedcba' } + ] + for (const identity of identities) { + // Dispatch is rate limited, so each claim needs its own interval. + context.advance(60_000) + await freshHeartbeats(context) + const sourceControl = await activatePreferredSource(context, identity) + const attempt = await context.store.claimRegionalRehome() + await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: 2, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: 2 + }) + await context.store.releaseActivity(identity, sourceControl) + } + probe.reset() + probe.failNoWaitOnce = true + const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy') + + let completed: number + try { + completed = await context.store.completeReadyRegionalRehomes() + } finally { + busy.restore() + } + + expect(completed).toBe(1) + expect(busy.entries).toEqual([ + { + event: 'orca_relay_sweep_cell_inventory_busy', + sweep: 'complete-ready-regional-rehomes', + skipped: 1 + } + ]) + await context.database.close() + }) + + // Why: only inventory contention is ordinary. Every other failure must keep its + // existing propagation and its dispatch-failure accounting. + it('propagates a claim failure that is not inventory contention', async () => { + const probe = new CellInventoryLockProbe() + const context = await setup({ wrap: (database) => probe.wrap(database) }) + await activatePreferredSource(context, { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }) + probe.reset() + probe.failWith = new Error('relay_capacity_exhausted') + const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy') + + try { + await expect(context.store.claimRegionalRehome()).rejects.toThrow( + 'relay_capacity_exhausted' + ) + } finally { + busy.restore() + } + + expect(busy.entries).toEqual([]) + await context.database.close() + }) + + // Why: the transaction dies at the first contended candidate, so every + // candidate behind it is abandoned too. Reporting one would understate the tick. + it('reports every candidate the contended tick abandoned', async () => { + const probe = new CellInventoryLockProbe() + const context = await setup({ wrap: (database) => probe.wrap(database) }) + await activatePreferredSource(context, { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }) + await activatePreferredSource(context, { userId: 'user-2', relayHostId: 'ponmlkjihgfedcba' }) + await activatePreferredSource(context, { userId: 'user-3', relayHostId: 'aaaabbbbccccdddd' }) + probe.reset() + probe.failNoWait = true + const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy') + + try { + expect(await context.store.claimRegionalRehome()).toBeNull() + } finally { + busy.restore() + } + + expect(busy.entries).toEqual([ + { + event: 'orca_relay_sweep_cell_inventory_busy', + sweep: 'claim-regional-rehome', + skipped: 3 + } + ]) + await context.database.close() + }) + + it('skips a redrain tick on a contended cell inventory', async () => { + const probe = new CellInventoryLockProbe() + const context = await setup({ wrap: (database) => probe.wrap(database) }) + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + await activatePreferredSource(context, identity) + const attempt = await context.store.claimRegionalRehome() + await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: 2, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: 2 + }) + context.advance(60 * 60_000 + 1) + await freshHeartbeats(context) + probe.reset() + probe.failNoWait = true + const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy') + + let redrain: unknown + try { + redrain = await context.store.claimRegionalRehome() + } finally { + busy.restore() + } + + expect(redrain).toBeNull() + expect(probe.locks).not.toEqual([]) + expect(probe.locks.every((options) => options?.failIfUnavailable === true)).toBe(true) + expect(busy.entries).toEqual([ + { + event: 'orca_relay_sweep_cell_inventory_busy', + sweep: 'claim-regional-rehome', + skipped: 1 + } + ]) + + probe.failNoWait = false + expect(await context.store.claimRegionalRehome()).toMatchObject({ + attemptId: attempt!.attemptId, + sendAttempts: 2 + }) + await context.database.close() + }) + + it('skips a completion tick on a contended cell inventory without quarantining it', async () => { + const probe = new CellInventoryLockProbe() + const context = await setup({ wrap: (database) => probe.wrap(database) }) + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + const sourceControl = await activatePreferredSource(context, identity) + const attempt = await context.store.claimRegionalRehome() + await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: 2, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: 2 + }) + await context.store.releaseActivity(identity, sourceControl) + probe.reset() + probe.failNoWait = true + const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy') + const failures = collectCandidateFailureWarnings() + + let completed: number + try { + completed = await context.store.completeReadyRegionalRehomes() + } finally { + failures.restore() + busy.restore() + } + + expect(completed).toBe(0) + expect(probe.locks).not.toEqual([]) + expect(probe.locks.every((options) => options?.failIfUnavailable === true)).toBe(true) + expect(failures.entries).toEqual([]) + expect(busy.entries).toEqual([ + { + event: 'orca_relay_sweep_cell_inventory_busy', + sweep: 'complete-ready-regional-rehomes', + skipped: 1 + } + ]) + + probe.failNoWait = false + expect(await context.store.completeReadyRegionalRehomes()).toBe(1) + await context.database.close() + }) + + // Why: a contended inventory is another director settling the same row, not a + // poisoned candidate. Quarantining on it would exclude a healthy attempt from + // the sweep's LIMIT pages for 15 minutes. + it('skips an abort tick on a contended cell inventory without quarantining it', async () => { + const probe = new CellInventoryLockProbe() + const context = await setup({ wrap: (database) => probe.wrap(database) }) + const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } + const sourceControl = await activatePreferredSource(context, identity) + const attempt = await context.store.claimRegionalRehome() + await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + const targetControl = await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: 2, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: 2 + }) + await context.store.releaseActivity(identity, sourceControl) + await context.store.releaseActivity(identity, targetControl) + context.advance(24 * 60 * 60_000) + await heartbeat(context.store, source, sourceIncarnation, 1, 2) + probe.reset() + probe.failNoWait = true + const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy') + const failures = collectCandidateFailureWarnings() + + let aborted: number + try { + aborted = await context.store.abortExpiredRegionalRehomes() + } finally { + failures.restore() + busy.restore() + } + + expect(aborted).toBe(0) + expect(probe.locks).not.toEqual([]) + expect(probe.locks.every((options) => options?.failIfUnavailable === true)).toBe(true) + expect(failures.entries).toEqual([]) + expect(busy.entries).toEqual([ + { + event: 'orca_relay_sweep_cell_inventory_busy', + sweep: 'abort-expired-regional-rehomes', + skipped: 1 + } + ]) + + probe.failNoWait = false + expect(await context.store.abortExpiredRegionalRehomes()).toBe(1) + await context.database.close() + }) + it('rolls back an inactive registered target only after the 24-hour bound', async () => { const context = await setup() const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } @@ -1208,10 +1497,12 @@ function collectDisableWarnings() { } } -async function setup(options: { sourceProtocol?: number } = {}) { +async function setup( + options: { sourceProtocol?: number; wrap?: (database: RelayDatabase) => RelayDatabase } = {} +) { let clock = 1_000_000 const database = await openInMemoryRelayDatabase() - const store = new RelayAssignmentStore(database, () => clock, { + const store = new RelayAssignmentStore(options.wrap?.(database) ?? database, () => clock, { requireLiveCells: true, heartbeatTtlMs: 45_000 }) @@ -1397,3 +1688,43 @@ async function heartbeat( } }) } + +class CellInventoryLockProbe { + readonly locks: (RelayLockOptions | undefined)[] = [] + failNoWait = false + // Contends one candidate only, so the sweep must carry on to the next. + failNoWaitOnce = false + failWith: Error | null = null + + reset(): void { + this.locks.length = 0 + } + + wrap(database: RelayDatabase): RelayDatabase { + const probe = this + const decorate = (delegate: RelayDatabase): RelayDatabase => ({ + query: async (sql, params) => await delegate.query(sql, params), + queryLocked: async (sql, params, options) => { + if (sql.trim() === 'SELECT * FROM relay_cells ORDER BY cell_id ASC') { + probe.locks.push(options) + if (probe.failWith) throw probe.failWith + if (options?.failIfUnavailable && probe.failNoWaitOnce) { + probe.failNoWaitOnce = false + throw new Error('database_lock_unavailable') + } + if (probe.failNoWait && options?.failIfUnavailable) { + throw new Error('database_lock_unavailable') + } + } + return await delegate.queryLocked(sql, params, options) + }, + transaction: async (operation, options) => + await delegate.transaction( + async (transaction) => await operation(decorate(transaction)), + options + ), + close: async () => undefined + }) + return decorate(database) + } +} diff --git a/cloud/apps/relay/src/regional-rehome-worker.ts b/cloud/apps/relay/src/regional-rehome-worker.ts index 97c63a61025..47a2748cff4 100644 --- a/cloud/apps/relay/src/regional-rehome-worker.ts +++ b/cloud/apps/relay/src/regional-rehome-worker.ts @@ -3,6 +3,7 @@ import type { RelayAssignmentStore } from './assignment-store.js' import type { RelayConfig } from './config.js' import { googleMetadataIdentityToken } from './google-metadata-identity-token.js' import type { RegionalRehomeSafetySnapshot } from './relay-observability.js' +import { jitteredSweepIntervalMs } from './relay-sweep-schedule.js' type RegionalRehomeWorkerOptions = { fetch?: typeof fetch @@ -10,6 +11,7 @@ type RegionalRehomeWorkerOptions = { now?: () => number intervalMs?: number requestTimeoutMs?: number + random?: () => number safetySnapshot?: () => RegionalRehomeSafetySnapshot } @@ -109,7 +111,10 @@ export function startRegionalRehomeWorker( inFlight = false } } - const timer = setInterval(() => void run(), options.intervalMs ?? 1_000) + const timer = setInterval( + () => void run(), + options.intervalMs ?? jitteredSweepIntervalMs(1_000, options.random) + ) timer.unref() void run() return { diff --git a/cloud/apps/relay/src/relay-observability.ts b/cloud/apps/relay/src/relay-observability.ts index 6125ede8d1a..2266217d607 100644 --- a/cloud/apps/relay/src/relay-observability.ts +++ b/cloud/apps/relay/src/relay-observability.ts @@ -1,6 +1,7 @@ import { monitorEventLoopDelay, performance } from 'node:perf_hooks' import type { RelayRegion } from '@orca-cloud/relay-contract' import type { ControlRenewalOutcome } from './assignment-store.js' +import type { CellInventoryHoldCounts } from './cell-inventory-hold-samples.js' import type { PostgresPoolPressureCounts } from './postgres-pool-pressure.js' import type { RelayReadinessObservation } from './relay-readiness.js' @@ -20,7 +21,9 @@ export function observedRelayRequests(counts: RelayRuntimeCounts): number { return counts.preAuthConnections + counts.controls + counts.splices + counts.pendingSplices } -export type RelayProcessCounts = RelayRuntimeCounts & PostgresPoolPressureCounts +export type RelayProcessCounts = RelayRuntimeCounts & + PostgresPoolPressureCounts & + Partial export type RegionalRehomeRuntimeSafety = { observedAt: number diff --git a/cloud/apps/relay/src/relay-sweep-schedule.test.ts b/cloud/apps/relay/src/relay-sweep-schedule.test.ts new file mode 100644 index 00000000000..d5ef450cc43 --- /dev/null +++ b/cloud/apps/relay/src/relay-sweep-schedule.test.ts @@ -0,0 +1,55 @@ +import { readFileSync } from 'node:fs' +import { describe, expect, it, vi } from 'vitest' +import { startRegionalRehomeWorker } from './regional-rehome-worker.js' +import { jitteredSweepIntervalMs, SWEEP_JITTER_FRACTION } from './relay-sweep-schedule.js' + +describe('sweep schedule jitter', () => { + it('spreads instances across a bounded window above the base period', () => { + expect(jitteredSweepIntervalMs(30_000, () => 0)).toBe(30_000) + expect(jitteredSweepIntervalMs(30_000, () => 0.5)).toBe(33_000) + // Math.random() never returns 1, so the open bound is the real ceiling. + expect(jitteredSweepIntervalMs(30_000, () => 0.999)).toBeLessThan(36_000) + }) + + // Why: a shorter period would raise the very lock traffic the offset spreads. + it('never schedules a sweep sooner than its base period', () => { + for (const random of [0, 0.25, 0.5, 0.75, 0.999]) { + expect(jitteredSweepIntervalMs(1_000, () => random)).toBeGreaterThanOrEqual(1_000) + } + expect(SWEEP_JITTER_FRACTION).toBeGreaterThan(0) + }) + + it('jitters the regional rehome dispatch tick, which every director runs each second', () => { + const timers: number[] = [] + const setIntervalSpy = vi + .spyOn(globalThis, 'setInterval') + .mockImplementation(((_handler: unknown, delayMs?: number) => { + timers.push(delayMs ?? 0) + return { unref: () => undefined, [Symbol.dispose]: () => undefined } as never + }) as never) + + try { + startRegionalRehomeWorker( + { + role: 'director', + rehomeAudience: 'https://rehome.example.test', + rehomeDirectorServiceAccount: 'rehome@example.test' + } as never, + { claimRegionalRehome: async () => null } as never, + { random: () => 0.5, safetySnapshot: () => ({}) as never } + ) + } finally { + setIntervalSpy.mockRestore() + } + + expect(timers).toEqual([1_100]) + }) + + // Why: index.ts boots a server on import, so its wiring can only be read. + it('jitters the director assignment cleanup tick', () => { + const source = readFileSync(new URL('./index.ts', import.meta.url), 'utf8') + const cleanup = /runAssignmentCleanup\(assignments\)\s*\},\s*([^\n]*?)\)\n/.exec(source) + + expect(cleanup?.[1]).toBe('jitteredSweepIntervalMs(30_000)') + }) +}) diff --git a/cloud/apps/relay/src/relay-sweep-schedule.ts b/cloud/apps/relay/src/relay-sweep-schedule.ts new file mode 100644 index 00000000000..f73e6b69ead --- /dev/null +++ b/cloud/apps/relay/src/relay-sweep-schedule.ts @@ -0,0 +1,13 @@ +// Why: every director instance boots from the same rollout, so its periodic +// sweeps land on the same wall-clock second across instances and pile onto the +// one global cell-inventory lock together. A per-process offset spreads the +// arrivals; the sweeps are idempotent, so a slightly longer period is free. +export const SWEEP_JITTER_FRACTION = 0.2 + +export function jitteredSweepIntervalMs( + baseMs: number, + random: () => number = Math.random +): number { + // Only ever longer: a shorter period would raise the very load being spread. + return baseMs + Math.floor(random() * baseMs * SWEEP_JITTER_FRACTION) +} From 53adf5e2e630fa47002e70f7d909117fec57a2b4 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:42:54 -0700 Subject: [PATCH 11/76] fix(git): share one failed-command error-text reader between local and the SSH relay (#18398) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(git): share one error-text reader between the local and relay branch-delete fallbacks The relay and the desktop each carried their own `getErrorText`, and they had drifted: the relay read `message` + `stderr` + `stdout`, the desktop only `message` + `stderr`. A `git branch -d` refusal arriving on `stdout` therefore routed the SSH removal through prune-and-retry while the local removal gave up and preserved the branch. Against a real binary the two agree, because Git prints the refusal through `error()` on every supported version — verified on 2.25.1, 2.38.1, 2.49.1 and 2.55.0, none of which put a byte of it on stdout. What the desktop copy actually missed is that Orca classifies errors it built itself, with the Git output on `.stdout`: `worktree remove`'s submodule retry attaches `git status --porcelain` that way on both paths. The stdout-reading form is also already the shared spelling — `isSubmoduleWorktreeRemovalRefusal` uses it for both hosts — so this converges on it rather than on the shorter one. Move the reader to src/shared/git-command-failure-text.ts and the predicate it feeds to src/shared/git-branch-delete-refusal.ts, and delete all three copies. The predicate carries both refusal wordings live in the supported range: Git through 2.40 says "checked out at", 2.43+ says "used by worktree at". The real-binary contract now pins that boundary: the refusal is recognized, it lands on stderr, and stdout stays empty on every Git in the matrix. * fix(test): consolidate the duplicate worktree import in the parity test --- src/main/git/worktree-branch-removal.ts | 7 +- src/main/git/worktree-operation-options.ts | 23 +- .../git-branch-delete-refusal-parity.test.ts | 205 ++++++++++++++++++ src/relay/git-handler-worktree-remove.ts | 24 +- src/shared/git-binary-compatibility.test.ts | 24 ++ src/shared/git-branch-delete-refusal.ts | 16 ++ src/shared/git-command-failure-text.ts | 27 +++ src/shared/worktree/submodule-removal.ts | 18 +- 8 files changed, 281 insertions(+), 63 deletions(-) create mode 100644 src/relay/git-branch-delete-refusal-parity.test.ts create mode 100644 src/shared/git-branch-delete-refusal.ts create mode 100644 src/shared/git-command-failure-text.ts diff --git a/src/main/git/worktree-branch-removal.ts b/src/main/git/worktree-branch-removal.ts index 35385254bca..dc09311596c 100644 --- a/src/main/git/worktree-branch-removal.ts +++ b/src/main/git/worktree-branch-removal.ts @@ -7,12 +7,9 @@ import { withLocalGitCapabilityCacheForExecution } from './git-capability-state' import { withRepoRefMaintenancePaused } from './local-repo-ref-maintenance' import { gitExecFileAsync } from './runner' import { parseWorktreeList } from '../../shared/git-worktree-porcelain-parser' +import { isBranchCheckedOutInWorktreeError } from '../../shared/git-branch-delete-refusal' import type { GitWorktreeExecOptions, RemoveWorktreeOptions } from './worktree-operation-options' -import { - gitExecOptions, - isBranchCheckedOutInWorktreeError, - normalizeLocalBranchRef -} from './worktree-operation-options' +import { gitExecOptions, normalizeLocalBranchRef } from './worktree-operation-options' export async function deleteBranchAfterWorktreeRemoval( repoPath: string, diff --git a/src/main/git/worktree-operation-options.ts b/src/main/git/worktree-operation-options.ts index 9376fe63f85..6f956c6c422 100644 --- a/src/main/git/worktree-operation-options.ts +++ b/src/main/git/worktree-operation-options.ts @@ -2,6 +2,7 @@ import type { LocalBaseRefRefreshResult, LocalBaseRefUpdateSuggestion } from '../../shared/worktree/base-ref-drift-types' +import { readGitCommandFailureText } from '../../shared/git-command-failure-text' import type { RemoveWorktreeResult } from '../../shared/worktree/create-types' import type { GitWorktreeInfo } from '../../shared/worktree/types' @@ -95,28 +96,8 @@ export function getErrorCode(error: unknown): string | undefined { : undefined } -function getErrorText(error: unknown): string { - if (typeof error === 'object' && error !== null) { - const parts: string[] = [] - if ('message' in error && typeof error.message === 'string') { - parts.push(error.message) - } - if ('stderr' in error && typeof error.stderr === 'string') { - parts.push(error.stderr) - } - return parts.join('\n') - } - return String(error) -} - export function isNotGitRepositoryError(error: unknown): boolean { - return /not a git repository/i.test(getErrorText(error)) -} - -export function isBranchCheckedOutInWorktreeError(error: unknown): boolean { - return /cannot delete branch .*(?:used by worktree|checked out)|branch .*is checked out/i.test( - getErrorText(error) - ) + return /not a git repository/i.test(readGitCommandFailureText(error)) } export function normalizeLocalBranchRef(branch: string): string { diff --git a/src/relay/git-branch-delete-refusal-parity.test.ts b/src/relay/git-branch-delete-refusal-parity.test.ts new file mode 100644 index 00000000000..71344bca940 --- /dev/null +++ b/src/relay/git-branch-delete-refusal-parity.test.ts @@ -0,0 +1,205 @@ +/** + * The relay and the desktop each carried their own `getErrorText`, and they had + * drifted: the relay read `message` + `stderr` + `stdout`, the desktop only + * `message` + `stderr`. So a `git branch -d` refusal that arrived on `stdout` + * routed the SSH removal through prune-and-retry while the local removal gave up + * and preserved the branch. + * + * These tests push the same failure through both published removal entry points — + * `removeWorktreeOp` (what `git.removeWorktree` runs on the host) and `removeWorktree` + * (the local runner) — and require the same branch-deletion commands and the same + * `RemoveWorktreeResult`. A second error-text reader on either side fails here. + */ +import type * as FsPromises from 'node:fs/promises' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { gitExecFileAsyncMock, resolveGitDirMock, moveWorktreeDirectoryToTrashMock } = vi.hoisted( + () => ({ + gitExecFileAsyncMock: vi.fn(), + resolveGitDirMock: vi.fn(), + moveWorktreeDirectoryToTrashMock: vi.fn() + }) +) + +vi.mock('../main/worktree-trash', () => ({ + moveWorktreeDirectoryToTrash: moveWorktreeDirectoryToTrashMock, + restoreWorktreeDirectoryFromTrash: vi.fn(async () => true), + scheduleWorktreeTrashDeletion: vi.fn() +})) + +vi.mock('../main/git/runner', () => ({ + gitExecFileAsync: gitExecFileAsyncMock, + gitExecFileSync: vi.fn(), + translateWslOutputPaths: (output: string) => output +})) + +vi.mock('../main/git/status', () => ({ + resolveGitDir: resolveGitDirMock, + runWithGitReadCacheInvalidation: (run: () => Promise) => run() +})) + +vi.mock('fs/promises', async () => { + const actual = await vi.importActual('fs/promises') + return { + ...actual, + stat: vi.fn(async () => { + throw enoent() + }), + readFile: vi.fn() + } +}) + +import { GitCapabilityCache } from '../shared/git-capability-cache' +import type { RemoveWorktreeResult } from '../shared/worktree/create-types' +import { clearGitCapabilityStateForTests } from '../main/git/git-capability-state' +import { _resetWorktreeScanCacheForTests, removeWorktree } from '../main/git/worktree' +import { __resetSparseCheckoutStateCacheForTests } from '../main/git/worktree-sparse-checkout-cache' +import type { GitExec } from './git-handler-ops' +import { removeWorktreeOp } from './git-handler-worktree-ops' + +const REPO_PATH = '/repo' +const WORKTREE_PATH = '/repo-feature' +const BRANCH = 'feature/test' + +function enoent(): Error { + return Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) +} + +/** Only the branch-deletion phase; the two entry points legitimately reach it by different routes. */ +function branchDeletionCalls(calls: string[][]): string[] { + return calls + .map((args) => args.join(' ')) + .filter((call) => call.startsWith('branch ') || call === 'worktree prune') +} + +function worktreeListPorcelain(withFeature: boolean): string { + const blocks = [[`worktree ${REPO_PATH}`, 'HEAD abc123', 'branch refs/heads/main']] + if (withFeature) { + blocks.push([`worktree ${WORKTREE_PATH}`, 'HEAD def456', `branch refs/heads/${BRANCH}`]) + } + return `${blocks.map((block) => block.join('\n')).join('\n\n')}\n` +} + +type RefusalStream = 'stdout' | 'stderr' + +const REFUSAL_TEXT = `error: cannot delete branch '${BRANCH}' used by worktree at '/repo-stale'` + +/** + * A `branch -d` rejection carrying the refusal on exactly one stream. `message` stays + * generic so the assertion is about the stream, not about Node's stderr echo. + */ +function branchDeleteRefusal(stream: RefusalStream): Error { + return Object.assign(new Error('Command failed: git branch -d'), { + code: 1, + stdout: stream === 'stdout' ? REFUSAL_TEXT : '', + stderr: stream === 'stderr' ? REFUSAL_TEXT : '' + }) +} + +/** Refuses the first `branch -d`, accepts the retry that follows `worktree prune`. */ +function scriptRelayGit(stream: RefusalStream): { + git: GitExec + calls: string[][] +} { + const calls: string[][] = [] + let branchDeleteCount = 0 + const git = vi.fn(async (args) => { + calls.push(args) + if (args[0] === 'rev-parse') { + return { stdout: `${REPO_PATH}/.git\n`, stderr: '' } + } + if (args[0] === 'worktree' && args[1] === 'list') { + return { stdout: worktreeListPorcelain(true), stderr: '' } + } + if (args[0] === 'branch' && args[1] === '-d') { + branchDeleteCount += 1 + if (branchDeleteCount === 1) { + throw branchDeleteRefusal(stream) + } + return { stdout: '', stderr: '' } + } + return { stdout: '', stderr: '' } + }) + return { git, calls } +} + +function scriptDesktopGit(stream: RefusalStream): string[][] { + const calls: string[][] = [] + let branchDeleteCount = 0 + gitExecFileAsyncMock.mockImplementation(async (args: string[]) => { + calls.push(args) + if (args[0] === 'worktree' && args[1] === 'list') { + return { stdout: worktreeListPorcelain(branchDeleteCount === 0), stderr: '' } + } + if (args[0] === 'branch' && args[1] === '-d') { + branchDeleteCount += 1 + if (branchDeleteCount === 1) { + throw branchDeleteRefusal(stream) + } + return { stdout: '', stderr: '' } + } + return { stdout: '', stderr: '' } + }) + return calls +} + +async function removeOverRelay( + stream: RefusalStream +): Promise<{ result: RemoveWorktreeResult; branchCalls: string[] }> { + const { git, calls } = scriptRelayGit(stream) + const result = await removeWorktreeOp( + git, + { worktreePath: WORKTREE_PATH }, + new GitCapabilityCache() + ) + return { result, branchCalls: branchDeletionCalls(calls) } +} + +async function removeLocally( + stream: RefusalStream +): Promise<{ result: RemoveWorktreeResult; branchCalls: string[] }> { + const calls = scriptDesktopGit(stream) + const result = await removeWorktree(REPO_PATH, WORKTREE_PATH) + return { result, branchCalls: branchDeletionCalls(calls) } +} + +beforeEach(() => { + clearGitCapabilityStateForTests() + _resetWorktreeScanCacheForTests() + __resetSparseCheckoutStateCacheForTests() + gitExecFileAsyncMock.mockReset() + resolveGitDirMock.mockReset() + resolveGitDirMock.mockImplementation(async (worktreePath: string) => `${worktreePath}/.git`) + moveWorktreeDirectoryToTrashMock.mockReset() + // Default: the checkout cannot be renamed aside, so removal runs `worktree remove` in place. + moveWorktreeDirectoryToTrashMock.mockResolvedValue(undefined) +}) + +describe('relay/desktop branch-delete refusal parity', () => { + it('prunes and retries on both paths when the refusal arrives on stdout', async () => { + const relay = await removeOverRelay('stdout') + const local = await removeLocally('stdout') + + expect(relay.branchCalls).toEqual(local.branchCalls) + expect(relay.result).toEqual(local.result) + expect(local.branchCalls).toEqual([ + `branch -d -- ${BRANCH}`, + 'worktree prune', + `branch -d -- ${BRANCH}` + ]) + expect(local.result).toEqual({}) + }) + + it('prunes and retries on both paths when the refusal arrives on stderr, as real Git sends it', async () => { + const relay = await removeOverRelay('stderr') + const local = await removeLocally('stderr') + + expect(relay.branchCalls).toEqual(local.branchCalls) + expect(relay.result).toEqual(local.result) + expect(local.branchCalls).toEqual([ + `branch -d -- ${BRANCH}`, + 'worktree prune', + `branch -d -- ${BRANCH}` + ]) + }) +}) diff --git a/src/relay/git-handler-worktree-remove.ts b/src/relay/git-handler-worktree-remove.ts index 474e03bab88..bf8e65a066e 100644 --- a/src/relay/git-handler-worktree-remove.ts +++ b/src/relay/git-handler-worktree-remove.ts @@ -1,5 +1,6 @@ import * as path from 'node:path' import type { RemoveWorktreeResult } from '../shared/worktree/create-types' +import { isBranchCheckedOutInWorktreeError } from '../shared/git-branch-delete-refusal' import { assertWorktreeUnlockedForRemoval } from '../shared/worktree/removal' import { isSubmoduleWorktreeRemovalRefusal } from '../shared/worktree/submodule-removal' import { deleteAlreadyMergedRelayBranchAfterSafeDeleteFailure } from './git-handler-branch-cleanup' @@ -7,29 +8,6 @@ import type { GitExec } from './git-handler-ops' import type { GitCapabilityCache } from '../shared/git-capability-cache' import { readRelayWorktreeList } from './git-handler-worktree-list' -function getErrorText(error: unknown): string { - if (typeof error === 'object' && error !== null) { - const parts: string[] = [] - if ('message' in error && typeof error.message === 'string') { - parts.push(error.message) - } - if ('stderr' in error && typeof error.stderr === 'string') { - parts.push(error.stderr) - } - if ('stdout' in error && typeof error.stdout === 'string') { - parts.push(error.stdout) - } - return parts.join('\n') - } - return String(error) -} - -function isBranchCheckedOutInWorktreeError(error: unknown): boolean { - return /cannot delete branch .*(?:used by worktree|checked out)|branch .*is checked out/i.test( - getErrorText(error) - ) -} - function normalizeLocalBranchRef(branch: string): string { return branch.replace(/^refs\/heads\//, '') } diff --git a/src/shared/git-binary-compatibility.test.ts b/src/shared/git-binary-compatibility.test.ts index 5ad37398d14..6387f200b4c 100644 --- a/src/shared/git-binary-compatibility.test.ts +++ b/src/shared/git-binary-compatibility.test.ts @@ -8,6 +8,7 @@ import { isUnsupportedMergeTreeMergeBaseError, isUnsupportedMergeTreeWriteTreeError } from './git-merge-tree-capability' +import { isBranchCheckedOutInWorktreeError } from './git-branch-delete-refusal' import { isForEachRefExcludeUnsupportedError } from './git-ref-command-capabilities' import { isNoWriteFetchHeadUnsupportedError } from './git-fetch-head-capability' import { @@ -140,6 +141,29 @@ describeBinaryCompatibility('real Git binary compatibility', () => { ).resolves.toBeDefined() }) + // Why pin this: worktree removal decides whether to prune and retry `branch -d` by + // matching Git's refusal text, and the wording moved inside the supported range + // (<=2.40 "Cannot delete branch 'x' checked out at", >=2.43 "cannot delete branch 'x' + // used by worktree at"). It is also the only evidence that the refusal is a stderr + // message on every supported Git rather than something a caller could read off stdout. + it('refuses to delete a branch another worktree holds, on stderr, in a recognized wording', async () => { + await runGit(['worktree', 'add', '-b', 'compat-held', 'held-wt']) + try { + const refusal = await runGit(['branch', '-d', '--', 'compat-held']).then( + () => null, + (error: unknown) => error + ) + expect(refusal).not.toBeNull() + expect(isBranchCheckedOutInWorktreeError(refusal)).toBe(true) + const streams = refusal as { stdout?: string; stderr?: string } + expect(streams.stderr ?? '').toMatch(/delete branch .*compat-held/i) + expect(streams.stdout ?? '').toBe('') + } finally { + await runGit(['worktree', 'remove', '--force', 'held-wt']) + await runGit(['branch', '-D', 'compat-held']) + } + }) + it('deregisters a worktree whose directory was renamed away', async () => { // Orca renames the checkout into a trash directory and then clears the registration, so every // supported Git must accept `worktree remove --force` on the now-missing path. diff --git a/src/shared/git-branch-delete-refusal.ts b/src/shared/git-branch-delete-refusal.ts new file mode 100644 index 00000000000..30d03746c2d --- /dev/null +++ b/src/shared/git-branch-delete-refusal.ts @@ -0,0 +1,16 @@ +import { readGitCommandFailureText } from './git-command-failure-text' + +/** + * `git branch -d/-D` refused because the branch is the HEAD of some worktree. + * + * Both wordings are live in Orca's supported range: Git through 2.40 says + * "Cannot delete branch 'x' checked out at ''", and 2.43+ says "cannot delete + * branch 'x' used by worktree at ''". Every version prints it through `error()`, + * so it arrives on stderr. Callers treat a match as "the blocker may be a stale + * worktree record", prune, and retry once. + */ +export function isBranchCheckedOutInWorktreeError(error: unknown): boolean { + return /cannot delete branch .*(?:used by worktree|checked out)|branch .*is checked out/i.test( + readGitCommandFailureText(error) + ) +} diff --git a/src/shared/git-command-failure-text.ts b/src/shared/git-command-failure-text.ts new file mode 100644 index 00000000000..1ebfd322cfe --- /dev/null +++ b/src/shared/git-command-failure-text.ts @@ -0,0 +1,27 @@ +/** + * The text a failed Git invocation left behind, for the predicates that classify a + * failure by what Git said. + * + * Why all three streams and not just `message` + `stderr`: the errors Orca classifies + * do not all come straight out of `execFile`. Node puts Git's stderr in both `message` + * and `stderr`, but Orca also throws its own failures with the Git output on `stdout` + * (`worktree remove`'s submodule retry attaches `git status --porcelain` output that + * way on both the local runner and the relay). Reading all three is what keeps the + * local and relay classifiers from disagreeing about the same error object. + * + * Against a real binary this reads no differently: Git emits every refusal this module + * classifies through `error()`/`die()`, i.e. stderr only, on 2.25 through 2.55. + */ +export function readGitCommandFailureText(error: unknown): string { + if (typeof error !== 'object' || error === null) { + return String(error) + } + const parts: string[] = [] + for (const field of ['message', 'stderr', 'stdout'] as const) { + const value = (error as Record)[field] + if (typeof value === 'string' && value) { + parts.push(value) + } + } + return parts.join('\n') +} diff --git a/src/shared/worktree/submodule-removal.ts b/src/shared/worktree/submodule-removal.ts index 8a6f91a2cf8..2b9306c4650 100644 --- a/src/shared/worktree/submodule-removal.ts +++ b/src/shared/worktree/submodule-removal.ts @@ -1,16 +1,4 @@ -function getErrorText(error: unknown): string { - if (typeof error === 'object' && error !== null) { - const parts: string[] = [] - for (const field of ['message', 'stderr', 'stdout'] as const) { - const value = (error as Record)[field] - if (typeof value === 'string' && value) { - parts.push(value) - } - } - return parts.join('\n') - } - return String(error) -} +import { readGitCommandFailureText } from '../git-command-failure-text' // Why: `git worktree remove` (non-force) categorically refuses any worktree // containing an initialised submodule, even when parent and submodule are @@ -18,5 +6,7 @@ function getErrorText(error: unknown): string { // cleanliness and retry with --force. Both the local runner and the relay pin // English git output (UNTRANSLATED_GIT_OUTPUT_ENV), so text matching is stable. export function isSubmoduleWorktreeRemovalRefusal(error: unknown): boolean { - return /working trees containing submodules cannot be moved or removed/i.test(getErrorText(error)) + return /working trees containing submodules cannot be moved or removed/i.test( + readGitCommandFailureText(error) + ) } From 5a626dcdf48e723962f268c8f7adca77c4fd5a32 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:42:58 -0700 Subject: [PATCH 12/76] refactor(git): share push-target resolution between local and the SSH relay (#18406) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `src/relay/git-handler-push-target.ts` and `src/main/git/remote.ts` carried identical ~160-line copies of the resolver that decides which remote a plain `git push` hits. Identical today is exactly when to share it: the cost of a future divergence is pushing to the wrong remote, which retrying does not undo. Move the resolver to src/shared/git-push-target-resolution.ts, parameterized on a `(args) => Promise<{ stdout }>` runner — the only thing the two hosts actually differ in — and delete both copies. The relay entry point keeps only the work that is genuinely relay-side: re-validating an explicit target that arrived over the wire and running `check-ref-format` on it. No behavior change on either path, and nothing new or different is published, so this engages no rule in remote-wire-compatibility. No git command changes. src/relay/git-push-target-local-parity.test.ts scripts one repository's config and requires `git.push` over the real relay dispatcher and the desktop's `gitPush` to emit the same push argv, plus the argv each case should produce. --- src/main/git/remote.ts | 158 +------------ src/relay/git-handler-push-target.ts | 160 +------------- .../git-push-target-local-parity.test.ts | 209 ++++++++++++++++++ src/shared/git-push-target-resolution.ts | 140 ++++++++++++ 4 files changed, 361 insertions(+), 306 deletions(-) create mode 100644 src/relay/git-push-target-local-parity.test.ts create mode 100644 src/shared/git-push-target-resolution.ts diff --git a/src/main/git/remote.ts b/src/main/git/remote.ts index 20cf8415d04..aa7932687ca 100644 --- a/src/main/git/remote.ts +++ b/src/main/git/remote.ts @@ -3,8 +3,7 @@ import { runPullWithDivergenceFallback } from '../../shared/git-remote-error' import { resolveEffectiveGitUpstream } from '../../shared/git-effective-upstream' -import { gitRefTargetsBranchOnRemote } from '../../shared/git-remote-branch-name' -import { findGitRemoteNameByFetchUrl } from '../../shared/git-remote-url-index' +import { resolveConfiguredGitPushTarget } from '../../shared/git-push-target-resolution' import type { GitPushTarget } from '../../shared/worktree/types' import type { GitRuntimeOptions } from './git-runtime-options' import { gitOptionsForWorktree } from './git-runtime-options' @@ -20,157 +19,6 @@ import { runWithGitWorktreeOperationLock } from '../../shared/git-worktree-opera export { gitPullRebaseFromBase } from './remote-rebase' -async function getConfiguredPushTarget( - worktreePath: string, - options: GitRuntimeOptions = {} -): Promise<{ remote: string; refspec: string } | null> { - try { - const { stdout: branchStdout } = await gitExecFileAsync( - ['symbolic-ref', '--quiet', '--short', 'HEAD'], - gitOptionsForWorktree(worktreePath, options) - ) - const branch = branchStdout.trim() - if (!branch) { - return null - } - - const [pushRemote, { stdout: mergeStdout }] = await Promise.all([ - getConfiguredPushRemote(worktreePath, branch, options), - gitExecFileAsync( - ['config', '--get', `branch.${branch}.merge`], - gitOptionsForWorktree(worktreePath, options) - ) - ]) - const remote = pushRemote?.remote - const mergeRef = mergeStdout.trim() - const branchRef = mergeRef.replace(/^refs\/heads\//, '') - if (!remote || !branchRef || remote === '.' || branchRef === mergeRef) { - return null - } - if (await branchMergeTargetsConfiguredBase(worktreePath, branch, remote, branchRef, options)) { - return null - } - if (!canPushConfiguredMergeBranch(pushRemote, branch, branchRef)) { - return null - } - return { remote, refspec: `HEAD:${branchRef}` } - } catch { - return null - } -} - -async function getConfigValue( - worktreePath: string, - key: string, - options: GitRuntimeOptions = {} -): Promise { - try { - const { stdout } = await gitExecFileAsync( - ['config', '--get', key], - gitOptionsForWorktree(worktreePath, options) - ) - const value = stdout.trim() - return value || null - } catch { - return null - } -} - -function isUrlValuedRemote(remote: string): boolean { - return /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(remote) || /^[^@/:]+@[^:]+:.+/.test(remote) -} - -type ConfiguredPushRemote = { - remote: string - branchRemote: string | null -} - -// One `git remote -v` instead of `git remote` plus a serial `git remote get-url` -// per remote; both print the same insteadOf-expanded fetch URL. -async function findRemoteNameForUrl( - worktreePath: string, - remoteUrl: string, - options: GitRuntimeOptions = {} -): Promise { - try { - const { stdout } = await gitExecFileAsync( - ['remote', '-v'], - gitOptionsForWorktree(worktreePath, options) - ) - return findGitRemoteNameByFetchUrl(stdout, (candidateUrl) => candidateUrl === remoteUrl) - } catch { - return null - } -} - -async function normalizePushRemote( - worktreePath: string, - remote: string, - options: GitRuntimeOptions = {} -): Promise { - if (!isUrlValuedRemote(remote)) { - return remote - } - return (await findRemoteNameForUrl(worktreePath, remote, options)) ?? remote -} - -async function getConfiguredPushRemote( - worktreePath: string, - branch: string, - options: GitRuntimeOptions = {} -): Promise { - const branchRemote = await getConfigValue(worktreePath, `branch.${branch}.remote`, options) - const remote = - (await getConfigValue(worktreePath, `branch.${branch}.pushRemote`, options)) ?? - (await getConfigValue(worktreePath, 'remote.pushDefault', options)) ?? - branchRemote - if (!remote) { - return null - } - const normalizedRemote = await normalizePushRemote(worktreePath, remote, options) - // The two usually name the same URL; resolving it twice reads the remote table twice. - if (!branchRemote) { - return { remote: normalizedRemote, branchRemote: null } - } - return { - remote: normalizedRemote, - branchRemote: - branchRemote === remote - ? normalizedRemote - : await normalizePushRemote(worktreePath, branchRemote, options) - } -} - -async function branchMergeTargetsConfiguredBase( - worktreePath: string, - branch: string, - remote: string, - branchRef: string, - options: GitRuntimeOptions = {} -): Promise { - return gitRefTargetsBranchOnRemote( - await getConfigValue(worktreePath, `branch.${branch}.base`, options), - remote, - branchRef - ) -} - -function canPushConfiguredMergeBranch( - pushRemote: ConfiguredPushRemote | null, - branch: string, - branchRef: string -): boolean { - if (!pushRemote) { - return false - } - if (branchRef === branch) { - return true - } - // Why: branch.merge belongs to branch.remote. A pushDefault fork must not - // inherit origin/main as its destination branch. - return pushRemote.remote !== 'origin' && pushRemote.branchRemote === pushRemote.remote -} - function explicitPushTarget(target: GitPushTarget): { remote: string; refspec: string } { return { remote: target.remoteName, refspec: `HEAD:${target.branchName}` } } @@ -197,7 +45,9 @@ export async function gitPush( // from worktree config, not the upstream relationship. const target = pushTarget ? explicitPushTarget(pushTarget) - : await getConfiguredPushTarget(worktreePath, options) + : await resolveConfiguredGitPushTarget((args) => + gitExecFileAsync(args, gitOptionsForWorktree(worktreePath, options)) + ) const args = [ 'push', ...(options.forceWithLease ? ['--force-with-lease'] : []), diff --git a/src/relay/git-handler-push-target.ts b/src/relay/git-handler-push-target.ts index d81111d45d3..6663b5b3ad3 100644 --- a/src/relay/git-handler-push-target.ts +++ b/src/relay/git-handler-push-target.ts @@ -1,168 +1,24 @@ import { assertGitPushTargetShape } from '../shared/git-push-target-validation' -import { gitRefTargetsBranchOnRemote } from '../shared/git-remote-branch-name' -import { findGitRemoteNameByFetchUrl } from '../shared/git-remote-url-index' +import { + resolveConfiguredGitPushTarget, + type ResolvedGitPushTarget +} from '../shared/git-push-target-resolution' import type { GitPushTarget } from '../shared/worktree/types' type RelayGit = (args: string[], cwd: string) => Promise<{ stdout: string; stderr: string }> -export type ResolvedPushTarget = { - remote: string - refspec: string -} - -async function getConfiguredPushTarget( - git: RelayGit, - worktreePath: string -): Promise { - try { - const { stdout: branchStdout } = await git( - ['symbolic-ref', '--quiet', '--short', 'HEAD'], - worktreePath - ) - const branch = branchStdout.trim() - if (!branch) { - return null - } - const [pushRemote, { stdout: mergeStdout }] = await Promise.all([ - getConfiguredPushRemote(git, worktreePath, branch), - git(['config', '--get', `branch.${branch}.merge`], worktreePath) - ]) - const remote = pushRemote?.remote - const mergeRef = mergeStdout.trim() - const branchRef = mergeRef.replace(/^refs\/heads\//, '') - if (!remote || !branchRef || remote === '.' || branchRef === mergeRef) { - return null - } - if (await branchMergeTargetsConfiguredBase(git, worktreePath, branch, remote, branchRef)) { - return null - } - if (!canPushConfiguredMergeBranch(pushRemote, branch, branchRef)) { - return null - } - return { remote, refspec: `HEAD:${branchRef}` } - } catch { - return null - } -} - -async function getConfigValue( - git: RelayGit, - worktreePath: string, - key: string -): Promise { - try { - const { stdout } = await git(['config', '--get', key], worktreePath) - const value = stdout.trim() - return value || null - } catch { - return null - } -} - -function isUrlValuedRemote(remote: string): boolean { - return /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(remote) || /^[^@/:]+@[^:]+:.+/.test(remote) -} - -type ConfiguredPushRemote = { - remote: string - branchRemote: string | null -} - -// Host-side twin of `src/main/git/remote.ts`: one `git remote -v` instead of -// `git remote` plus a serial `git remote get-url` per remote. -async function findRemoteNameForUrl( - git: RelayGit, - worktreePath: string, - remoteUrl: string -): Promise { - try { - const { stdout } = await git(['remote', '-v'], worktreePath) - return findGitRemoteNameByFetchUrl(stdout, (candidateUrl) => candidateUrl === remoteUrl) - } catch { - return null - } -} - -async function normalizePushRemote( - git: RelayGit, - worktreePath: string, - remote: string -): Promise { - if (!isUrlValuedRemote(remote)) { - return remote - } - return (await findRemoteNameForUrl(git, worktreePath, remote)) ?? remote -} - -async function getConfiguredPushRemote( - git: RelayGit, - worktreePath: string, - branch: string -): Promise { - // Why: mirror the local gitPush resolver so SSH worktrees do not drift to a - // different target when branch.pushRemote or remote.pushDefault is present. - const branchRemote = await getConfigValue(git, worktreePath, `branch.${branch}.remote`) - const remote = - (await getConfigValue(git, worktreePath, `branch.${branch}.pushRemote`)) ?? - (await getConfigValue(git, worktreePath, 'remote.pushDefault')) ?? - branchRemote - if (!remote) { - return null - } - const normalizedRemote = await normalizePushRemote(git, worktreePath, remote) - // The two usually name the same URL; resolving it twice reads the remote table twice. - if (!branchRemote) { - return { remote: normalizedRemote, branchRemote: null } - } - return { - remote: normalizedRemote, - branchRemote: - branchRemote === remote - ? normalizedRemote - : await normalizePushRemote(git, worktreePath, branchRemote) - } -} - -async function branchMergeTargetsConfiguredBase( - git: RelayGit, - worktreePath: string, - branch: string, - remote: string, - branchRef: string -): Promise { - return gitRefTargetsBranchOnRemote( - await getConfigValue(git, worktreePath, `branch.${branch}.base`), - remote, - branchRef - ) -} - -function canPushConfiguredMergeBranch( - pushRemote: ConfiguredPushRemote | null, - branch: string, - branchRef: string -): boolean { - if (!pushRemote) { - return false - } - if (branchRef === branch) { - return true - } - // Why: branch.merge belongs to branch.remote. A pushDefault fork must not - // inherit origin/main as its destination branch. - return pushRemote.remote !== 'origin' && pushRemote.branchRemote === pushRemote.remote -} - export async function resolveRelayPushTarget( git: RelayGit, worktreePath: string, pushTarget: unknown -): Promise { +): Promise { if (pushTarget === undefined) { - return getConfiguredPushTarget(git, worktreePath) + return resolveConfiguredGitPushTarget((args) => git(args, worktreePath)) } assertGitPushTargetShape(pushTarget) const explicitTarget: GitPushTarget = pushTarget + // Why here and not in the shared resolver: an explicit target arrives over the wire, + // so the host re-validates its shape and asks Git to vet the branch name itself. await git(['check-ref-format', '--branch', explicitTarget.branchName], worktreePath) return { remote: explicitTarget.remoteName, diff --git a/src/relay/git-push-target-local-parity.test.ts b/src/relay/git-push-target-local-parity.test.ts new file mode 100644 index 00000000000..152b062d88e --- /dev/null +++ b/src/relay/git-push-target-local-parity.test.ts @@ -0,0 +1,209 @@ +/** + * Push-target resolution decides which remote a plain `git push` hits, and a wrong + * answer is not recoverable by retrying. The relay and the desktop used to carry + * identical ~160-line copies of it; they now share one implementation. + * + * These tests script one repository's Git config and require `git.push` over the real + * relay dispatcher and the desktop's `gitPush` to emit the *same push argv*, plus the + * argv each case is supposed to produce — so a second implementation on either side + * fails here even if it is wrong in the same direction on both. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { gitExecFileAsyncMock } = vi.hoisted(() => ({ gitExecFileAsyncMock: vi.fn() })) + +vi.mock('../main/git/runner', () => ({ + gitExecFileAsync: gitExecFileAsyncMock +})) + +import { gitPush } from '../main/git/remote' +import { RelayContext } from './context' +import { GitHandler } from './git-handler' +import { createMockDispatcher, type RelayDispatcher } from './git-handler-test-setup' + +const WORKTREE_PATH = '/worktree' + +type GitConfigFixture = { + /** Empty means detached HEAD: `symbolic-ref --quiet --short HEAD` prints nothing. */ + branch: string + merge?: string + branchRemote?: string + pushRemote?: string + pushDefault?: string + base?: string + /** remote name -> fetch URL, as `git remote -v` prints it. */ + remotes?: Record +} + +type GitSpyTarget = { + git(args: string[], cwd: string): Promise<{ stdout: string; stderr: string }> +} + +/** One scripted repository, driven identically by both hosts. */ +function scriptGit(fixture: GitConfigFixture) { + const configValues = new Map() + const put = (key: string, value: string | undefined): void => { + if (value !== undefined) { + configValues.set(key, value) + } + } + put(`branch.${fixture.branch}.merge`, fixture.merge) + put(`branch.${fixture.branch}.remote`, fixture.branchRemote) + put(`branch.${fixture.branch}.pushRemote`, fixture.pushRemote) + put(`branch.${fixture.branch}.base`, fixture.base) + put('remote.pushDefault', fixture.pushDefault) + + const calls: string[][] = [] + return { + calls, + run: async (args: string[]): Promise<{ stdout: string; stderr: string }> => { + calls.push(args) + if (args[0] === 'symbolic-ref') { + return { stdout: `${fixture.branch}\n`, stderr: '' } + } + if (args[0] === 'config' && args[1] === '--get') { + const value = configValues.get(args[2] ?? '') + // Why throw: `git config --get` exits 1 for a missing key, and the resolver's + // fallback chain reads that rejection, not an empty string. + if (value === undefined) { + throw Object.assign(new Error('missing config key'), { code: 1 }) + } + return { stdout: `${value}\n`, stderr: '' } + } + if (args[0] === 'remote' && args[1] === '-v') { + const lines = Object.entries(fixture.remotes ?? {}).flatMap(([name, url]) => [ + `${name}\t${url} (fetch)`, + `${name}\t${url} (push)` + ]) + return { stdout: `${lines.join('\n')}\n`, stderr: '' } + } + if (args[0] === 'push') { + return { stdout: '', stderr: '' } + } + throw new Error(`Unexpected git command: ${args.join(' ')}`) + } + } +} + +function pushArgv(calls: string[][]): string[] { + const push = calls.find((args) => args[0] === 'push') + if (!push) { + throw new Error('no push command was issued') + } + return push +} + +async function pushOverRelay(fixture: GitConfigFixture): Promise { + const dispatcher = createMockDispatcher() + const handler = new GitHandler(dispatcher as unknown as RelayDispatcher, new RelayContext()) + const script = scriptGit(fixture) + vi.spyOn(handler as unknown as GitSpyTarget, 'git').mockImplementation((args) => script.run(args)) + await dispatcher.callRequest('git.push', { worktreePath: WORKTREE_PATH }) + return pushArgv(script.calls) +} + +async function pushLocally(fixture: GitConfigFixture): Promise { + const script = scriptGit(fixture) + gitExecFileAsyncMock.mockImplementation((args: string[]) => script.run(args)) + await gitPush(WORKTREE_PATH) + return pushArgv(script.calls) +} + +async function expectSamePushArgv(fixture: GitConfigFixture, expected: string[]): Promise { + const relayArgv = await pushOverRelay(fixture) + const localArgv = await pushLocally(fixture) + expect(relayArgv).toEqual(localArgv) + expect(localArgv).toEqual(expected) +} + +const FIRST_PUBLISH = ['push', '--set-upstream', 'origin', 'HEAD'] + +beforeEach(() => { + gitExecFileAsyncMock.mockReset() +}) + +describe('relay/desktop push-target parity', () => { + it('sends a review branch to the fork its pushDefault names', async () => { + await expectSamePushArgv( + { + branch: 'review/pr-1738', + merge: 'refs/heads/contributor/fix', + branchRemote: 'fork', + pushDefault: 'fork' + }, + ['push', '--set-upstream', 'fork', 'HEAD:contributor/fix'] + ) + }) + + it('refuses to inherit origin/main as a destination for a differently named branch', async () => { + // branch.merge belongs to branch.remote; a branch tracking origin/main must + // first-publish under its own name rather than push onto main. + await expectSamePushArgv( + { + branch: 'feature/fix', + merge: 'refs/heads/main', + branchRemote: 'origin' + }, + FIRST_PUBLISH + ) + }) + + it('refuses a pushDefault fork whose branch.remote names a different remote', async () => { + await expectSamePushArgv( + { + branch: 'review/pr-1738', + merge: 'refs/heads/contributor/fix', + branchRemote: 'origin', + pushDefault: 'fork' + }, + FIRST_PUBLISH + ) + }) + + it('refuses when branch.base names the same remote branch as branch.merge', async () => { + await expectSamePushArgv( + { + branch: 'feature/fix', + merge: 'refs/heads/release', + branchRemote: 'fork', + pushRemote: 'fork', + base: 'fork/release' + }, + FIRST_PUBLISH + ) + }) + + it('resolves a URL-valued pushRemote back to its remote name', async () => { + await expectSamePushArgv( + { + branch: 'review/pr-1738', + merge: 'refs/heads/contributor/fix', + branchRemote: 'git@example.invalid:contributor/repo.git', + pushRemote: 'git@example.invalid:contributor/repo.git', + remotes: { + origin: 'git@example.invalid:upstream/repo.git', + fork: 'git@example.invalid:contributor/repo.git' + } + }, + ['push', '--set-upstream', 'fork', 'HEAD:contributor/fix'] + ) + }) + + it('treats a local-repository remote as no configured target', async () => { + await expectSamePushArgv( + { + branch: 'feature/fix', + merge: 'refs/heads/feature/fix', + branchRemote: '.' + }, + FIRST_PUBLISH + ) + }) + + it('first-publishes a branch with no configured remote at all', async () => { + await expectSamePushArgv( + { branch: 'feature/fix', merge: 'refs/heads/feature/fix' }, + FIRST_PUBLISH + ) + }) +}) diff --git a/src/shared/git-push-target-resolution.ts b/src/shared/git-push-target-resolution.ts new file mode 100644 index 00000000000..63fe7828d9e --- /dev/null +++ b/src/shared/git-push-target-resolution.ts @@ -0,0 +1,140 @@ +import type { GitCommandRunner } from './git-effective-upstream' +import { gitRefTargetsBranchOnRemote } from './git-remote-branch-name' +import { findGitRemoteNameByFetchUrl } from './git-remote-url-index' + +export type ResolvedGitPushTarget = { + remote: string + refspec: string +} + +async function getConfigValue(runGit: GitCommandRunner, key: string): Promise { + try { + const { stdout } = await runGit(['config', '--get', key]) + const value = stdout.trim() + return value || null + } catch { + return null + } +} + +function isUrlValuedRemote(remote: string): boolean { + return /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(remote) || /^[^@/:]+@[^:]+:.+/.test(remote) +} + +type ConfiguredPushRemote = { + remote: string + branchRemote: string | null +} + +// One `git remote -v` instead of `git remote` plus a serial `git remote get-url` +// per remote; both print the same insteadOf-expanded fetch URL. +async function findRemoteNameForUrl( + runGit: GitCommandRunner, + remoteUrl: string +): Promise { + try { + const { stdout } = await runGit(['remote', '-v']) + return findGitRemoteNameByFetchUrl(stdout, (candidateUrl) => candidateUrl === remoteUrl) + } catch { + return null + } +} + +async function normalizePushRemote(runGit: GitCommandRunner, remote: string): Promise { + if (!isUrlValuedRemote(remote)) { + return remote + } + return (await findRemoteNameForUrl(runGit, remote)) ?? remote +} + +async function getConfiguredPushRemote( + runGit: GitCommandRunner, + branch: string +): Promise { + const branchRemote = await getConfigValue(runGit, `branch.${branch}.remote`) + const remote = + (await getConfigValue(runGit, `branch.${branch}.pushRemote`)) ?? + (await getConfigValue(runGit, 'remote.pushDefault')) ?? + branchRemote + if (!remote) { + return null + } + const normalizedRemote = await normalizePushRemote(runGit, remote) + // The two usually name the same URL; resolving it twice reads the remote table twice. + if (!branchRemote) { + return { remote: normalizedRemote, branchRemote: null } + } + return { + remote: normalizedRemote, + branchRemote: + branchRemote === remote ? normalizedRemote : await normalizePushRemote(runGit, branchRemote) + } +} + +async function branchMergeTargetsConfiguredBase( + runGit: GitCommandRunner, + branch: string, + remote: string, + branchRef: string +): Promise { + return gitRefTargetsBranchOnRemote( + await getConfigValue(runGit, `branch.${branch}.base`), + remote, + branchRef + ) +} + +function canPushConfiguredMergeBranch( + pushRemote: ConfiguredPushRemote | null, + branch: string, + branchRef: string +): boolean { + if (!pushRemote) { + return false + } + if (branchRef === branch) { + return true + } + // Why: branch.merge belongs to branch.remote. A pushDefault fork must not + // inherit origin/main as its destination branch. + return pushRemote.remote !== 'origin' && pushRemote.branchRemote === pushRemote.remote +} + +/** + * Which remote and refspec a plain `git push` from this worktree should hit, or `null` + * to fall back to first-publish (`origin HEAD`). + * + * Why shared: this decides where commits land, and a wrong answer is not recoverable by + * retrying. The local runner and the SSH relay must never be able to answer differently + * for the same repository — they differ only in how `runGit` reaches the Git binary. + */ +export async function resolveConfiguredGitPushTarget( + runGit: GitCommandRunner +): Promise { + try { + const { stdout: branchStdout } = await runGit(['symbolic-ref', '--quiet', '--short', 'HEAD']) + const branch = branchStdout.trim() + if (!branch) { + return null + } + const [pushRemote, { stdout: mergeStdout }] = await Promise.all([ + getConfiguredPushRemote(runGit, branch), + runGit(['config', '--get', `branch.${branch}.merge`]) + ]) + const remote = pushRemote?.remote + const mergeRef = mergeStdout.trim() + const branchRef = mergeRef.replace(/^refs\/heads\//, '') + if (!remote || !branchRef || remote === '.' || branchRef === mergeRef) { + return null + } + if (await branchMergeTargetsConfiguredBase(runGit, branch, remote, branchRef)) { + return null + } + if (!canPushConfiguredMergeBranch(pushRemote, branch, branchRef)) { + return null + } + return { remote, refspec: `HEAD:${branchRef}` } + } catch { + return null + } +} From 232d04f5414edef2fc219f7726d15b9b15fd0d9b Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:43:01 -0700 Subject: [PATCH 13/76] fix(dashboard): open remote sessions from every agent reveal path (#18403) Three reveal paths called bare setActiveWorktree + activateTabAndFocusPane, skipping setActiveView('terminal'), ensureWorktreeHasInitialTerminal and resumeSleepingAgentSessionsForWorktree. A parked SSH workspace has no resident tab until those run, so the reveal landed on a workspace with no terminal. Route all three through the incumbent activateAndRevealWorkspace dispatcher (which the sidebar and "Jump to workspace" already use, and which also handles folder workspaces). The Activity row-click additionally early-returned when the thread's tab was absent from tabsByWorktree/unifiedTabsByWorktree, which made a cold-parked remote thread a silent no-op; residency is now probed after activation, so a revived tab is focused and a genuinely retained thread still activates its workspace instead of doing nothing. Also stop asserting `exited` from an absence of local state: SshPtyProvider reports no authoritative buffer snapshot and the relay has no snapshot RPC, so a null preview snapshot for a remote pty is loss of contact. The preview and the no-pty dialog branch now say the remote preview is unavailable rather than claiming the pane closed. Adding the relay snapshot RPC stays out of scope -- it needs capability negotiation. Fixes #16731 --- .../activity/activity-thread-actions.test.ts | 106 ++++++++++++------ .../activity/activity-thread-actions.ts | 40 +++---- .../AgentTerminalDialog.test.tsx | 26 +++++ .../dashboard-popout/AgentTerminalDialog.tsx | 6 +- .../AgentTerminalPreview.test.tsx | 8 ++ .../dashboard-popout/AgentTerminalPreview.tsx | 7 +- ...rminal-preview-unavailable-message.test.ts | 18 +++ .../terminal-preview-unavailable-message.ts | 27 +++++ .../dashboard/AgentDashboardDrawer.test.tsx | 67 ++++++++--- .../dashboard/AgentDashboardDrawer.tsx | 5 +- .../dashboard/reveal-dashboard-agent.ts | 23 ++++ .../useDashboardPopoutBridge.test.tsx | 61 +++++++++- .../dashboard/useDashboardPopoutBridge.ts | 5 +- src/renderer/src/i18n/locales/en.json | 1 + 14 files changed, 308 insertions(+), 92 deletions(-) create mode 100644 src/renderer/src/components/dashboard-popout/terminal-preview-unavailable-message.test.ts create mode 100644 src/renderer/src/components/dashboard-popout/terminal-preview-unavailable-message.ts create mode 100644 src/renderer/src/components/dashboard/reveal-dashboard-agent.ts diff --git a/src/renderer/src/components/activity/activity-thread-actions.test.ts b/src/renderer/src/components/activity/activity-thread-actions.test.ts index ce2de01fe02..91375ec974b 100644 --- a/src/renderer/src/components/activity/activity-thread-actions.test.ts +++ b/src/renderer/src/components/activity/activity-thread-actions.test.ts @@ -49,12 +49,23 @@ describe('activity thread host routing', () => { const setActiveWorktree = vi.fn() const acknowledgeAgents = vi.fn() const setSelectedPaneKey = vi.fn() + let state: Record + + function makeActions(): ReturnType { + return createActivityThreadActions({ + getMarkAllReadThreads: () => [thread], + acknowledgeAgents, + unacknowledgeAgents: vi.fn(), + setSelectedPaneKey + }) + } beforeEach(() => { vi.clearAllMocks() mocks.activateStructuredAgentSessionTab.mockReturnValue(false) + mocks.activateAndRevealWorkspace.mockReturnValue({ primaryTabId: null }) getKnownWorktreeById.mockReturnValue(thread.worktree) - mocks.getState.mockReturnValue({ + state = { getKnownWorktreeById, worktreesByRepo: { [thread.worktree.repoId]: [thread.worktree] }, detectedWorktreesByRepo: {}, @@ -77,21 +88,19 @@ describe('activity thread host routing', () => { setActiveRepo: vi.fn(), setActiveWorktree, setActiveTabType: vi.fn() - }) + } + mocks.getState.mockImplementation(() => state) }) - it('selects the matching host when the same workspace id is active elsewhere', () => { - const actions = createActivityThreadActions({ - getMarkAllReadThreads: () => [thread], - acknowledgeAgents, - unacknowledgeAgents: vi.fn(), - setSelectedPaneKey + it('routes the row click through the full activation sequence for the matching host', () => { + makeActions().selectThread(thread) + + // Bare setActiveWorktree skips setActiveView('terminal'), initial-terminal seeding and + // sleeping-session resume — the workspace dispatcher is the only path that runs them. + expect(mocks.activateAndRevealWorkspace).toHaveBeenCalledWith(thread.worktree.id, { + executionHostId: REMOTE_HOST }) - - actions.selectThread(thread) - - expect(getKnownWorktreeById).toHaveBeenCalledWith(thread.worktree.id, REMOTE_HOST) - expect(setActiveWorktree).toHaveBeenCalledWith(thread.worktree.id, REMOTE_HOST) + expect(setActiveWorktree).not.toHaveBeenCalled() expect(mocks.activateTabAndFocusPane).toHaveBeenCalledWith( thread.tab.id, '11111111-1111-4111-8111-111111111111', @@ -99,23 +108,56 @@ describe('activity thread host routing', () => { ) }) - it('activates a structured agent session instead of looking for a terminal pane', () => { - mocks.activateStructuredAgentSessionTab.mockReturnValue(true) - mocks.getState.mockReturnValue({ - ...mocks.getState(), - tabsByWorktree: { [thread.worktree.id]: [] }, - unifiedTabsByWorktree: { - [thread.worktree.id]: [{ id: thread.tab.id, contentType: 'agent-session' }] - } - }) - const actions = createActivityThreadActions({ - getMarkAllReadThreads: () => [thread], - acknowledgeAgents, - unacknowledgeAgents: vi.fn(), - setSelectedPaneKey + it('opens a cold-parked remote thread whose tab activation revives', () => { + // The reported SSH symptom: the tab is not resident because the session was never + // revived, so a residency probe before activation made the click a silent no-op. + state.tabsByWorktree = {} + mocks.activateAndRevealWorkspace.mockImplementation(() => { + state.tabsByWorktree = { [thread.worktree.id]: [thread.tab] } + return { primaryTabId: thread.tab.id } }) - actions.selectThread(thread) + makeActions().selectThread(thread) + + expect(setSelectedPaneKey).toHaveBeenCalledWith(thread.paneKey) + expect(mocks.activateAndRevealWorkspace).toHaveBeenCalledWith(thread.worktree.id, { + executionHostId: REMOTE_HOST + }) + expect(mocks.activateTabAndFocusPane).toHaveBeenCalledWith( + thread.tab.id, + '11111111-1111-4111-8111-111111111111', + { flashFocusedPane: true, scrollToBottomIfOutputSinceLastView: true } + ) + }) + + it('still activates the workspace when a retained thread has no tab to focus', () => { + state.tabsByWorktree = {} + + makeActions().selectThread(thread) + + expect(mocks.activateAndRevealWorkspace).toHaveBeenCalledWith(thread.worktree.id, { + executionHostId: REMOTE_HOST + }) + expect(mocks.activateTabAndFocusPane).not.toHaveBeenCalled() + }) + + it('focuses nothing when the workspace itself is gone', () => { + mocks.activateAndRevealWorkspace.mockReturnValue(false) + + makeActions().selectThread(thread) + + expect(mocks.activateStructuredAgentSessionTab).not.toHaveBeenCalled() + expect(mocks.activateTabAndFocusPane).not.toHaveBeenCalled() + }) + + it('activates a structured agent session instead of looking for a terminal pane', () => { + mocks.activateStructuredAgentSessionTab.mockReturnValue(true) + state.tabsByWorktree = { [thread.worktree.id]: [] } + state.unifiedTabsByWorktree = { + [thread.worktree.id]: [{ id: thread.tab.id, contentType: 'agent-session' }] + } + + makeActions().selectThread(thread) expect(mocks.activateStructuredAgentSessionTab).toHaveBeenCalledWith({ worktreeId: thread.worktree.id, @@ -126,14 +168,8 @@ describe('activity thread host routing', () => { it('jumps to and probes the matching host-qualified workspace', () => { expect(hasActivityThreadWorkspace(thread)).toBe(true) - const actions = createActivityThreadActions({ - getMarkAllReadThreads: () => [thread], - acknowledgeAgents, - unacknowledgeAgents: vi.fn(), - setSelectedPaneKey - }) - actions.jumpToWorkspace(thread) + makeActions().jumpToWorkspace(thread) expect(acknowledgeAgents).toHaveBeenCalledWith([thread.paneKey]) expect(mocks.activateAndRevealWorkspace).toHaveBeenCalledWith(thread.worktree.id, { diff --git a/src/renderer/src/components/activity/activity-thread-actions.ts b/src/renderer/src/components/activity/activity-thread-actions.ts index b0971da7ea7..f9f77587a1e 100644 --- a/src/renderer/src/components/activity/activity-thread-actions.ts +++ b/src/renderer/src/components/activity/activity-thread-actions.ts @@ -1,5 +1,6 @@ import { activateTabAndFocusPane } from '@/lib/activate-tab-and-focus-pane' import { activateStructuredAgentSessionTab } from '@/lib/structured-agent-session-tab-activation' +import { activateAndRevealWorkspace } from '@/lib/worktree-activation' import { jumpToWorktreeFromSidebar } from '@/lib/worktree-jump-navigation' import { useAppStore } from '@/store' import { @@ -74,38 +75,31 @@ export function createActivityThreadActions({ } const activateThreadTarget = (thread: AgentPaneThread): void => { - const state = useAppStore.getState() const executionHostId = getActivityThreadExecutionHostId( thread, - getSettingsFocusedExecutionHostId(state.settings) + getSettingsFocusedExecutionHostId(useAppStore.getState().settings) ) - const worktree = state.getKnownWorktreeById(thread.worktree.id, executionHostId) - if (!worktree) { + // Why the full sequence (not bare setActiveWorktree): a cold-parked thread — the normal + // state of an SSH session that was never revived — has no resident tab until + // resumeSleepingAgentSessionsForWorktree/ensureWorktreeHasInitialTerminal run inside here. + // Probing tab residency first is what made a remote row click a silent no-op (#16731). + if (activateAndRevealWorkspace(thread.worktree.id, { executionHostId }) === false) { return } - const liveTabs = state.tabsByWorktree[worktree.id] ?? [] - const hasLiveTerminal = liveTabs.some((tab) => tab.id === thread.tab.id) - const hasLiveAgentSession = (state.unifiedTabsByWorktree?.[worktree.id] ?? []).some( - (tab) => tab.id === thread.tab.id && tab.contentType === 'agent-session' - ) - // Why: retained threads can outlive their target; reorienting the workspace for a - // dead terminal or structured session would just confuse the user. - if (!hasLiveTerminal && !hasLiveAgentSession) { - return - } - if (state.activeRepoId !== worktree.repoId) { - state.setActiveRepo(worktree.repoId) - } if ( - state.activeWorktreeId !== worktree.id || - state.activeWorkspaceExecutionHostId !== executionHostId + activateStructuredAgentSessionTab({ worktreeId: thread.worktree.id, tabId: thread.tab.id }) ) { - state.setActiveWorktree(worktree.id, executionHostId) - } - if (activateStructuredAgentSessionTab({ worktreeId: worktree.id, tabId: thread.tab.id })) { return } - state.setActiveTabType('terminal') + // Read post-activation: the tab this thread points at may have only just been revived. + const activated = useAppStore.getState() + const liveTabs = activated.tabsByWorktree[thread.worktree.id] ?? [] + if (!liveTabs.some((tab) => tab.id === thread.tab.id)) { + // Retained threads outlive their tab; the workspace is still activated, but there is + // no pane to focus and focusing a sibling would be worse than focusing nothing. + return + } + activated.setActiveTabType('terminal') const parsed = parsePaneKey(thread.paneKey) activateTabAndFocusPane( thread.tab.id, diff --git a/src/renderer/src/components/dashboard-popout/AgentTerminalDialog.test.tsx b/src/renderer/src/components/dashboard-popout/AgentTerminalDialog.test.tsx index 7022d7e5731..770e1974625 100644 --- a/src/renderer/src/components/dashboard-popout/AgentTerminalDialog.test.tsx +++ b/src/renderer/src/components/dashboard-popout/AgentTerminalDialog.test.tsx @@ -91,6 +91,32 @@ describe('AgentTerminalDialog', () => { expect(screen.getByTestId('preview')).toHaveAttribute('data-terminal-input', 'null') }) + it('does not claim a remote pane closed when the card carries no live pty', () => { + render( + {}} + onReveal={() => {}} + /> + ) + + // Loss of contact with an SSH host is `unverifiable`, never `exited`. + expect(screen.getByText(/remote session/)).toBeInTheDocument() + expect(screen.queryByText(/pane has closed/)).not.toBeInTheDocument() + }) + + it('still reports a closed pane for a local card with no live pty', () => { + render( + {}} + onReveal={() => {}} + /> + ) + + expect(screen.getByText(/pane has closed/)).toBeInTheDocument() + }) + it('labels acknowledged completions idle without review or pin controls', () => { render( ) : (
- {translate( - 'dashboardPopout.terminal.closed', - "No live terminal — this agent's pane has closed." - )} + {terminalPreviewUnavailableMessage({ hostKind: card.hostKind })}
)}
diff --git a/src/renderer/src/components/dashboard-popout/AgentTerminalPreview.test.tsx b/src/renderer/src/components/dashboard-popout/AgentTerminalPreview.test.tsx index 4c91a22a3b8..c4856a23834 100644 --- a/src/renderer/src/components/dashboard-popout/AgentTerminalPreview.test.tsx +++ b/src/renderer/src/components/dashboard-popout/AgentTerminalPreview.test.tsx @@ -612,6 +612,14 @@ describe('AgentTerminalPreview', () => { expect(unsubscribe).toHaveBeenCalledWith('pty-1') }) + it('does not claim a remote pane closed when no snapshot can exist for it', async () => { + connect.mockResolvedValueOnce({ snapshot: null, replay: [] }) + const view = render() + + await waitFor(() => expect(view.getByText(/remote session/)).toBeInTheDocument()) + expect(view.queryByText(/pane has closed/)).not.toBeInTheDocument() + }) + it('connects a replacement pty after the previous pty was gone', async () => { connect.mockResolvedValueOnce({ snapshot: null, replay: [] }).mockResolvedValueOnce({ snapshot: { data: 'replacement', cols: 80, rows: 24, seq: 1 }, diff --git a/src/renderer/src/components/dashboard-popout/AgentTerminalPreview.tsx b/src/renderer/src/components/dashboard-popout/AgentTerminalPreview.tsx index f2a702782e9..ec05a7105a5 100644 --- a/src/renderer/src/components/dashboard-popout/AgentTerminalPreview.tsx +++ b/src/renderer/src/components/dashboard-popout/AgentTerminalPreview.tsx @@ -17,7 +17,7 @@ import { installPreviewTerminalCompatibility } from './preview-terminal-compatib import { createPreviewClipboardPaster } from './preview-terminal-paste' import { installPreviewImeBridge, type PreviewImeBridge } from './preview-terminal-ime-bridge' import type { DashboardCardTerminalInput } from '../../../../shared/dashboard-snapshot' -import { translate } from '@/i18n/i18n' +import { terminalPreviewUnavailableMessage } from './terminal-preview-unavailable-message' import { getBuiltinTheme, resolveEffectiveTerminalAppearance } from '@/lib/terminal-theme' import { cn } from '@/lib/utils' import { useAppStore } from '@/store' @@ -430,10 +430,7 @@ export function AgentTerminalPreview({ > {ptyGone ? (
- {translate( - 'dashboardPopout.terminal.closed', - "No live terminal — this agent's pane has closed." - )} + {terminalPreviewUnavailableMessage({ ptyId })}
) : null}
{ + it('claims the pane closed only for a pty the client could have observed', () => { + expect(terminalPreviewUnavailableMessage({ ptyId: 'pty-1' })).toMatch(/pane has closed/) + expect(terminalPreviewUnavailableMessage({ hostKind: 'local' })).toMatch(/pane has closed/) + }) + + it('reports an unobservable remote preview instead of asserting the pane exited', () => { + // SshPtyProvider provides no authoritative buffer snapshot and the relay has no snapshot + // RPC, so a null snapshot is loss of contact. See docs/reference/ssh-execution-boundary.md. + const fromPtyId = terminalPreviewUnavailableMessage({ ptyId: 'ssh:devbox@@pty-3' }) + expect(fromPtyId).toMatch(/remote session/) + expect(fromPtyId).not.toMatch(/pane has closed/) + expect(terminalPreviewUnavailableMessage({ hostKind: 'ssh' })).toBe(fromPtyId) + }) +}) diff --git a/src/renderer/src/components/dashboard-popout/terminal-preview-unavailable-message.ts b/src/renderer/src/components/dashboard-popout/terminal-preview-unavailable-message.ts new file mode 100644 index 00000000000..07080084bfc --- /dev/null +++ b/src/renderer/src/components/dashboard-popout/terminal-preview-unavailable-message.ts @@ -0,0 +1,27 @@ +import { translate } from '@/i18n/i18n' +import type { DashboardCardHostKind } from '../../../../shared/dashboard-snapshot' +import { parseAppSshPtyId } from '../../../../shared/ssh-pty-id' + +/** + * A missing buffer snapshot only proves the pane exited when the client could have + * observed it. `SshPtyProvider` reports no authoritative buffer snapshot and the relay + * exposes no snapshot RPC, so for a remote pty the absence is loss of contact — + * `unverifiable`, never `exited`. See docs/reference/ssh-execution-boundary.md. + */ +export function terminalPreviewUnavailableMessage(source: { + ptyId?: string | null + hostKind?: DashboardCardHostKind +}): string { + const isRemote = + source.hostKind === 'ssh' || + (typeof source.ptyId === 'string' && parseAppSshPtyId(source.ptyId) !== null) + return isRemote + ? translate( + 'dashboardPopout.terminal.remotePreviewUnavailable', + 'No preview for this remote session — open the workspace to view the terminal.' + ) + : translate( + 'dashboardPopout.terminal.closed', + "No live terminal — this agent's pane has closed." + ) +} diff --git a/src/renderer/src/components/dashboard/AgentDashboardDrawer.test.tsx b/src/renderer/src/components/dashboard/AgentDashboardDrawer.test.tsx index 75b76e396b4..97c31c4747a 100644 --- a/src/renderer/src/components/dashboard/AgentDashboardDrawer.test.tsx +++ b/src/renderer/src/components/dashboard/AgentDashboardDrawer.test.tsx @@ -8,13 +8,18 @@ const mocks = vi.hoisted(() => ({ useLiveDashboardSnapshot: vi.fn(() => ({ generatedAt: 1, cards: [] })), blockingOverlay: false, boardProps: null as Record | null, - activateTabAndFocusPane: vi.fn() + activateTabAndFocusPane: vi.fn(), + activateAndRevealWorkspace: vi.fn(() => ({ primaryTabId: null }) as unknown) })) vi.mock('@/lib/activate-tab-and-focus-pane', () => ({ activateTabAndFocusPane: mocks.activateTabAndFocusPane })) +vi.mock('@/lib/worktree-activation', () => ({ + activateAndRevealWorkspace: mocks.activateAndRevealWorkspace +})) + vi.mock('./useLiveDashboardSnapshot', () => ({ useLiveDashboardSnapshot: mocks.useLiveDashboardSnapshot })) @@ -49,6 +54,9 @@ beforeEach(() => { false ) mocks.useLiveDashboardSnapshot.mockClear() + mocks.activateTabAndFocusPane.mockClear() + mocks.activateAndRevealWorkspace.mockClear() + mocks.activateAndRevealWorkspace.mockReturnValue({ primaryTabId: null }) mocks.blockingOverlay = false mocks.boardProps = null ;(window as unknown as { api: unknown }).api = { @@ -95,34 +103,63 @@ describe('AgentDashboardDrawer', () => { expect(mocks.boardProps?.initialView).toBeUndefined() }) - it('reveals a colliding worktree on the card execution host', () => { - const setActiveWorktree = vi.spyOn(useAppStore.getState(), 'setActiveWorktree') + type RevealAgent = (args: { + repoId: string + worktreeId: string + executionHostId?: string + tabId: string + leafId: string | null + }) => void + + function revealFromBoard(executionHostId: string): void { render() act(() => useAppStore.setState({ agentDashboardDrawerOpen: true })) const onRevealAgent = mocks.boardProps?.onRevealAgent expect(onRevealAgent).toBeTypeOf('function') - act(() => { - ;( - onRevealAgent as (args: { - repoId: string - worktreeId: string - executionHostId?: string - tabId: string - leafId: string | null - }) => void - )({ + ;(onRevealAgent as RevealAgent)({ repoId: 'repo-1', worktreeId: 'shared-worktree', - executionHostId: 'runtime:env-1', + executionHostId, tabId: 'tab-1', leafId: 'leaf-1' }) }) + } - expect(setActiveWorktree).toHaveBeenCalledWith('shared-worktree', 'runtime:env-1') + it('reveals a colliding worktree on the card execution host', () => { + const setActiveWorktree = vi.spyOn(useAppStore.getState(), 'setActiveWorktree') + + revealFromBoard('runtime:env-1') + + // Bare setActiveWorktree skips the terminal view switch, initial-terminal seeding and + // sleeping-session resume the shared dispatcher runs. + expect(mocks.activateAndRevealWorkspace).toHaveBeenCalledWith('shared-worktree', { + executionHostId: 'runtime:env-1' + }) + expect(setActiveWorktree).not.toHaveBeenCalled() expect(mocks.activateTabAndFocusPane).toHaveBeenCalledWith('tab-1', 'leaf-1', { flashFocusedPane: true }) }) + + it('activates a parked SSH workspace before reaching for its pane', () => { + revealFromBoard('ssh:devbox') + + expect(mocks.activateAndRevealWorkspace).toHaveBeenCalledWith('shared-worktree', { + executionHostId: 'ssh:devbox' + }) + // Ordering is the fix: a parked remote tab only exists after activation revives it. + expect(mocks.activateAndRevealWorkspace.mock.invocationCallOrder[0]).toBeLessThan( + mocks.activateTabAndFocusPane.mock.invocationCallOrder[0] as number + ) + }) + + it('skips pane focus when the revealed workspace is gone', () => { + mocks.activateAndRevealWorkspace.mockReturnValue(false) + + revealFromBoard('ssh:devbox') + + expect(mocks.activateTabAndFocusPane).not.toHaveBeenCalled() + }) }) diff --git a/src/renderer/src/components/dashboard/AgentDashboardDrawer.tsx b/src/renderer/src/components/dashboard/AgentDashboardDrawer.tsx index a5df13c2395..347255324fd 100644 --- a/src/renderer/src/components/dashboard/AgentDashboardDrawer.tsx +++ b/src/renderer/src/components/dashboard/AgentDashboardDrawer.tsx @@ -1,7 +1,7 @@ import { useCallback, useEffect, useRef, useState } from 'react' import { useAppStore } from '@/store' import { Sheet, SheetContent, SheetTitle } from '@/components/ui/sheet' -import { activateTabAndFocusPane } from '@/lib/activate-tab-and-focus-pane' +import { revealDashboardAgent } from './reveal-dashboard-agent' import { AgentKanbanBoard } from '../dashboard-popout/AgentKanbanBoard' import type { AgentRevealArgs } from '../dashboard-popout/AgentTerminalDialog' import { @@ -47,8 +47,7 @@ function AgentDashboardDrawerBody({ }, []) const handleRevealAgent = useCallback( (args: AgentRevealArgs) => { - useAppStore.getState().setActiveWorktree(args.worktreeId, args.executionHostId) - activateTabAndFocusPane(args.tabId, args.leafId, { flashFocusedPane: true }) + revealDashboardAgent(args) onClose() }, [onClose] diff --git a/src/renderer/src/components/dashboard/reveal-dashboard-agent.ts b/src/renderer/src/components/dashboard/reveal-dashboard-agent.ts new file mode 100644 index 00000000000..9da364c72ef --- /dev/null +++ b/src/renderer/src/components/dashboard/reveal-dashboard-agent.ts @@ -0,0 +1,23 @@ +import { activateTabAndFocusPane } from '@/lib/activate-tab-and-focus-pane' +import { activateAndRevealWorkspace } from '@/lib/worktree-activation' +import type { DashboardRevealAgentArgs } from '../../../../shared/dashboard-snapshot' + +/** + * Click-to-focus from either Agent Dashboard surface (pop-out relay or in-window drawer). + * + * Why the workspace dispatcher rather than a bare `setActiveWorktree`: only the shared + * sequence switches the view back to terminal, resumes sleeping agent sessions, and seeds a + * terminal surface. A parked SSH workspace has no resident tab until those run, so the bare + * call revealed a workspace with nothing in it (#16731). + */ +export function revealDashboardAgent(args: DashboardRevealAgentArgs): boolean { + const activated = activateAndRevealWorkspace( + args.worktreeId, + args.executionHostId ? { executionHostId: args.executionHostId } : undefined + ) + if (activated === false) { + return false + } + activateTabAndFocusPane(args.tabId, args.leafId, { flashFocusedPane: true }) + return true +} diff --git a/src/renderer/src/components/dashboard/useDashboardPopoutBridge.test.tsx b/src/renderer/src/components/dashboard/useDashboardPopoutBridge.test.tsx index aa427ed55c5..e80bf56d778 100644 --- a/src/renderer/src/components/dashboard/useDashboardPopoutBridge.test.tsx +++ b/src/renderer/src/components/dashboard/useDashboardPopoutBridge.test.tsx @@ -21,7 +21,9 @@ const mocks = vi.hoisted(() => ({ offRevealAgent: vi.fn(), offAckAgent: vi.fn(), offPopoutOpenChanged: vi.fn(), - offSnapshotRequested: vi.fn() + offSnapshotRequested: vi.fn(), + activateTabAndFocusPane: vi.fn(), + activateAndRevealWorkspace: vi.fn() })) vi.mock('@/store', () => ({ @@ -35,7 +37,11 @@ vi.mock('@/store', () => ({ })) vi.mock('@/lib/activate-tab-and-focus-pane', () => ({ - activateTabAndFocusPane: vi.fn() + activateTabAndFocusPane: mocks.activateTabAndFocusPane +})) + +vi.mock('@/lib/worktree-activation', () => ({ + activateAndRevealWorkspace: mocks.activateAndRevealWorkspace })) vi.mock('./build-dashboard-snapshot', () => ({ @@ -159,7 +165,8 @@ describe('useDashboardPopoutBridge', () => { expect(mocks.buildDashboardSnapshot).toHaveBeenCalledTimes(1) }) - it('reveals the agent on its exact execution host', async () => { + it('reveals the agent on its exact execution host through the full activation', async () => { + mocks.activateAndRevealWorkspace.mockReturnValue({ primaryTabId: null }) await act(async () => root.render()) await act(async () => @@ -172,7 +179,53 @@ describe('useDashboardPopoutBridge', () => { }) ) - expect(mocks.setActiveWorktree).toHaveBeenCalledWith('shared-worktree', 'runtime:env-1') + // Bare setActiveWorktree skips the terminal view switch, initial-terminal seeding and + // sleeping-session resume, so a parked pane is never revived (#16731). + expect(mocks.activateAndRevealWorkspace).toHaveBeenCalledWith('shared-worktree', { + executionHostId: 'runtime:env-1' + }) + expect(mocks.setActiveWorktree).not.toHaveBeenCalled() + expect(mocks.activateTabAndFocusPane).toHaveBeenCalledWith('tab-1', 'leaf-1', { + flashFocusedPane: true + }) + }) + + it('activates a parked SSH workspace before reaching for its pane', async () => { + mocks.activateAndRevealWorkspace.mockReturnValue({ primaryTabId: 'tab-1' }) + await act(async () => root.render()) + + await act(async () => + mocks.onRevealAgent.mock.calls[0][0]({ + repoId: 'repo-1', + worktreeId: 'remote-worktree', + executionHostId: 'ssh:devbox', + tabId: 'tab-1', + leafId: 'leaf-1' + }) + ) + + expect(mocks.activateAndRevealWorkspace).toHaveBeenCalledWith('remote-worktree', { + executionHostId: 'ssh:devbox' + }) + expect(mocks.activateAndRevealWorkspace.mock.invocationCallOrder[0]).toBeLessThan( + mocks.activateTabAndFocusPane.mock.invocationCallOrder[0] as number + ) + }) + + it('skips pane focus when the revealed workspace is gone', async () => { + mocks.activateAndRevealWorkspace.mockReturnValue(false) + await act(async () => root.render()) + + await act(async () => + mocks.onRevealAgent.mock.calls[0][0]({ + repoId: 'repo-1', + worktreeId: 'deleted-worktree', + tabId: 'tab-1', + leafId: 'leaf-1' + }) + ) + + expect(mocks.activateTabAndFocusPane).not.toHaveBeenCalled() }) it('ignores unrelated store writes while retaining every snapshot input', () => { diff --git a/src/renderer/src/components/dashboard/useDashboardPopoutBridge.ts b/src/renderer/src/components/dashboard/useDashboardPopoutBridge.ts index e6163a70773..f80de74a748 100644 --- a/src/renderer/src/components/dashboard/useDashboardPopoutBridge.ts +++ b/src/renderer/src/components/dashboard/useDashboardPopoutBridge.ts @@ -1,6 +1,6 @@ import { useEffect } from 'react' import { useAppStore, type AppState } from '@/store' -import { activateTabAndFocusPane } from '@/lib/activate-tab-and-focus-pane' +import { revealDashboardAgent } from './reveal-dashboard-agent' import { runSleepWorktree } from '../sidebar/sleep-worktree-flow' import type { RepoIcon } from '../../../../shared/repo-icon' import { buildDashboardSnapshot, type DashboardSnapshotState } from './build-dashboard-snapshot' @@ -133,8 +133,7 @@ export function useDashboardPopoutBridge(enabled: boolean): void { return } return window.api.dashboard.onRevealAgent((args) => { - useAppStore.getState().setActiveWorktree(args.worktreeId, args.executionHostId) - activateTabAndFocusPane(args.tabId, args.leafId, { flashFocusedPane: true }) + revealDashboardAgent(args) }) }, [enabled]) diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index 2ba5712f4fa..28149e7ca70 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -17230,6 +17230,7 @@ }, "terminal": { "closed": "No live terminal — this agent's pane has closed.", + "remotePreviewUnavailable": "No preview for this remote session — open the workspace to view the terminal.", "focusWorktree": "Open worktree", "close": "Close" }, From 9bed758e36951fd2ab9a1d9a7178729591a7048a Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:43:05 -0700 Subject: [PATCH 14/76] fix(cli): reject runtime selectors on `host list` and `environment list` (#18405) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `orca host list --environment m4air` was not ignoring the flag — it was applying it to half the answer. `shouldIgnoreRemoteSelection` never pinned the `host` family, so the SSH-target lookup was routed to m4air while paired servers were still read from this machine's own pairing store, and the handler stamped the envelope `_meta.runtimeId: "local"` regardless. The result was one listing describing two hosts: the openclaw row silently disappeared, which reads as "m4air has no SSH targets". `environment list --environment X` had the pin but no guard, so the flag vanished with no signal at all. Reject rather than route. `host list` answers "what can this machine target and with what flag"; its paired-server half comes from a client-local store and cannot be routed at all, so any routed answer is necessarily half-substituted — rule 1 of docs/reference/ssh-execution-boundary.md. `environment list` is entirely client-local, so there is no other host to ask. This matches the `account` and `artifacts` precedent, the only two pinned families that already paired the pin with a rejection guard. - pin the `host` family so an ambient ORCA_ENVIRONMENT cannot produce the same two-machine listing with no flag to reject; `runtimeId: "local"` is now true - extract the duplicated `rejectRemoteSelectionFlags` from account.ts and artifacts.ts into src/cli/remote-selection-flag-rejection.ts - `environment show` / `environment rm` / `environment add` are untouched: there `--environment` and `--pairing-code` name the row to act on, not a route --- src/cli/handlers/account.ts | 19 +- src/cli/handlers/artifacts.ts | 25 ++- src/cli/handlers/environment.ts | 32 ++- .../index-local-command-routing-flags.test.ts | 184 ++++++++++++++++++ src/cli/index.ts | 4 + src/cli/remote-selection-flag-rejection.ts | 29 +++ src/cli/specs/environment.ts | 8 +- 7 files changed, 272 insertions(+), 29 deletions(-) create mode 100644 src/cli/index-local-command-routing-flags.test.ts create mode 100644 src/cli/remote-selection-flag-rejection.ts diff --git a/src/cli/handlers/account.ts b/src/cli/handlers/account.ts index a6ee5d73246..5a8bd4d3c6c 100644 --- a/src/cli/handlers/account.ts +++ b/src/cli/handlers/account.ts @@ -7,6 +7,7 @@ import type { CommandHandler, HandlerContext } from '../dispatch' import { printResult } from '../format' import { RuntimeClientError } from '../runtime-client' import { stripElectronRunAsNode } from '../runtime/launch' +import { rejectRemoteSelectionFlags } from '../remote-selection-flag-rejection' import { deleteActiveClaudeKeychainCredentialsStrict, readActiveClaudeKeychainCredentialsStrict, @@ -276,15 +277,11 @@ async function addCodexAccount({ client, json }: HandlerContext): Promise * mistake this feature exists to avoid. A `--help` note does not reach someone who * already typed the flag. */ -function rejectRemoteSelectionFlags(ctx: HandlerContext, command: string): void { - for (const flag of ['environment', 'pairing-code']) { - if (ctx.flags.has(flag)) { - throw new RuntimeClientError( - 'invalid_argument', - `\`--${flag}\` does not retarget \`${command}\`. Run it on the host whose accounts you want to manage.` - ) - } - } +function rejectAccountRemoteSelectionFlags(ctx: HandlerContext, command: string): void { + rejectRemoteSelectionFlags( + ctx.flags, + `\`${command}\`. Run it on the host whose accounts you want to manage.` + ) } async function assertAccountImportSupported({ client }: HandlerContext): Promise { @@ -316,14 +313,14 @@ export const ACCOUNT_HANDLERS: Record = { `Unsupported --agent "${agent}". Use "claude" or "codex".` ) } - rejectRemoteSelectionFlags(ctx, 'orca account add') + rejectAccountRemoteSelectionFlags(ctx, 'orca account add') // Why: fail on runtime version skew before burning a full OAuth round trip. await assertAccountImportSupported(ctx) await ctx.client.call('accounts.list', { refreshUsage: false }) await (agent === 'claude' ? addClaudeAccount(ctx) : addCodexAccount(ctx)) }, 'account list': async (ctx) => { - rejectRemoteSelectionFlags(ctx, 'orca account list') + rejectAccountRemoteSelectionFlags(ctx, 'orca account list') const { client, json } = ctx // Why: this command renders no usage numbers, so skip the forced provider // refresh — it is one serial network round-trip per managed account. diff --git a/src/cli/handlers/artifacts.ts b/src/cli/handlers/artifacts.ts index 2306dcc406a..8546d7997f3 100644 --- a/src/cli/handlers/artifacts.ts +++ b/src/cli/handlers/artifacts.ts @@ -18,6 +18,7 @@ import { ARTIFACT_SHARING_DISABLED_NEXT_STEPS } from '../../shared/artifact-sharing-gate' import type { CommandHandler, HandlerContext } from '../dispatch' +import { rejectRemoteSelectionFlags } from '../remote-selection-flag-rejection' import { RuntimeClientError } from '../runtime-client' import { formatArtifactListPage, formatArtifactShared } from '../artifact-format' import { printResult } from '../format' @@ -44,15 +45,11 @@ function cloudOptions(ctx: HandlerContext): ArtifactCloudOptions { } } -function rejectRemoteSelectionFlags(ctx: HandlerContext): void { - for (const flag of ['environment', 'pairing-code']) { - if (ctx.flags.has(flag)) { - throw new RuntimeClientError( - 'invalid_argument', - `\`--${flag}\` does not retarget artifact commands; artifacts use the signed-in desktop account.` - ) - } - } +function rejectArtifactRemoteSelectionFlags(ctx: HandlerContext): void { + rejectRemoteSelectionFlags( + ctx.flags, + 'artifact commands; artifacts use the signed-in desktop account.' + ) } function artifactContentType(path: string): ArtifactWriteRequest['contentType'] | null { @@ -165,7 +162,7 @@ function requireOperation(operation: ArtifactCloudOperation): T { export const ARTIFACT_HANDLERS: Record = { 'artifacts list': async (ctx) => { - rejectRemoteSelectionFlags(ctx) + rejectArtifactRemoteSelectionFlags(ctx) const cursor = stringFlag(ctx, 'cursor') const response = await ctx.client.call>( 'artifacts.list', @@ -178,7 +175,7 @@ export const ARTIFACT_HANDLERS: Record = { printResult({ ...response, result: value }, ctx.json, formatArtifactListPage) }, 'artifacts share': async (ctx) => { - rejectRemoteSelectionFlags(ctx) + rejectArtifactRemoteSelectionFlags(ctx) const response = await ctx.client.call>( 'artifacts.share', await readArtifactRequest(ctx) @@ -187,7 +184,7 @@ export const ARTIFACT_HANDLERS: Record = { printResult({ ...response, result: value }, ctx.json, formatArtifactShared) }, 'artifacts update': async (ctx) => { - rejectRemoteSelectionFlags(ctx) + rejectArtifactRemoteSelectionFlags(ctx) const response = await ctx.client.call>( 'artifacts.update', await readArtifactRequest(ctx) @@ -196,7 +193,7 @@ export const ARTIFACT_HANDLERS: Record = { printResult({ ...response, result: value }, ctx.json, formatArtifactShared) }, 'artifacts unshare': async (ctx) => { - rejectRemoteSelectionFlags(ctx) + rejectArtifactRemoteSelectionFlags(ctx) const remoteInput = parseRemoteArtifactInput(process.env[REMOTE_ARTIFACT_INPUT_ENV]) const sourceKey = remoteInput?.sourceKey ?? resolve(ctx.cwd, requireStringFlag(ctx, 'file')) const response = await ctx.client.call>('artifacts.unshare', { @@ -207,7 +204,7 @@ export const ARTIFACT_HANDLERS: Record = { printResult({ ...response, result: { deleted: true } }, ctx.json, () => 'Artifact deleted.') }, 'artifacts delete': async (ctx) => { - rejectRemoteSelectionFlags(ctx) + rejectArtifactRemoteSelectionFlags(ctx) const response = await ctx.client.call>('artifacts.delete', { id: requireStringFlag(ctx, 'id'), ...cloudOptions(ctx) diff --git a/src/cli/handlers/environment.ts b/src/cli/handlers/environment.ts index 2a433021769..37b437af2c3 100644 --- a/src/cli/handlers/environment.ts +++ b/src/cli/handlers/environment.ts @@ -3,6 +3,7 @@ import { formatEnvironment, formatEnvironmentList, formatHostList, printResult } import { listSshTargets } from '../host-selector-alternatives' import { getDefaultUserDataPath, RuntimeClientError } from '../runtime-client' import type { RuntimeRpcSuccess } from '../runtime-client' +import { rejectRemoteSelectionFlags } from '../remote-selection-flag-rejection' import { redactRuntimeEnvironment } from '../../shared/runtime-environments' import { addEnvironmentFromPairingCode, @@ -33,7 +34,12 @@ export const ENVIRONMENT_HANDLERS: Record = { // Why: an agent told "run it on " had nowhere to look. `orca environment list` showed // paired servers only, and nothing in the CLI listed SSH targets at all, so the wrong-axis // guess was the only move available. This is the one place that answers both. - 'host list': async ({ client, json }) => { + 'host list': async ({ client, flags, json }) => { + rejectLocalPairingStoreRetargeting( + flags, + '`orca host list`. It answers from this machine\u2019s own pairing store, so a routed answer would name servers paired with a different machine.', + 'Run `orca host list` on that machine to see the SSH targets registered there.' + ) const environments = listEnvironments(getDefaultUserDataPath()).map((environment) => ({ kind: 'environment' as const, name: environment.name, @@ -53,7 +59,12 @@ export const ENVIRONMENT_HANDLERS: Record = { ] printResult(localSuccess({ hosts }), json, formatHostList) }, - 'environment list': async ({ json }) => { + 'environment list': async ({ flags, json }) => { + rejectLocalPairingStoreRetargeting( + flags, + '`orca environment list`. Paired servers are stored on this machine, so there is no other host to ask.', + 'Run `orca environment list` on that machine to see the servers paired with it.' + ) const environments = listEnvironments(getDefaultUserDataPath()).map(redactRuntimeEnvironment) printResult(localSuccess({ environments }), json, formatEnvironmentList) }, @@ -78,6 +89,23 @@ export const ENVIRONMENT_HANDLERS: Record = { } } +/** + * These two listings are pinned local by `shouldIgnoreRemoteSelection`, so a runtime selector is + * dropped for routing. It used to still reach the SSH half of `host list` through the routed + * client, producing a listing whose SSH rows came from the named server and whose paired-server + * rows came from this machine — one answer describing two hosts, stamped `runtimeId: local`. + * Failing is the only answer that is true of a single machine. + */ +function rejectLocalPairingStoreRetargeting( + flags: Map, + suffix: string, + crossHostNextStep: string +): void { + rejectRemoteSelectionFlags(flags, suffix, { + nextSteps: [crossHostNextStep, 'Drop the flag to answer for this machine.'] + }) +} + function getRequiredStringFlag(flags: Map, name: string): string { const value = flags.get(name) if (typeof value !== 'string' || value.length === 0) { diff --git a/src/cli/index-local-command-routing-flags.test.ts b/src/cli/index-local-command-routing-flags.test.ts new file mode 100644 index 00000000000..b8db44915d2 --- /dev/null +++ b/src/cli/index-local-command-routing-flags.test.ts @@ -0,0 +1,184 @@ +import { describe, expect, it, vi } from 'vitest' + +const { + callMock, + runtimeClientConstructorMock, + serveOrcaAppMock, + getDefaultUserDataPathMock, + addEnvironmentFromPairingCodeMock, + listEnvironmentsMock, + removeEnvironmentMock, + resolveEnvironmentMock, + spawnMock +} = vi.hoisted(() => ({ + callMock: vi.fn(), + runtimeClientConstructorMock: vi.fn(), + serveOrcaAppMock: vi.fn(), + getDefaultUserDataPathMock: vi.fn(() => '/tmp/orca-user-data'), + addEnvironmentFromPairingCodeMock: vi.fn(), + listEnvironmentsMock: vi.fn(), + removeEnvironmentMock: vi.fn(), + resolveEnvironmentMock: vi.fn(), + spawnMock: vi.fn() +})) + +vi.mock('./runtime-client', async () => { + const { createRuntimeClientModuleMock } = await import('./index-test-harness.js') + return createRuntimeClientModuleMock({ + callMock, + runtimeClientConstructorMock, + serveOrcaAppMock, + getDefaultUserDataPathMock + }) +}) + +vi.mock('./runtime/environments', () => ({ + addEnvironmentFromPairingCode: addEnvironmentFromPairingCodeMock, + listEnvironments: listEnvironmentsMock, + removeEnvironment: removeEnvironmentMock, + resolveEnvironment: resolveEnvironmentMock +})) + +vi.mock('child_process', async () => { + const { createChildProcessModuleMock } = await import('./index-test-harness.js') + return createChildProcessModuleMock(spawnMock) +}) + +import { main } from './index' +import { okFixture, queueFixtures } from './test-fixtures' +import { pairRuntimeEnvironment, useWorktreeAwarenessEnvironment } from './index-test-harness' + +const SSH_TARGET = { id: 'ssh-1777360569033-yvz2mp', label: 'openclaw' } + +/** Every SSH-target lookup answers with the one target only this machine's runtime knows about. */ +function queueSshTargetLookups(count: number): void { + queueFixtures( + callMock, + ...Array.from({ length: count }, () => okFixture('req_ssh_targets', { targets: [SSH_TARGET] })) + ) +} + +describe('runtime-selector flags on locally pinned CLI commands', () => { + useWorktreeAwarenessEnvironment({ + callMock, + serveOrcaAppMock, + getDefaultUserDataPathMock, + addEnvironmentFromPairingCodeMock, + listEnvironmentsMock, + spawnMock + }) + + it('answers `host list` from this machine and stamps the runtime that actually answered', async () => { + pairRuntimeEnvironment(listEnvironmentsMock, 'env-m4air', 'm4air') + queueSshTargetLookups(1) + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) + + await main(['host', 'list', '--json'], '/tmp/repo') + + const printed = JSON.parse(String(logSpy.mock.calls[0]?.[0])) + expect(printed._meta.runtimeId).toBe('local') + expect(printed.result.hosts.map((host: { id: string }) => host.id)).toEqual([ + 'local', + SSH_TARGET.id, + 'env-m4air' + ]) + // The tell: `runtimeId: local` is only honest if no routed client was ever built. + expect(runtimeClientConstructorMock).toHaveBeenCalledWith(null, null) + }) + + it('rejects `host list --environment` instead of answering with a half-routed listing', async () => { + // Why: pre-fix this routed the SSH lookup to m4air while reading paired servers from this + // machine, dropped the openclaw row, and still stamped `_meta.runtimeId: "local"` — one + // listing describing two hosts, which reads as "m4air has no SSH targets". + pairRuntimeEnvironment(listEnvironmentsMock, 'env-m4air', 'm4air') + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) + + await main(['host', 'list', '--environment', 'm4air', '--json'], '/tmp/repo') + + const printed = JSON.parse(String(logSpy.mock.calls[0]?.[0])) + expect(printed.ok).toBe(false) + expect(printed.error.code).toBe('invalid_argument') + expect(printed.error.message).toContain('`--environment` does not retarget `orca host list`') + expect(process.exitCode).toBe(1) + expect(callMock).not.toHaveBeenCalled() + expect(runtimeClientConstructorMock).not.toHaveBeenCalledWith(null, 'm4air') + process.exitCode = 0 + }) + + it('rejects `environment list --environment` rather than repeating the local answer', async () => { + pairRuntimeEnvironment(listEnvironmentsMock, 'env-m4air', 'm4air') + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) + + await main(['environment', 'list', '--environment', 'm4air', '--json'], '/tmp/repo') + + const printed = JSON.parse(String(logSpy.mock.calls[0]?.[0])) + expect(printed.ok).toBe(false) + expect(printed.error.code).toBe('invalid_argument') + expect(printed.error.message).toContain( + '`--environment` does not retarget `orca environment list`' + ) + process.exitCode = 0 + }) + + it('rejects `--pairing-code` on both listings for the same reason', async () => { + pairRuntimeEnvironment(listEnvironmentsMock, 'env-m4air', 'm4air') + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) + + await main(['host', 'list', '--pairing-code', 'orca://pair?code=x', '--json'], '/tmp/repo') + await main( + ['environment', 'list', '--pairing-code', 'orca://pair?code=x', '--json'], + '/tmp/repo' + ) + + for (const call of logSpy.mock.calls) { + const printed = JSON.parse(String(call[0])) + expect(printed.ok).toBe(false) + expect(printed.error.message).toContain('`--pairing-code` does not retarget') + } + expect(callMock).not.toHaveBeenCalled() + process.exitCode = 0 + }) + + it('keeps `host list` local when ORCA_ENVIRONMENT is set ambiently', async () => { + // Why: the ambient variable produced the same two-machine listing as the explicit flag, with + // no flag to reject. Pinning the family is what makes `runtimeId: local` true in both cases. + process.env.ORCA_ENVIRONMENT = 'm4air' + pairRuntimeEnvironment(listEnvironmentsMock, 'env-m4air', 'm4air') + queueSshTargetLookups(1) + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) + + await main(['host', 'list', '--json'], '/tmp/repo') + + const printed = JSON.parse(String(logSpy.mock.calls[0]?.[0])) + expect(printed.ok).toBe(true) + expect(printed.result.hosts.some((host: { id: string }) => host.id === SSH_TARGET.id)).toBe( + true + ) + expect(runtimeClientConstructorMock).toHaveBeenCalledWith(null, null) + expect(runtimeClientConstructorMock).not.toHaveBeenCalledWith(undefined, undefined) + }) + + it('still treats --environment as the selector argument on `environment show` and `rm`', async () => { + // Why: the guard must not fire where the flag names the row to act on rather than a route. + const environment = { + id: 'env-m4air', + name: 'm4air', + createdAt: 1, + updatedAt: 1, + lastUsedAt: null, + runtimeId: null, + endpoints: [], + preferredEndpointId: null + } + resolveEnvironmentMock.mockReturnValue(environment) + removeEnvironmentMock.mockReturnValue(environment) + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) + + await main(['environment', 'show', '--environment', 'm4air', '--json'], '/tmp/repo') + await main(['environment', 'rm', '--environment', 'm4air', '--json'], '/tmp/repo') + + for (const call of logSpy.mock.calls) { + expect(JSON.parse(String(call[0])).ok).toBe(true) + } + }) +}) diff --git a/src/cli/index.ts b/src/cli/index.ts index c29bfff7060..9389113b195 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -31,6 +31,10 @@ function shouldIgnoreRemoteSelection(commandPath: string[]): boolean { commandPath[0] === 'account' || commandPath[0] === 'artifacts' || commandPath[0] === 'environment' || + // Why: `host list` answers "what can this machine target, and with what flag". Half of that + // answer (paired servers) is read from this machine's own pairing store and cannot be routed, + // so routing the other half produced one listing describing two machines at once. + commandPath[0] === 'host' || commandPath[0] === 'serve' || commandPath[0] === 'agent' || commandPath[0] === 'vm' || diff --git a/src/cli/remote-selection-flag-rejection.ts b/src/cli/remote-selection-flag-rejection.ts new file mode 100644 index 00000000000..e2609fa604a --- /dev/null +++ b/src/cli/remote-selection-flag-rejection.ts @@ -0,0 +1,29 @@ +import { RuntimeClientError } from './runtime/types' + +/** + * The flags that pick which runtime answers a command. `shouldIgnoreRemoteSelection` + * in `src/cli/index.ts` pins some command families to the local runtime, which drops + * these silently — so every pinned family pairs the pin with this rejection instead. + */ +export const REMOTE_SELECTION_FLAGS = ['environment', 'pairing-code'] as const + +/** + * Fails a pinned command that was given a runtime selector, rather than answering + * for a machine the caller did not name. `suffix` completes "`--` does not + * retarget …" and should say what the command answers for and where to run it. + */ +export function rejectRemoteSelectionFlags( + flags: ReadonlyMap, + suffix: string, + data?: Record +): void { + for (const flag of REMOTE_SELECTION_FLAGS) { + if (flags.has(flag)) { + throw new RuntimeClientError( + 'invalid_argument', + `\`--${flag}\` does not retarget ${suffix}`, + data + ) + } + } +} diff --git a/src/cli/specs/environment.ts b/src/cli/specs/environment.ts index d6ceb795028..7bf90615270 100644 --- a/src/cli/specs/environment.ts +++ b/src/cli/specs/environment.ts @@ -10,7 +10,8 @@ export const ENVIRONMENT_COMMAND_SPECS: CommandSpec[] = [ notes: [ 'Answers "what can I target and what do I pass" in one place: this machine, the SSH targets registered on it, and the Orca servers paired with it.', 'The three kinds are reached differently. A paired Orca server is a connection, selected with --environment . An SSH target is a machine the connected Orca host reaches, selected with --host ssh:. Passing one where the other belongs is the most common way to get an empty or missing-host answer.', - "SSH targets are read from the Orca host you are currently connected to, so this lists that host's targets and not another server's." + "SSH targets are read from this machine's own Orca runtime, so this lists that machine's targets and not another server's. Run `orca host list` on the other machine to see the targets registered there.", + '--environment and --pairing-code are rejected rather than ignored: paired servers come from this machine\u2019s pairing store, so a routed answer would describe two machines at once.' ], examples: ['orca host list', 'orca host list --json'] }, @@ -25,7 +26,10 @@ export const ENVIRONMENT_COMMAND_SPECS: CommandSpec[] = [ path: ['environment', 'list'], summary: 'List saved Orca runtime environments', usage: 'orca environment list [--json]', - allowedFlags: [...GLOBAL_FLAGS] + allowedFlags: [...GLOBAL_FLAGS], + notes: [ + 'Answers from this machine\u2019s pairing store. --environment and --pairing-code are rejected rather than ignored, because there is no other host that could answer.' + ] }, { path: ['environment', 'show'], From 95eed528012dfd44b7244b3ef17beaefb8390568 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:43:09 -0700 Subject: [PATCH 15/76] fix(cli): report which hosts a worktree listing covered, and stop the cap starving remote ones (#18417) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `orca worktree list` returned zero of 24 SSH worktrees at the default limit (#18104). Rows are resolved repo by repo, so every SSH repo's rows land contiguously at the end of the fleet order — the 24 remote rows sat at indices 496-520 of 521 and a plain `slice(0, 200)` never reached them. The omission was not fully silent: text output printed `truncated: showing 200 of 521` and JSON carried `totalCount` / `truncated`. What was missing is that the omission was *categorically every remote host* — no host column, no `hostScope`, nothing to distinguish "200 of 521" from "one host is entirely absent". Per docs/reference/ssh-execution-boundary.md, a listing that does not name its scope reads as absolute. Adopt the mechanism `terminal list` already has rather than inventing a second one: - `RuntimeTerminalListHostScope` becomes an alias of a shared `RuntimeListingHostScope`, now also carried (optional, so old hosts are unaffected) on `worktree.list` and `worktree.ps` results. - `src/shared/host-balanced-listing-page.ts` round-robins the row cap across hosts and returns the survivors in the caller's original relative order, so the page stays a subsequence of the unbounded listing and nothing downstream re-sorts. An uncapped listing is returned unchanged. - `worktree list` / `worktree ps` text output gains a `host=` column and the same trailing `scope:` line `terminal list` prints. Third defect, same mechanism: `hostScope.omittedHostIds` is built from the runtime's own bookkeeping, so it names `runtime:` ids for servers that are no longer paired — 6 of 9 in the recorded QA run hard-error when queried. Since `hostScope` is *the* documented way to complete a partial listing, that makes the mechanism unreliable for its intended use. Annotate rather than filter. Dropping an id would shrink what the listing admits it did not cover, and the boundary doc requires a listing to name its gaps — the gap is real whether or not this machine can name the host that owns it. `src/cli/omitted-host-scope-selectors.ts` resolves each omitted id against this machine's pairing store and the runtime's SSH-target registry and attaches the exact flag that reaches it, or `null` marked "not selectable from this machine". This is a client-side annotation: nothing new goes over the wire, it answers "can I select it" and never "is it up", and the SSH round trip is only paid when an `ssh:` host was actually omitted. No `--host` filter was added; the host column plus scope line covers the reported need without a new selector axis. --- src/cli/handlers/terminal.ts | 20 +- src/cli/handlers/worktree.ts | 24 +- ...index-omitted-host-scope-selectors.test.ts | 246 ++++++++++++++++++ .../index-terminal-list-host-scope.test.ts | 5 +- src/cli/omitted-host-scope-selectors.ts | 126 +++++++++ src/cli/specs/core.ts | 7 +- src/cli/specs/worktree-listing-scope-notes.ts | 6 + src/cli/terminal-format.ts | 20 +- src/cli/workspace-format.ts | 27 +- .../runtime/orca-runtime-get-worktree-ps.ts | 17 +- .../orca-runtime-stop-requested-pty-ids.ts | 3 +- ...creation-and-orchestration-part-04.spec.ts | 2 + ...me-managed-worktree-metadata-sweep.test.ts | 3 +- .../runtime-managed-worktree-queries.test.ts | 3 +- .../runtime-managed-worktree-queries.ts | 13 +- .../runtime/worktree-list-host-scope.test.ts | 170 ++++++++++++ .../runtime/worktree-listing-host-scope.ts | 64 +++++ .../runtime/worktree-ps-host-scope.test.ts | 131 ++++++++++ src/shared/host-balanced-listing-page.ts | 49 ++++ src/shared/runtime-listing-host-scope.ts | 12 + src/shared/runtime-terminal-contracts.ts | 7 +- src/shared/runtime-worktree-contracts.ts | 5 + 22 files changed, 895 insertions(+), 65 deletions(-) create mode 100644 src/cli/index-omitted-host-scope-selectors.test.ts create mode 100644 src/cli/omitted-host-scope-selectors.ts create mode 100644 src/cli/specs/worktree-listing-scope-notes.ts create mode 100644 src/main/runtime/worktree-list-host-scope.test.ts create mode 100644 src/main/runtime/worktree-listing-host-scope.ts create mode 100644 src/main/runtime/worktree-ps-host-scope.test.ts create mode 100644 src/shared/host-balanced-listing-page.ts create mode 100644 src/shared/runtime-listing-host-scope.ts diff --git a/src/cli/handlers/terminal.ts b/src/cli/handlers/terminal.ts index 72e59b86655..3ff6142275a 100644 --- a/src/cli/handlers/terminal.ts +++ b/src/cli/handlers/terminal.ts @@ -32,6 +32,10 @@ import { getOptionalStringFlag, getRequiredStringFlag } from '../flags' +import { + annotateOmittedHostScope, + type WithAnnotatedHostScope +} from '../omitted-host-scope-selectors' import { RuntimeClientError } from '../runtime-client' import { getBrowserWorktreeSelector, @@ -90,12 +94,16 @@ const terminalFocusHandler: CommandHandler = async ({ flags, client, cwd, json } export const TERMINAL_HANDLERS: Record = { 'terminal list': async ({ flags, client, cwd, json }) => { - const result = await client.call('terminal.list', { - worktree: await getOptionalWorktreeSelector(flags, 'worktree', cwd, client), - limit: getOptionalPositiveIntegerFlag(flags, 'limit'), - // Why: agent JSON calls dominate; topology stays available through an explicit opt-in. - includeVisualLayouts: !json || flags.has('include-visual-layouts') - }) + const result = await client.call>( + 'terminal.list', + { + worktree: await getOptionalWorktreeSelector(flags, 'worktree', cwd, client), + limit: getOptionalPositiveIntegerFlag(flags, 'limit'), + // Why: agent JSON calls dominate; topology stays available through an explicit opt-in. + includeVisualLayouts: !json || flags.has('include-visual-layouts') + } + ) + await annotateOmittedHostScope(client, result.result) printResult(result, json, formatTerminalList) }, 'terminal show': async ({ flags, client, cwd, json }) => { diff --git a/src/cli/handlers/worktree.ts b/src/cli/handlers/worktree.ts index 484bcf9ea6d..262599234f0 100644 --- a/src/cli/handlers/worktree.ts +++ b/src/cli/handlers/worktree.ts @@ -7,6 +7,10 @@ import type { } from '../../shared/runtime-types' import type { CommandHandler } from '../dispatch' import { formatWorktreeList, formatWorktreePs, formatWorktreeShow, printResult } from '../format' +import { + annotateOmittedHostScope, + type WithAnnotatedHostScope +} from '../omitted-host-scope-selectors' import { RuntimeClientError } from '../runtime-client' import { getOptionalNullableNumberFlag, @@ -171,16 +175,22 @@ async function getCreateRepoSelector( export const WORKTREE_HANDLERS: Record = { 'worktree ps': async ({ flags, client, json }) => { - const result = await client.call('worktree.ps', { - limit: getOptionalPositiveIntegerFlag(flags, 'limit') - }) + const result = await client.call>( + 'worktree.ps', + { limit: getOptionalPositiveIntegerFlag(flags, 'limit') } + ) + await annotateOmittedHostScope(client, result.result) printResult(result, json, formatWorktreePs) }, 'worktree list': async ({ flags, client, json }) => { - const result = await client.call('worktree.list', { - repo: getOptionalStringFlag(flags, 'repo'), - limit: getOptionalPositiveIntegerFlag(flags, 'limit') - }) + const result = await client.call>( + 'worktree.list', + { + repo: getOptionalStringFlag(flags, 'repo'), + limit: getOptionalPositiveIntegerFlag(flags, 'limit') + } + ) + await annotateOmittedHostScope(client, result.result) printResult(result, json, formatWorktreeList) }, 'worktree show': async ({ flags, client, cwd, json }) => { diff --git a/src/cli/index-omitted-host-scope-selectors.test.ts b/src/cli/index-omitted-host-scope-selectors.test.ts new file mode 100644 index 00000000000..1fbd77289f5 --- /dev/null +++ b/src/cli/index-omitted-host-scope-selectors.test.ts @@ -0,0 +1,246 @@ +import { describe, expect, it, vi } from 'vitest' + +const { + callMock, + runtimeClientConstructorMock, + serveOrcaAppMock, + getDefaultUserDataPathMock, + addEnvironmentFromPairingCodeMock, + listEnvironmentsMock, + spawnMock +} = vi.hoisted(() => ({ + callMock: vi.fn(), + runtimeClientConstructorMock: vi.fn(), + serveOrcaAppMock: vi.fn(), + getDefaultUserDataPathMock: vi.fn(() => '/tmp/orca-user-data'), + addEnvironmentFromPairingCodeMock: vi.fn(), + listEnvironmentsMock: vi.fn(), + spawnMock: vi.fn() +})) + +vi.mock('./runtime-client', async () => { + const { createRuntimeClientModuleMock } = await import('./index-test-harness.js') + return createRuntimeClientModuleMock({ + callMock, + runtimeClientConstructorMock, + serveOrcaAppMock, + getDefaultUserDataPathMock + }) +}) + +vi.mock('./runtime/environments', () => ({ + addEnvironmentFromPairingCode: addEnvironmentFromPairingCodeMock, + listEnvironments: listEnvironmentsMock, + removeEnvironment: vi.fn(), + resolveEnvironment: vi.fn() +})) + +vi.mock('child_process', async () => { + const { createChildProcessModuleMock } = await import('./index-test-harness.js') + return createChildProcessModuleMock(spawnMock) +}) + +import { main } from './index' +import { okFixture, queueFixtures } from './test-fixtures' +import { pairRuntimeEnvironment, useWorktreeAwarenessEnvironment } from './index-test-harness' + +const TERMINAL_ROW = { + handle: 'term_1', + ptyId: 'pty-1', + worktreeId: 'repo::/wt', + worktreePath: '/wt', + branch: 'main', + tabId: 'tab-1', + leafId: 'leaf-1', + title: 'worker', + connected: true, + writable: true, + lastOutputAt: null, + preview: '', + executionHostId: 'local' +} + +describe('omittedHostIds selector annotation', () => { + useWorktreeAwarenessEnvironment({ + callMock, + serveOrcaAppMock, + getDefaultUserDataPathMock, + addEnvironmentFromPairingCodeMock, + listEnvironmentsMock, + spawnMock + }) + + it('marks a stale runtime host that no caller can select', async () => { + // Why: `omittedHostIds` is built from the runtime's own bookkeeping, so it names `runtime:` + // ids for servers that are no longer paired. An agent looping over the list to complete a + // partial listing hard-errors on those — 6 of 9 in the recorded QA run. + pairRuntimeEnvironment(listEnvironmentsMock, 'env-paired', 'm4air') + queueFixtures( + callMock, + okFixture('req_terminal_list', { + terminals: [TERMINAL_ROW], + totalCount: 1, + truncated: false, + hostScope: { + hostIds: ['local'], + omittedHostIds: ['runtime:env-paired', 'runtime:env-retired'] + } + }) + ) + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) + + await main(['terminal', 'list', '--json'], '/tmp/repo') + + const printed = JSON.parse(String(logSpy.mock.calls[0]?.[0])) + expect(printed.result.hostScope.omittedHostIds).toEqual([ + 'runtime:env-paired', + 'runtime:env-retired' + ]) + expect(printed.result.hostScope.omittedHostSelectors).toEqual([ + { hostId: 'runtime:env-paired', selector: '--environment m4air' }, + { hostId: 'runtime:env-retired', selector: null } + ]) + }) + + it('says which omitted hosts are not selectable in the human listing', async () => { + pairRuntimeEnvironment(listEnvironmentsMock, 'env-paired', 'm4air') + queueFixtures( + callMock, + okFixture('req_terminal_list', { + terminals: [TERMINAL_ROW], + totalCount: 1, + truncated: false, + hostScope: { + hostIds: ['local'], + omittedHostIds: ['runtime:env-paired', 'runtime:env-retired'] + } + }) + ) + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) + + await main(['terminal', 'list'], '/tmp/repo') + + const printed = String(logSpy.mock.calls[0]?.[0]) + expect(printed).toContain('runtime:env-paired (--environment m4air)') + expect(printed).toContain('runtime:env-retired (not selectable from this machine)') + }) + + it('resolves an omitted SSH host against the targets the runtime actually knows', async () => { + listEnvironmentsMock.mockReturnValue([]) + queueFixtures( + callMock, + okFixture('req_terminal_list', { + terminals: [TERMINAL_ROW], + totalCount: 1, + truncated: false, + hostScope: { hostIds: ['local'], omittedHostIds: ['ssh:box-1', 'ssh:box-gone'] } + }), + okFixture('req_ssh_targets', { targets: [{ id: 'box-1', label: 'openclaw' }] }) + ) + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) + + await main(['terminal', 'list', '--json'], '/tmp/repo') + + const printed = JSON.parse(String(logSpy.mock.calls[0]?.[0])) + expect(printed.result.hostScope.omittedHostSelectors).toEqual([ + { hostId: 'ssh:box-1', selector: '--host ssh:box-1' }, + { hostId: 'ssh:box-gone', selector: null } + ]) + }) + + it('never keeps a host id out of omittedHostIds', async () => { + // Why: filtering the unreachable ones would shrink what the listing admits it did not cover. + // The gap is real whether or not this machine can name the host that owns it. + listEnvironmentsMock.mockReturnValue([]) + queueFixtures( + callMock, + okFixture('req_terminal_list', { + terminals: [], + totalCount: 0, + truncated: false, + hostScope: { hostIds: [], omittedHostIds: ['runtime:env-retired'] } + }) + ) + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) + + await main(['terminal', 'list', '--json'], '/tmp/repo') + + const printed = JSON.parse(String(logSpy.mock.calls[0]?.[0])) + expect(printed.result.hostScope.omittedHostIds).toEqual(['runtime:env-retired']) + }) + + it('costs no extra round trip when nothing was omitted', async () => { + queueFixtures( + callMock, + okFixture('req_terminal_list', { + terminals: [TERMINAL_ROW], + totalCount: 1, + truncated: false, + hostScope: { hostIds: ['local'], omittedHostIds: [] } + }) + ) + vi.spyOn(console, 'log').mockImplementation(() => {}) + + await main(['terminal', 'list', '--json'], '/tmp/repo') + + expect(callMock).toHaveBeenCalledTimes(1) + }) +}) + +describe('worktree listings report their host coverage', () => { + useWorktreeAwarenessEnvironment({ + callMock, + serveOrcaAppMock, + getDefaultUserDataPathMock, + addEnvironmentFromPairingCodeMock, + listEnvironmentsMock, + spawnMock + }) + + it('prints a host column and the scope line for `worktree list`', async () => { + listEnvironmentsMock.mockReturnValue([]) + queueFixtures( + callMock, + okFixture('req_worktree_list', { + worktrees: [ + { + id: 'repo-ssh::/remote/wt', + branch: 'main', + path: '/remote/wt', + hostId: 'ssh:box-1', + displayName: 'remote', + parentWorktreeId: null, + childWorktreeIds: [], + linkedIssue: null, + comment: '' + } + ], + totalCount: 521, + truncated: true, + hostScope: { hostIds: ['ssh:box-1'], omittedHostIds: ['runtime:env-retired'] } + }) + ) + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) + + await main(['worktree', 'list'], '/tmp/repo') + + const printed = String(logSpy.mock.calls[0]?.[0]) + expect(printed).toContain('host=ssh:box-1') + expect(printed).toContain('scope: ssh:box-1') + expect(printed).toContain('runtime:env-retired (not selectable from this machine)') + expect(printed).toContain('truncated: showing 1 of 521') + }) + + it('does not claim a scope for `worktree ps` when the host reported none', async () => { + queueFixtures( + callMock, + okFixture('req_worktree_ps', { worktrees: [], totalCount: 0, truncated: false }) + ) + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) + + await main(['worktree', 'ps'], '/tmp/repo') + + const printed = String(logSpy.mock.calls[0]?.[0]) + expect(printed).toContain('scope: unverifiable') + }) +}) diff --git a/src/cli/index-terminal-list-host-scope.test.ts b/src/cli/index-terminal-list-host-scope.test.ts index b38cd71451f..04b486df671 100644 --- a/src/cli/index-terminal-list-host-scope.test.ts +++ b/src/cli/index-terminal-list-host-scope.test.ts @@ -88,7 +88,10 @@ describe('orca terminal list host scope', () => { expect(printed.result.terminals[0].executionHostId).toBe('ssh:box-1') expect(printed.result.hostScope).toEqual({ hostIds: ['ssh:box-1'], - omittedHostIds: ['local'] + omittedHostIds: ['local'], + // The CLI annotates each omitted host with the flag that reaches it; see + // index-omitted-host-scope-selectors.test.ts. + omittedHostSelectors: [{ hostId: 'local', selector: '--host local' }] }) }) diff --git a/src/cli/omitted-host-scope-selectors.ts b/src/cli/omitted-host-scope-selectors.ts new file mode 100644 index 00000000000..2666b116375 --- /dev/null +++ b/src/cli/omitted-host-scope-selectors.ts @@ -0,0 +1,126 @@ +import { + parseExecutionHostId, + type ExecutionHostId, + type ParsedExecutionHost +} from '../shared/execution-host' +import type { RuntimeListingHostScope } from '../shared/runtime-listing-host-scope' +import { + findEnvironmentByName, + findSshTargetByName, + listSshTargets, + type SshTargetSummary +} from './host-selector-alternatives' +import type { RuntimeClient } from './runtime-client' + +export type OmittedHostScopeSelector = { + hostId: ExecutionHostId + /** The flag that routes a follow-up query to this host, or null when it names nothing here. */ + selector: string | null +} + +/** A host scope annotated on this machine. The runtime never sends `omittedHostSelectors`. */ +export type ListingHostScopeWithSelectors = RuntimeListingHostScope & { + omittedHostSelectors?: OmittedHostScopeSelector[] +} + +export type WithAnnotatedHostScope = Omit & { + hostScope?: ListingHostScopeWithSelectors +} + +/** + * Resolves how to reach each host a listing did not cover. + * + * `hostScope` is the documented way to complete a partial listing, but `omittedHostIds` is built + * from the runtime's own bookkeeping — repos, folder workspaces, and workspace sessions — so it + * names `runtime:` ids for servers that are no longer paired. An agent looping over the list to + * finish the job hard-errors on those. + * + * The ids are kept rather than filtered: dropping one would shrink what the listing admits it did + * not cover, and `docs/reference/ssh-execution-boundary.md` requires a listing to name its gaps. + * A `null` selector marks the ones this machine cannot name, which is the part a caller needs. + * Only the local pairing store and SSH-target registry are consulted, so this answers "can I + * select it", never "is it up" — no host is claimed live or exited on this path. + */ +export async function resolveOmittedHostScopeSelectors( + client: RuntimeClient, + omittedHostIds: readonly ExecutionHostId[] +): Promise { + const parsed = omittedHostIds.map((hostId) => ({ + hostId, + host: parseExecutionHostId(hostId) + })) + const environments = parsed.some((entry) => entry.host?.kind === 'runtime') + ? await listPairedEnvironments() + : [] + // Why: SSH targets need a round trip, so only pay for it when an ssh host was actually omitted. + const sshTargets = parsed.some((entry) => entry.host?.kind === 'ssh') + ? await listSshTargets(client) + : [] + return parsed.map(({ hostId, host }) => ({ + hostId, + selector: resolveSelector(host, environments, sshTargets) + })) +} + +async function listPairedEnvironments(): Promise<{ id: string; name: string }[]> { + const [{ listEnvironments }, { getDefaultUserDataPath }] = await Promise.all([ + import('./runtime/environments.js'), + import('./runtime-client.js') + ]) + return listEnvironments(getDefaultUserDataPath()).map((environment) => ({ + id: environment.id, + name: environment.name + })) +} + +function resolveSelector( + host: ParsedExecutionHost | null, + environments: readonly { id: string; name: string }[], + sshTargets: readonly SshTargetSummary[] +): string | null { + if (host?.kind === 'local') { + return '--host local' + } + if (host?.kind === 'ssh') { + return findSshTargetByName(sshTargets, host.targetId) ? `--host ssh:${host.targetId}` : null + } + if (host?.kind === 'runtime') { + const environment = findEnvironmentByName(environments, host.environmentId) + return environment ? `--environment ${environment.name}` : null + } + return null +} + +/** Renders a scope line; an absent scope means the host never reported one, not full coverage. */ +export function formatListingHostScope(scope: ListingHostScopeWithSelectors | undefined): string { + if (!scope) { + return 'scope: unverifiable — this host does not report which hosts it lists' + } + const covered = scope.hostIds.length > 0 ? scope.hostIds.join(', ') : 'none' + if (scope.omittedHostIds.length === 0) { + return `scope: ${covered}` + } + const selectorByHostId = new Map( + (scope.omittedHostSelectors ?? []).map((entry) => [entry.hostId, entry.selector]) + ) + const omitted = scope.omittedHostIds.map((hostId) => { + if (!selectorByHostId.has(hostId)) { + return hostId + } + const selector = selectorByHostId.get(hostId) + return selector ? `${hostId} (${selector})` : `${hostId} (not selectable from this machine)` + }) + return `scope: ${covered} — not covered: ${omitted.join(', ')}` +} + +/** Attaches the resolved selectors in place; a listing with no omitted hosts pays nothing. */ +export async function annotateOmittedHostScope( + client: RuntimeClient, + result: { hostScope?: ListingHostScopeWithSelectors } +): Promise { + const scope = result.hostScope + if (!scope || scope.omittedHostIds.length === 0) { + return + } + scope.omittedHostSelectors = await resolveOmittedHostScopeSelectors(client, scope.omittedHostIds) +} diff --git a/src/cli/specs/core.ts b/src/cli/specs/core.ts index 112d9528d2e..f2236ef86e9 100644 --- a/src/cli/specs/core.ts +++ b/src/cli/specs/core.ts @@ -1,5 +1,6 @@ import type { CommandSpec } from '../args' import { GLOBAL_FLAGS } from '../args' +import { WORKTREE_LISTING_SCOPE_NOTES } from './worktree-listing-scope-notes' import { SERVE_COMMAND_SPECS } from './serve' import { TERMINAL_CLOSE_COMMAND_SPEC } from './terminal-close' @@ -65,7 +66,8 @@ export const CORE_COMMAND_SPECS: CommandSpec[] = [ path: ['worktree', 'list'], summary: 'List Orca-managed worktrees', usage: 'orca worktree list [--repo ] [--limit ] [--json]', - allowedFlags: [...GLOBAL_FLAGS, 'repo', 'limit'] + allowedFlags: [...GLOBAL_FLAGS, 'repo', 'limit'], + notes: [...WORKTREE_LISTING_SCOPE_NOTES] }, { path: ['worktree', 'show'], @@ -180,7 +182,8 @@ export const CORE_COMMAND_SPECS: CommandSpec[] = [ path: ['worktree', 'ps'], summary: 'Show a compact orchestration summary across worktrees', usage: 'orca worktree ps [--limit ] [--json]', - allowedFlags: [...GLOBAL_FLAGS, 'limit'] + allowedFlags: [...GLOBAL_FLAGS, 'limit'], + notes: [...WORKTREE_LISTING_SCOPE_NOTES] }, { path: ['terminal', 'list'], diff --git a/src/cli/specs/worktree-listing-scope-notes.ts b/src/cli/specs/worktree-listing-scope-notes.ts new file mode 100644 index 00000000000..91449cc6584 --- /dev/null +++ b/src/cli/specs/worktree-listing-scope-notes.ts @@ -0,0 +1,6 @@ +/** Shared by `worktree list` and `worktree ps`, which report host coverage the same way. */ +export const WORKTREE_LISTING_SCOPE_NOTES: readonly string[] = [ + 'Each row carries the execution host that owns it (`host=`), and the trailing `scope:` line names every host the page covers plus the ones it does not.', + 'A host named under `not covered` may still have workspaces; an empty answer for it is not evidence that it has none. Each is annotated with the flag that reaches it, or marked not selectable from this machine.', + 'The row cap is shared across hosts, so a host whose rows sort last is not starved out of the page.' +] diff --git a/src/cli/terminal-format.ts b/src/cli/terminal-format.ts index edf4cbaa22c..e61a2e48b76 100644 --- a/src/cli/terminal-format.ts +++ b/src/cli/terminal-format.ts @@ -1,10 +1,10 @@ import { PTY_LIVE_NOTE, describeUnconfirmedStop } from '../shared/pty-liveness-verdict' import { structuredChatPtyWriteRefusalCopy } from '../shared/agent-session-pty-write-refusal-copy' +import { formatListingHostScope, type WithAnnotatedHostScope } from './omitted-host-scope-selectors' import type { RuntimeTerminalClose, RuntimeTerminalCreate, RuntimeTerminalFocus, - RuntimeTerminalListHostScope, RuntimeTerminalListResult, RuntimeTerminalVisualLayout, RuntimeTerminalVisualLayoutNode, @@ -18,8 +18,10 @@ import type { RuntimeTerminalWait } from '../shared/runtime-types' -export function formatTerminalList(result: RuntimeTerminalListResult): string { - const scope = formatTerminalListHostScope(result.hostScope) +export function formatTerminalList( + result: WithAnnotatedHostScope +): string { + const scope = formatListingHostScope(result.hostScope) if (result.terminals.length === 0) { return `No terminals listed.\n${scope}` } @@ -37,18 +39,6 @@ export function formatTerminalList(result: RuntimeTerminalListResult): string { : bodyWithScope } -// Why: a listing that does not say what it covers reads as absolute, and an -// absent scope means the host is too old to know — not that it covered everything. -function formatTerminalListHostScope(scope: RuntimeTerminalListHostScope | undefined): string { - if (!scope) { - return 'scope: unverifiable — this host does not report which hosts it lists' - } - const covered = scope.hostIds.length > 0 ? scope.hostIds.join(', ') : 'none' - const omitted = - scope.omittedHostIds.length > 0 ? ` — not covered: ${scope.omittedHostIds.join(', ')}` : '' - return `scope: ${covered}${omitted}` -} - function formatTerminalVisualLayouts( layouts: readonly RuntimeTerminalVisualLayout[] | undefined ): string | null { diff --git a/src/cli/workspace-format.ts b/src/cli/workspace-format.ts index 8cdfce86b74..51a0369978b 100644 --- a/src/cli/workspace-format.ts +++ b/src/cli/workspace-format.ts @@ -7,6 +7,7 @@ import type { RuntimeWorktreeRecord } from '../shared/runtime-types' import type { MemorySnapshot, WorktreeMemory } from '../shared/process-stats-types' +import { formatListingHostScope, type WithAnnotatedHostScope } from './omitted-host-scope-selectors' export function formatMemorySnapshot(snapshot: MemorySnapshot): string { const topWorktrees = [...snapshot.worktrees].sort((a, b) => b.memory - a.memory).slice(0, 10) @@ -130,19 +131,21 @@ export function formatEnvironment(environment: PublicKnownRuntimeEnvironment): s ].join('\n') } -export function formatWorktreePs(result: RuntimeWorktreePsResult): string { +export function formatWorktreePs(result: WithAnnotatedHostScope): string { + const scope = formatListingHostScope(result.hostScope) if (result.worktrees.length === 0) { - return 'No worktrees found.' + return `No worktrees found.\n${scope}` } const body = result.worktrees .map( (worktree) => - `${worktree.repo} ${worktree.branch} live:${worktree.liveTerminalCount} pty:${worktree.hasAttachedPty ? 'yes' : 'no'} unread:${worktree.unread ? 'yes' : 'no'}\n${worktree.path}${worktree.preview ? `\npreview: ${worktree.preview}` : ''}` + `${worktree.repo} ${worktree.branch} host=${worktree.hostId ?? 'unverifiable'} live:${worktree.liveTerminalCount} pty:${worktree.hasAttachedPty ? 'yes' : 'no'} unread:${worktree.unread ? 'yes' : 'no'}\n${worktree.path}${worktree.preview ? `\npreview: ${worktree.preview}` : ''}` ) .join('\n\n') + const bodyWithScope = `${body}\n\n${scope}` return result.truncated - ? `${body}\n\ntruncated: showing ${result.worktrees.length} of ${result.totalCount}` - : body + ? `${bodyWithScope}\ntruncated: showing ${result.worktrees.length} of ${result.totalCount}` + : bodyWithScope } export function formatRepoList(result: RuntimeRepoList): string { @@ -168,19 +171,23 @@ export function formatRepoRefs(result: RuntimeRepoSearchRefs): string { return result.truncated ? `${result.refs.join('\n')}\n\ntruncated: yes` : result.refs.join('\n') } -export function formatWorktreeList(result: RuntimeWorktreeListResult): string { +export function formatWorktreeList( + result: WithAnnotatedHostScope +): string { + const scope = formatListingHostScope(result.hostScope) if (result.worktrees.length === 0) { - return 'No worktrees found.' + return `No worktrees found.\n${scope}` } const body = result.worktrees .map((worktree) => { const childCount = worktree.childWorktreeIds?.length ?? 0 - return `${String(worktree.id)} ${String(worktree.branch)} ${String(worktree.path)}\ndisplayName: ${String(worktree.displayName ?? '')}\nparentWorktreeId: ${String(worktree.parentWorktreeId ?? 'null')}\nchildWorktreeIds: ${childCount > 0 ? worktree.childWorktreeIds.join(',') : '[]'}\nlinkedIssue: ${String(worktree.linkedIssue ?? 'null')}\ncomment: ${String(worktree.comment ?? '')}` + return `${String(worktree.id)} ${String(worktree.branch)} host=${String(worktree.hostId ?? 'unverifiable')} ${String(worktree.path)}\ndisplayName: ${String(worktree.displayName ?? '')}\nparentWorktreeId: ${String(worktree.parentWorktreeId ?? 'null')}\nchildWorktreeIds: ${childCount > 0 ? worktree.childWorktreeIds.join(',') : '[]'}\nlinkedIssue: ${String(worktree.linkedIssue ?? 'null')}\ncomment: ${String(worktree.comment ?? '')}` }) .join('\n\n') + const bodyWithScope = `${body}\n\n${scope}` return result.truncated - ? `${body}\n\ntruncated: showing ${result.worktrees.length} of ${result.totalCount}` - : body + ? `${bodyWithScope}\ntruncated: showing ${result.worktrees.length} of ${result.totalCount}` + : bodyWithScope } export function formatWorktreeShow(result: { worktree: RuntimeWorktreeRecord }): string { diff --git a/src/main/runtime/orca-runtime-get-worktree-ps.ts b/src/main/runtime/orca-runtime-get-worktree-ps.ts index 94fc77f8158..42c9c7ff6d3 100644 --- a/src/main/runtime/orca-runtime-get-worktree-ps.ts +++ b/src/main/runtime/orca-runtime-get-worktree-ps.ts @@ -1,7 +1,7 @@ // @ts-nocheck -- mechanically split from OrcaRuntimeService; behavior is covered by AST equivalence and characterization tests. import { OrcaRuntimeWithStructuredAgentSessionRecoverTuiOwner } from './orca-runtime-structured-agent-session-recover-tui-owner' import { DEFAULT_WORKTREE_PS_LIMIT } from './orca-runtime-postlude' -import type { RuntimeWorktreePsSummary } from '../../shared/runtime-types' +import type { RuntimeWorktreePsResult } from '../../shared/runtime-types' import { buildRuntimeWorktreePsSummaries } from './runtime-worktree-ps-summaries' import { buildRuntimeWorktreeSummaryPathIndex } from './runtime-worktree-summary-paths' import { @@ -15,6 +15,7 @@ import { enrichMissingRepoGitRemoteIdentities } from '../repo-git-remote-identit import { ensureStructuredAgentSessionHost as installStructuredAgentSessionHost } from './structured-agent-session-runtime' import { getProfileUserDataPath } from '../orca-profiles/profile-storage-paths' import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host' +import { buildWorktreeListingPage } from './worktree-listing-host-scope' import { resolveTuiAgentLaunchArgs, resolveTuiAgentLaunchEnv @@ -30,11 +31,7 @@ export class OrcaRuntimeWithGetWorktreePs extends OrcaRuntimeWithStructuredAgent async getWorktreePs( limit = DEFAULT_WORKTREE_PS_LIMIT, sourceDefaultsSupported = true - ): Promise<{ - worktrees: RuntimeWorktreePsSummary[] - totalCount: number - truncated: boolean - }> { + ): Promise { if (!Number.isInteger(limit) || limit <= 0) { throw new Error('invalid_limit') } @@ -111,11 +108,9 @@ export class OrcaRuntimeWithGetWorktreePs extends OrcaRuntimeWithStructuredAgent }) const sorted = [...summaries.values()].sort(compareWorktreePs) - return { - worktrees: sorted.slice(0, limit), - totalCount: sorted.length, - truncated: sorted.length > limit - } + // Why: the same cap starvation as worktree.list — a host whose rows all sort last gets no + // page at all, which is indistinguishable from it having no workspaces (#18104). + return buildWorktreeListingPage(sorted, limit, this.listKnownExecutionHostIds()) } listRepos(): Repo[] { diff --git a/src/main/runtime/orca-runtime-stop-requested-pty-ids.ts b/src/main/runtime/orca-runtime-stop-requested-pty-ids.ts index 278ccd28a74..346006cd5fd 100644 --- a/src/main/runtime/orca-runtime-stop-requested-pty-ids.ts +++ b/src/main/runtime/orca-runtime-stop-requested-pty-ids.ts @@ -136,7 +136,8 @@ export class OrcaRuntimeWithStopRequestedPtyIds extends OrcaRuntimeWithRuntimeId listResolved: () => this.listResolvedWorktrees(), resolveRepo: (selector) => this.resolveRepoSelector(selector), selectRepos: (selector) => this.selectReposBySelector(selector), - scanRepo: (repo) => this.listRepoWorktreesForResolution(repo) + scanRepo: (repo) => this.listRepoWorktreesForResolution(repo), + listKnownHostIds: () => this.listKnownExecutionHostIds() }) protected readonly ptyForegroundAgent = new RuntimePtyForegroundAgent({ diff --git a/src/main/runtime/orca-runtime-tests/mobile-creation-and-orchestration-part-04.spec.ts b/src/main/runtime/orca-runtime-tests/mobile-creation-and-orchestration-part-04.spec.ts index 2d6042c280e..965a1a1bc3f 100644 --- a/src/main/runtime/orca-runtime-tests/mobile-creation-and-orchestration-part-04.spec.ts +++ b/src/main/runtime/orca-runtime-tests/mobile-creation-and-orchestration-part-04.spec.ts @@ -168,6 +168,8 @@ describe('OrcaRuntimeService', () => { agents: [] } ], + // Why: the summary now names the hosts it covered; an absent scope would read as absolute. + hostScope: { hostIds: ['local'], omittedHostIds: [] }, totalCount: 1, truncated: false }) diff --git a/src/main/runtime/runtime-managed-worktree-metadata-sweep.test.ts b/src/main/runtime/runtime-managed-worktree-metadata-sweep.test.ts index 6df19517d64..4913b31bd7d 100644 --- a/src/main/runtime/runtime-managed-worktree-metadata-sweep.test.ts +++ b/src/main/runtime/runtime-managed-worktree-metadata-sweep.test.ts @@ -44,7 +44,8 @@ function queries( listResolved: async () => [], resolveRepo: async () => repo, selectRepos: () => [repo], - scanRepo: async () => ({ ok, worktrees: [...worktrees] }) + scanRepo: async () => ({ ok, worktrees: [...worktrees] }), + listKnownHostIds: () => [] }) } diff --git a/src/main/runtime/runtime-managed-worktree-queries.test.ts b/src/main/runtime/runtime-managed-worktree-queries.test.ts index 354b6653324..01df53cbd1d 100644 --- a/src/main/runtime/runtime-managed-worktree-queries.test.ts +++ b/src/main/runtime/runtime-managed-worktree-queries.test.ts @@ -46,7 +46,8 @@ function queries(store: RuntimeStore): RuntimeManagedWorktreeQueries { listResolved: async () => [], resolveRepo: async () => store.getRepos()[0]!, selectRepos: () => store.getRepos(), - scanRepo: async () => ({ ok: true, worktrees: [] }) + scanRepo: async () => ({ ok: true, worktrees: [] }), + listKnownHostIds: () => [] }) } diff --git a/src/main/runtime/runtime-managed-worktree-queries.ts b/src/main/runtime/runtime-managed-worktree-queries.ts index b0ed2bc4a3b..5a5812236af 100644 --- a/src/main/runtime/runtime-managed-worktree-queries.ts +++ b/src/main/runtime/runtime-managed-worktree-queries.ts @@ -1,7 +1,8 @@ import type { DetectedWorktreeListResult, Worktree } from '../../shared/worktree/types' import type { Repo } from '../../shared/repo-types' import type { RuntimeWorktreeListResult } from '../../shared/runtime-types' -import { getRepoExecutionHostId } from '../../shared/execution-host' +import { getRepoExecutionHostId, type ExecutionHostId } from '../../shared/execution-host' +import { buildWorktreeListingPage } from './worktree-listing-host-scope' import { readWorktreeMetaForHost } from '../persistence/host-qualified-worktree-meta' import { getRepoOwnedWorktreeMeta } from '../worktree-metadata-ownership' import type { WorktreeMeta } from '../../shared/worktree/meta-types' @@ -38,6 +39,8 @@ type Dependencies = { resolveRepo(selector: string): Promise selectRepos(selector: string): Repo[] scanRepo(repo: Repo): Promise + /** Hosts this runtime has repos or workspaces on, so a host with no rows is still named. */ + listKnownHostIds(): Iterable } /** @@ -100,11 +103,9 @@ export class RuntimeManagedWorktreeQueries { (!repoId || worktree.repoId === repoId) && this.isVisible(worktree, matchers.get(worktree.repoId), sourceDefaultsSupported) ) - return { - worktrees: worktrees.slice(0, limit), - totalCount: worktrees.length, - truncated: worktrees.length > limit - } + // Why: a `--repo` listing was scoped by the caller, so naming every configured host as + // omitted would report a gap the caller deliberately excluded. + return buildWorktreeListingPage(worktrees, limit, repoId ? [] : this.deps.listKnownHostIds()) } resolveRepoForConnection(selector: string, connectionId?: string | null): Promise { diff --git a/src/main/runtime/worktree-list-host-scope.test.ts b/src/main/runtime/worktree-list-host-scope.test.ts new file mode 100644 index 00000000000..e497028f4c4 --- /dev/null +++ b/src/main/runtime/worktree-list-host-scope.test.ts @@ -0,0 +1,170 @@ +import { describe, expect, it, vi } from 'vitest' +import type { ExecutionHostId } from '../../shared/execution-host' +import type { Repo } from '../../shared/repo-types' +import { selectHostBalancedPage } from '../../shared/host-balanced-listing-page' +import { RuntimeManagedWorktreeQueries } from './runtime-managed-worktree-queries' +import type { ResolvedWorktree } from './runtime-worktree-path-identity' +import type { RuntimeStore } from './runtime-store-contract' + +const LOCAL_REPO: Repo = { + id: 'repo-local', + path: '/workspace/app', + displayName: 'app', + badgeColor: '#000000', + addedAt: 1 +} + +const SSH_REPO: Repo = { + ...LOCAL_REPO, + id: 'repo-ssh', + connectionId: 'box-1', + displayName: 'app (remote)' +} + +const settings = { + workspaceDir: '/worktrees', + nestWorkspaces: true, + refreshLocalBaseRefOnWorktreeCreate: false, + branchPrefix: 'none', + branchPrefixCustom: '' +} + +function worktree(repoId: string, path: string, hostId: string): ResolvedWorktree { + return { + id: `${repoId}::${path}`, + repoId, + path, + branch: 'main', + hostId, + displayName: path, + comment: '', + linkedIssue: null, + parentWorktreeId: null, + childWorktreeIds: [], + lineage: null, + git: { path, head: 'abc', branch: 'main', isBare: false, isMainWorktree: false } + } as unknown as ResolvedWorktree +} + +/** The reproduced shape from #18104: every remote row lands contiguously at the end. */ +function fleet(localCount: number, sshCount: number): ResolvedWorktree[] { + return [ + ...Array.from({ length: localCount }, (_, index) => + worktree(LOCAL_REPO.id, `/worktrees/local-${index}`, 'local') + ), + ...Array.from({ length: sshCount }, (_, index) => + worktree(SSH_REPO.id, `/remote/wt-${index}`, 'ssh:box-1') + ) + ] +} + +function queries( + resolved: ResolvedWorktree[], + knownHostIds: ExecutionHostId[] = ['local', 'ssh:box-1'] +): RuntimeManagedWorktreeQueries { + const store = { + getRepos: () => [LOCAL_REPO, SSH_REPO], + getRepo: () => LOCAL_REPO, + getAllWorktreeMeta: () => ({}), + getWorktreeMeta: () => undefined, + setWorktreeMeta: vi.fn(), + getAllWorktreeLineage: () => ({}), + getSettings: () => settings + } as unknown as RuntimeStore + return new RuntimeManagedWorktreeQueries({ + getStore: () => store, + listResolved: async () => resolved, + resolveRepo: async () => SSH_REPO, + selectRepos: () => [SSH_REPO], + scanRepo: async () => ({ ok: true, worktrees: [] }), + listKnownHostIds: () => knownHostIds + }) +} + +describe('worktree.list host coverage under the row cap', () => { + it('returns remote rows that sit entirely past the cap', async () => { + // Why #18104: 497 local + 24 SSH rows, SSH at indices 496-520, and a 200-row cap returned + // `{local: 200}` — zero of 24 remote worktrees, with nothing saying the gap was a whole host. + const result = await queries(fleet(497, 24)).list(undefined, 200) + + expect(result.totalCount).toBe(521) + expect(result.truncated).toBe(true) + expect(result.worktrees).toHaveLength(200) + const remote = result.worktrees.filter((row) => row.hostId === 'ssh:box-1') + expect(remote).toHaveLength(24) + expect(result.hostScope).toEqual({ hostIds: ['local', 'ssh:box-1'], omittedHostIds: [] }) + }) + + it('keeps the page a subsequence of the unbounded listing', async () => { + // Why: balancing decides which rows survive the cap, never how the survivors are ordered. + const resolved = fleet(497, 24) + const result = await queries(resolved).list(undefined, 200) + + const positions = result.worktrees.map((row) => resolved.findIndex((it) => it.id === row.id)) + expect(positions).toEqual([...positions].sort((left, right) => left - right)) + }) + + it('names a configured host that contributed no rows at all', async () => { + // Why: a repo whose scan failed contributes zero rows exactly like a host with no worktrees. + // docs/reference/ssh-execution-boundary.md forbids the listing from reading as absolute there. + const result = await queries(fleet(3, 0), ['local', 'ssh:box-1', 'runtime:paired']).list( + undefined, + 200 + ) + + expect(result.hostScope).toEqual({ + hostIds: ['local'], + omittedHostIds: ['runtime:paired', 'ssh:box-1'] + }) + }) + + it('does not report configured hosts as omitted from a --repo listing', async () => { + // Why: the caller scoped this themselves, so naming the hosts they excluded is noise. + const result = await queries(fleet(0, 5)).list('id:repo-ssh', 200) + + expect(result.hostScope).toEqual({ hostIds: ['ssh:box-1'], omittedHostIds: [] }) + }) + + it('leaves an uncapped listing byte-identical', async () => { + const resolved = fleet(4, 2) + const result = await queries(resolved).list(undefined, 200) + + expect(result.worktrees.map((row) => row.id)).toEqual(resolved.map((row) => row.id)) + expect(result.truncated).toBe(false) + }) +}) + +describe('selectHostBalancedPage', () => { + it('gives every host a share of the cap rather than filling it from the first', () => { + const rows = [ + ...Array.from({ length: 10 }, (_, index) => ({ host: 'local', index })), + ...Array.from({ length: 10 }, (_, index) => ({ host: 'ssh:box-1', index: index + 10 })) + ] + + const page = selectHostBalancedPage(rows, 4, (row) => row.host) + + expect(page.map((row) => row.host)).toEqual(['local', 'local', 'ssh:box-1', 'ssh:box-1']) + }) + + it('fills the cap from the remaining hosts when one runs out of rows', () => { + const rows = [ + { host: 'local', id: 'a' }, + { host: 'local', id: 'b' }, + { host: 'local', id: 'c' }, + { host: 'ssh:box-1', id: 'd' } + ] + + const page = selectHostBalancedPage(rows, 3, (row) => row.host) + + expect(page.map((row) => row.id)).toEqual(['a', 'b', 'd']) + }) + + it('buckets rows with no host together instead of dropping them', () => { + const rows = [{ id: 'a' }, { id: 'b' }, { id: 'c' }] + + expect(selectHostBalancedPage(rows, 2, () => undefined).map((row) => row.id)).toEqual([ + 'a', + 'b' + ]) + }) +}) diff --git a/src/main/runtime/worktree-listing-host-scope.ts b/src/main/runtime/worktree-listing-host-scope.ts new file mode 100644 index 00000000000..99650882670 --- /dev/null +++ b/src/main/runtime/worktree-listing-host-scope.ts @@ -0,0 +1,64 @@ +import type { ExecutionHostId } from '../../shared/execution-host' +import { selectHostBalancedPage } from '../../shared/host-balanced-listing-page' +import type { RuntimeListingHostScope } from '../../shared/runtime-listing-host-scope' + +/** + * Applies a worktree listing's row cap and reports which hosts the resulting page covers. + * + * Rows are resolved repo by repo, so every SSH repo's rows land contiguously at the end of the + * fleet order: 24 remote worktrees sat at indices 496-520 of 521 and a 200-row cap returned zero + * of them (#18104). Balancing the page across hosts fixes the starvation; the scope is what makes + * the remaining gap legible, because a host with no rows in the page is otherwise indistinguishable + * from a host with no worktrees — which `docs/reference/ssh-execution-boundary.md` forbids a + * listing from implying. + */ +export function buildWorktreeListingPage( + rows: readonly TRow[], + limit: number, + knownHostIds: Iterable +): { + worktrees: TRow[] + hostScope: RuntimeListingHostScope + totalCount: number + truncated: boolean +} { + const page = selectHostBalancedPage(rows, limit, (row) => row.hostId) + return { + worktrees: page, + hostScope: buildWorktreeListingHostScope({ + pageHostIds: page.map((row) => row.hostId), + matchedHostIds: rows.map((row) => row.hostId), + knownHostIds + }), + totalCount: rows.length, + truncated: rows.length > limit + } +} + +/** + * The worktree-listing counterpart of `buildTerminalListHostScope`: names the hosts the returned + * page covers, and every host it does not — including a configured repo whose scan failed, which + * contributes zero rows exactly like a host with no worktrees. + */ +export function buildWorktreeListingHostScope(args: { + /** Hosts of the rows actually returned. */ + pageHostIds: Iterable + /** Hosts of every row that matched, including those the cap dropped. */ + matchedHostIds: Iterable + /** Hosts this runtime has configured repos or workspaces on, even if they contributed no rows. */ + knownHostIds: Iterable +}): RuntimeListingHostScope { + const covered = new Set() + for (const hostId of args.pageHostIds) { + if (hostId) { + covered.add(hostId) + } + } + const omitted = new Set() + for (const hostId of [...args.matchedHostIds, ...args.knownHostIds]) { + if (hostId && !covered.has(hostId)) { + omitted.add(hostId) + } + } + return { hostIds: [...covered].sort(), omittedHostIds: [...omitted].sort() } +} diff --git a/src/main/runtime/worktree-ps-host-scope.test.ts b/src/main/runtime/worktree-ps-host-scope.test.ts new file mode 100644 index 00000000000..cf718cd267f --- /dev/null +++ b/src/main/runtime/worktree-ps-host-scope.test.ts @@ -0,0 +1,131 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const electronMocks = vi.hoisted(() => { + const ipcMain = { + on: vi.fn(() => ipcMain), + removeListener: vi.fn(() => ipcMain), + emit: vi.fn(() => true) + } + return { + BrowserWindow: { fromId: vi.fn((): unknown => null) }, + webContents: { fromId: vi.fn((): unknown => null) }, + ipcMain, + app: { getPath: vi.fn(() => '/tmp'), isPackaged: false } + } +}) +vi.mock('electron', () => electronMocks) + +const getSshGitProviderMock = vi.hoisted(() => vi.fn()) +vi.mock('../providers/ssh-git-dispatch', () => ({ + getSshGitProvider: getSshGitProviderMock, + getSshGitProviderGeneration: vi.fn(() => 0), + SSH_GIT_PROVIDER_UNAVAILABLE_MESSAGE: 'unavailable', + requireSshGitProvider: (connectionId: string) => getSshGitProviderMock(connectionId) +})) + +const listWorktreesStrictMock = vi.hoisted(() => vi.fn()) +vi.mock('../git/worktree', async (importOriginal) => ({ + ...(await importOriginal>()), + listWorktreesStrict: listWorktreesStrictMock +})) + +import { OrcaRuntimeService } from './orca-runtime' + +const LOCAL_REPO_ID = 'repo-local' +const LOCAL_REPO_PATH = '/Users/me/dev/app' +const SSH_REPO_ID = 'repo-ssh' +const SSH_REPO_PATH = '/home/user/app' +const SSH_CONNECTION_ID = 'box-1' + +function gitWorktree(path: string, isMain = false) { + return { path, head: 'abc', branch: 'main', isBare: false, isMainWorktree: isMain } +} + +/** Local rows sort ahead of the remote ones, mirroring the fleet order that starves the cap. */ +function makeStore() { + const metaById: Record = {} + return { + getRepo: (id: string) => + makeStore() + .getRepos() + .find((repo) => repo.id === id), + getRepos: () => [ + { + id: LOCAL_REPO_ID, + path: LOCAL_REPO_PATH, + displayName: 'app', + badgeColor: 'blue', + addedAt: 1 + }, + { + id: SSH_REPO_ID, + path: SSH_REPO_PATH, + displayName: 'app remote', + badgeColor: 'blue', + addedAt: 2, + connectionId: SSH_CONNECTION_ID + } + ], + getAllWorktreeMeta: () => metaById, + getWorktreeMeta: (id: string) => metaById[id], + setWorktreeMeta: (id: string, meta: Record) => { + metaById[id] = { ...(metaById[id] as object), ...meta } + return metaById[id] + }, + removeWorktreeMeta: () => {}, + getAllWorktreeLineage: () => ({}), + getAllWorkspaceLineage: () => ({}), + removeWorktreeLineage: vi.fn(), + removeWorkspaceLineage: vi.fn(), + getGitHubCache: () => undefined as never, + getSettings: () => ({ + workspaceDir: '/tmp/workspaces', + nestWorkspaces: false, + refreshLocalBaseRefOnWorktreeCreate: false, + branchPrefix: 'none', + branchPrefixCustom: '' + }), + getProjects: () => [] + } +} + +describe('worktree.ps host coverage', () => { + beforeEach(() => { + getSshGitProviderMock.mockReset() + listWorktreesStrictMock.mockReset() + listWorktreesStrictMock.mockResolvedValue([ + gitWorktree(LOCAL_REPO_PATH, true), + gitWorktree(`${LOCAL_REPO_PATH}-a`), + gitWorktree(`${LOCAL_REPO_PATH}-b`), + gitWorktree(`${LOCAL_REPO_PATH}-c`) + ]) + getSshGitProviderMock.mockReturnValue({ + listWorktrees: vi.fn(async () => [ + gitWorktree(SSH_REPO_PATH, true), + gitWorktree(`${SSH_REPO_PATH}-a`) + ]) + }) + }) + + it('names every host the page covers', async () => { + const runtime = new OrcaRuntimeService(makeStore() as never) + + const result = await runtime.getWorktreePs(10_000) + + expect(result.hostScope?.hostIds).toEqual(['local', `ssh:${SSH_CONNECTION_ID}`]) + expect(result.hostScope?.omittedHostIds).toEqual([]) + }) + + it('keeps a remote row in the page when the cap cannot hold every local row', async () => { + const runtime = new OrcaRuntimeService(makeStore() as never) + + const result = await runtime.getWorktreePs(2) + + expect(result.truncated).toBe(true) + expect(result.worktrees).toHaveLength(2) + expect(result.worktrees.map((worktree) => worktree.hostId)).toContain( + `ssh:${SSH_CONNECTION_ID}` + ) + expect(result.hostScope?.hostIds).toEqual(['local', `ssh:${SSH_CONNECTION_ID}`]) + }) +}) diff --git a/src/shared/host-balanced-listing-page.ts b/src/shared/host-balanced-listing-page.ts new file mode 100644 index 00000000000..f771d98fdd4 --- /dev/null +++ b/src/shared/host-balanced-listing-page.ts @@ -0,0 +1,49 @@ +/** + * Chooses which rows survive a listing's row cap so that no execution host is starved by it. + * + * Worktree rows are resolved repo by repo, so every SSH repo's rows land contiguously at the end + * of the fleet order — 24 remote worktrees sat at indices 496-520 of 521 and a 200-row cap + * returned zero of them (#18104). A per-host round robin gives each host a share of the cap. + * + * Chosen rows keep the caller's original relative order, so the page stays a subsequence of the + * unbounded listing and nothing downstream has to re-sort. An uncapped listing is returned as-is. + */ +export function selectHostBalancedPage( + rows: readonly TRow[], + limit: number, + getHostId: (row: TRow) => string | null | undefined +): TRow[] { + if (rows.length <= limit) { + return [...rows] + } + // Insertion order is first-appearance order per host, so the round robin is deterministic. + const indicesByHost = new Map() + rows.forEach((row, index) => { + const hostId = getHostId(row) ?? '' + const bucket = indicesByHost.get(hostId) + if (bucket) { + bucket.push(index) + } else { + indicesByHost.set(hostId, [index]) + } + }) + const buckets = [...indicesByHost.values()] + const cursors = buckets.map(() => 0) + const chosen: number[] = [] + while (chosen.length < limit) { + let advanced = false + for (let bucket = 0; bucket < buckets.length && chosen.length < limit; bucket += 1) { + const cursor = cursors[bucket] ?? 0 + const index = buckets[bucket]?.[cursor] + if (index !== undefined) { + chosen.push(index) + cursors[bucket] = cursor + 1 + advanced = true + } + } + if (!advanced) { + break + } + } + return chosen.sort((left, right) => left - right).map((index) => rows[index] as TRow) +} diff --git a/src/shared/runtime-listing-host-scope.ts b/src/shared/runtime-listing-host-scope.ts new file mode 100644 index 00000000000..6232b0a4259 --- /dev/null +++ b/src/shared/runtime-listing-host-scope.ts @@ -0,0 +1,12 @@ +import type { ExecutionHostId } from './execution-host' + +/** + * What a bounded listing did and did not cover, by execution host. An absent scope means the + * host is too old to report one — not that it covered everything. See + * `docs/reference/ssh-execution-boundary.md`: a listing is only evidence about the hosts it + * actually covered, so an empty answer for a host that is missing here proves nothing. + */ +export type RuntimeListingHostScope = { + hostIds: ExecutionHostId[] + omittedHostIds: ExecutionHostId[] +} diff --git a/src/shared/runtime-terminal-contracts.ts b/src/shared/runtime-terminal-contracts.ts index d2d293c3a72..a75a2256bdb 100644 --- a/src/shared/runtime-terminal-contracts.ts +++ b/src/shared/runtime-terminal-contracts.ts @@ -6,6 +6,7 @@ import type { import type { StartupCommandDelivery } from './codex-startup-delivery' import type { ExecutionHostId } from './execution-host' import type { PtyIncarnationId } from './pty-incarnation' +import type { RuntimeListingHostScope } from './runtime-listing-host-scope' import type { RuntimeMobileSessionTabsResult } from './runtime-session-contracts' import type { TabGroupLayoutNode } from './tab-types' import type { TerminalExitCause } from './terminal-exit-cause' @@ -83,10 +84,8 @@ export type RuntimeTerminalVisualLayout = { root: RuntimeTerminalVisualLayoutNode } -export type RuntimeTerminalListHostScope = { - hostIds: ExecutionHostId[] - omittedHostIds: ExecutionHostId[] -} +/** The shared listing-scope shape, kept under its incumbent name for existing consumers. */ +export type RuntimeTerminalListHostScope = RuntimeListingHostScope export type RuntimeTerminalListResult = { terminals: RuntimeTerminalSummary[] diff --git a/src/shared/runtime-worktree-contracts.ts b/src/shared/runtime-worktree-contracts.ts index 1a053a91d1b..df0d4c68742 100644 --- a/src/shared/runtime-worktree-contracts.ts +++ b/src/shared/runtime-worktree-contracts.ts @@ -6,6 +6,7 @@ import type { WorktreeLineage, WorktreeLineageWarning } from './worktree/lineage-types' +import type { RuntimeListingHostScope } from './runtime-listing-host-scope' import type { GitWorktreeInfo, Worktree } from './worktree/types' export type RuntimeWorktreeAgentRow = { @@ -125,6 +126,8 @@ export type RuntimeWorktreePsResult = { worktrees: RuntimeWorktreePsSummary[] totalCount: number truncated: boolean + /** Absent from hosts that predate the field; treat that scope as unverifiable. */ + hostScope?: RuntimeListingHostScope } export type RuntimeWorktreePsSnapshotResult = RuntimeWorktreePsResult & { snapshotId: string } @@ -150,4 +153,6 @@ export type RuntimeWorktreeListResult = { worktrees: RuntimeWorktreeRecord[] totalCount: number truncated: boolean + /** Absent from hosts that predate the field; treat that scope as unverifiable. */ + hostScope?: RuntimeListingHostScope } From f974e981628ab1b113492b75c4454f10595bf05e Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:43:40 -0400 Subject: [PATCH 16/76] test(cloud): derive both reachability directions for the relay inventory census (#18524) Mirrors stablyai/orca-cloud#472 (c82f98f), byte-identical under cloud/. --- cloud/apps/relay/src/assignment-store.ts | 2 +- .../src/cell-inventory-hold-samples.test.ts | 70 +++++++++++++ .../src/cell-inventory-lock-census.test.ts | 97 +++++++++++++------ .../relay/src/regional-rehome-store.test.ts | 61 +++++++++++- 4 files changed, 196 insertions(+), 34 deletions(-) create mode 100644 cloud/apps/relay/src/cell-inventory-hold-samples.test.ts diff --git a/cloud/apps/relay/src/assignment-store.ts b/cloud/apps/relay/src/assignment-store.ts index 3571b57cd22..96d66eb7dc2 100644 --- a/cloud/apps/relay/src/assignment-store.ts +++ b/cloud/apps/relay/src/assignment-store.ts @@ -7969,7 +7969,7 @@ function isDatabaseLockUnavailable(error: unknown): boolean { return error instanceof Error && error.message === 'database_lock_unavailable' } -function cellInventoryLockOptions(mode: CellInventoryLockMode): RelayLockOptions { +export function cellInventoryLockOptions(mode: CellInventoryLockMode): RelayLockOptions { if (mode === 'nowait') return { failIfUnavailable: true, measureHoldMs: true } if (mode === 'pool-default') return { measureHoldMs: true } return { lockTimeoutMs: CELL_INVENTORY_LOCK_TIMEOUT_MS, measureHoldMs: true } diff --git a/cloud/apps/relay/src/cell-inventory-hold-samples.test.ts b/cloud/apps/relay/src/cell-inventory-hold-samples.test.ts new file mode 100644 index 00000000000..abd37dcbb29 --- /dev/null +++ b/cloud/apps/relay/src/cell-inventory-hold-samples.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest' +import { + CellInventoryHoldSamples, + emptyCellInventoryHoldCounts +} from './cell-inventory-hold-samples.js' + +// Nearest rank, computed in integer arithmetic so it cannot inherit the float +// error the implementation's `0.95 * n` could in principle carry. +function nearestRankP95(sorted: number[]): number { + return sorted[Math.ceil((95 * sorted.length) / 100) - 1]! +} + +function samplesOf(values: number[]): CellInventoryHoldSamples { + const samples = new CellInventoryHoldSamples() + for (const value of values) samples.record(value) + return samples +} + +describe('cell inventory hold samples', () => { + it('reports nothing before the first hold', () => { + expect(new CellInventoryHoldSamples().readCounts()).toEqual( + emptyCellInventoryHoldCounts() + ) + }) + + // Why: the 500ms bound will be tuned against this percentile, so an off-by-one + // here reads as a hold the fleet never had. + it('places p95 at the nearest rank for every window size', () => { + for (let size = 1; size <= 400; size++) { + const values = Array.from({ length: size }, (_, index) => index + 1) + const shuffled = [...values].reverse() + + const counts = samplesOf(shuffled).readCounts() + + expect(counts.cellInventoryHoldMsP95).toBe(nearestRankP95(values)) + expect(counts.cellInventoryHoldMsMax).toBe(size) + expect(counts.cellInventoryHolds).toBe(size) + } + }) + + it('never reports a p95 above the max', () => { + for (let size = 1; size <= 200; size++) { + const counts = samplesOf(Array.from({ length: size }, (_, i) => i + 1)).readCounts() + + expect(counts.cellInventoryHoldMsP95).toBeLessThanOrEqual(counts.cellInventoryHoldMsMax) + } + }) + + it('ignores a hold that is not a finite, non-negative duration', () => { + const samples = samplesOf([Number.NaN, Number.POSITIVE_INFINITY, -1]) + + expect(samples.readCounts()).toEqual(emptyCellInventoryHoldCounts()) + }) + + // Why: the reservoir is bounded, so a heavy flush interval keeps the most + // recent holds rather than growing without limit or freezing on the oldest. + it('keeps the most recent holds once the reservoir is full', () => { + const counts = samplesOf(Array.from({ length: 2_100 }, (_, index) => index + 1)).readCounts() + + expect(counts.cellInventoryHolds).toBe(2_048) + expect(counts.cellInventoryHoldMsMax).toBe(2_100) + }) + + it('resets the window on consume so each flush reports its own holds', () => { + const samples = samplesOf([5, 10]) + + expect(samples.consumeCounts().cellInventoryHolds).toBe(2) + expect(samples.consumeCounts()).toEqual(emptyCellInventoryHoldCounts()) + }) +}) diff --git a/cloud/apps/relay/src/cell-inventory-lock-census.test.ts b/cloud/apps/relay/src/cell-inventory-lock-census.test.ts index d0527534935..b2b5b65684c 100644 --- a/cloud/apps/relay/src/cell-inventory-lock-census.test.ts +++ b/cloud/apps/relay/src/cell-inventory-lock-census.test.ts @@ -1,11 +1,11 @@ import { readFileSync } from 'node:fs' import { describe, expect, it } from 'vitest' -import type { CellInventoryLockMode } from './assignment-store.js' +import { cellInventoryLockOptions, type CellInventoryLockMode } from './assignment-store.js' // Which entry points can reach a call site. A site a sweep can enter must never // take the bounded wait: its 55P03 becomes a terminal transaction failure, and // the incident monitor freezes on a single one. -type Reachability = 'request' | 'sweep' | 'both' +type Reachability = 'request' | 'sweep' | 'both' | 'orphan' // 'caller' is not a CellInventoryLockMode: those sites take the mode threaded // from `assign`, which is 'request' for a client and 'pool-default' for the @@ -25,7 +25,8 @@ const CENSUS: CensusEntry[] = [ { method: 'assignOnce', mode: 'nowait', reach: 'both' }, { method: 'assignOnce', mode: 'nowait', reach: 'both' }, { method: 'refreshDrainMigrationLeasesOnce', mode: 'request', reach: 'request' }, - { method: 'changeActivity', mode: 'request', reach: 'request' }, + // Reachable from neither: changeActivity has no production callers, only tests. + { method: 'changeActivity', mode: 'request', reach: 'orphan' }, { method: 'acquireActivity', mode: 'request', reach: 'request' }, { method: 'activateControl', mode: 'request', reach: 'request' }, { method: 'startEvacuation', mode: 'request', reach: 'request' }, @@ -52,20 +53,15 @@ const CENSUS: CensusEntry[] = [ ] // The background sweeps, and nothing else. A method reachable from one of these -// can be entered by a sweep tick, whatever else can also enter it. -const SWEEP_ROOTS = [ - 'refreshRegionalRehomeLeases', - 'completeReadyEvacuations', - 'completeReadyRegionalRehomes', - 'abortExpiredEvacuations', - 'abortExpiredRegionalRehomes', - 'reapRegionalRehomeAttempts', - 'releaseExpiredActivityLeases', - 'releaseExpiredActivity', - 'releaseExpiredRegionPreferences', - 'evacuateDeadCells', - 'claimRegionalRehome', - 'recordRegionalRehomeDispatchFailure' +// can be entered by a sweep tick, whatever else can also enter it. Both lists are +// read from source, so a new sweep step or a new route widens the derivation here +// instead of silently widening what a bounded wait can be entered from. +const SWEEP_ENTRY_FILES = ['./assignment-cleanup-steps.ts', './regional-rehome-worker.ts'] +const REQUEST_ENTRY_FILES = [ + './app.ts', + './relay-server.ts', + './host-session-registry.ts', + './cell-admission-startup.ts' ] const DECLARATION = /^ {2}(?:private |public )?(?:static )?(?:async )?([A-Za-z_][\w]*)[(<]/ @@ -74,9 +70,18 @@ function storeSource(): string[] { return readFileSync(new URL('./assignment-store.ts', import.meta.url), 'utf8').split('\n') } -// Why: a hand-written reachability column is a claim, not a check. Derive it, so -// a new sweep edge into a bounded site fails here instead of in production. -function sweepReachableMethods(lines: string[]): Set { +function entryPoints(files: string[]): string[] { + return files.flatMap((file) => + [ + ...readFileSync(new URL(file, import.meta.url), 'utf8').matchAll( + /assignments\.([A-Za-z_][\w]*)\(/g + ) + ].map((call) => call[1]!) + ) +} + +// Same-class call graph: store methods only ever reach each other through `this.`. +function storeCallGraph(lines: string[]): Map> { const bounds: { name: string; start: number }[] = [] lines.forEach((line, index) => { const declaration = DECLARATION.exec(line) @@ -93,8 +98,12 @@ function sweepReachableMethods(lines: string[]): Set { } callees.set(method.name, names) }) + return callees +} + +function closure(callees: Map>, roots: string[]): Set { const reached = new Set() - const pending = [...SWEEP_ROOTS] + const pending = [...roots] while (pending.length > 0) { const name = pending.pop()! if (reached.has(name)) continue @@ -104,6 +113,23 @@ function sweepReachableMethods(lines: string[]): Set { return reached } +// Why: a hand-written reachability column is a claim, not a check. Derive both +// directions, so a new sweep edge into a bounded site fails here instead of in +// production, and so 'sweep' and 'both' stop being asserted by hand. +function derivedReachability(lines: string[]): (method: string) => Reachability { + const callees = storeCallGraph(lines) + const sweep = closure(callees, entryPoints(SWEEP_ENTRY_FILES)) + const request = closure(callees, entryPoints(REQUEST_ENTRY_FILES)) + return (method) => + sweep.has(method) + ? request.has(method) + ? 'both' + : 'sweep' + : request.has(method) + ? 'request' + : 'orphan' +} + function readCallSites(): { method: string; mode: CensusMode }[] { const sites: { method: string; mode: CensusMode }[] = [] let method = '' @@ -136,27 +162,42 @@ describe('cell inventory lock call-site census', () => { }) it('derives the same reachability the census claims', () => { - const reached = sweepReachableMethods(storeSource()) - const derived = readCallSites().map(({ method }) => reached.has(method)) + const reachOf = derivedReachability(storeSource()) - expect(derived).toEqual(CENSUS.map((entry) => entry.reach !== 'request')) + expect(readCallSites().map(({ method }) => reachOf(method))).toEqual( + CENSUS.map((entry) => entry.reach) + ) }) // Why: this is the whole point of the classification. A shorter wait on a // sweep-reachable site turns contention into a terminal transaction failure, // and relayPostgresRetryExhausted freezes the incident gate at zero. + // Why: the hold distribution is what the 500ms bound will be tuned against, so + // a mode that stops asking for it goes unmeasured in exactly the lane that + // matters. Nothing else in the suite reads the pool-default branch. + it('measures the hold in every lock mode', () => { + const modes: CellInventoryLockMode[] = ['request', 'nowait', 'pool-default'] + + expect(modes.map((mode) => cellInventoryLockOptions(mode).measureHoldMs)).toEqual([ + true, + true, + true + ]) + }) + it('never puts a sweep-reachable site on the bounded wait', () => { - const reached = sweepReachableMethods(storeSource()) + const reachOf = derivedReachability(storeSource()) const bounded = readCallSites().filter( - (site) => site.mode === 'request' && reached.has(site.method) + (site) => site.mode === 'request' && ['sweep', 'both'].includes(reachOf(site.method)) ) expect(bounded).toEqual([]) }) it('routes every sweep-only site to NOWAIT so it can skip the tick', () => { - const queueing = CENSUS.filter( - (entry) => entry.reach === 'sweep' && entry.mode !== 'nowait' + const reachOf = derivedReachability(storeSource()) + const queueing = readCallSites().filter( + (site) => reachOf(site.method) === 'sweep' && site.mode !== 'nowait' ) expect(queueing).toEqual([]) diff --git a/cloud/apps/relay/src/regional-rehome-store.test.ts b/cloud/apps/relay/src/regional-rehome-store.test.ts index 6f57283396f..26f711189ee 100644 --- a/cloud/apps/relay/src/regional-rehome-store.test.ts +++ b/cloud/apps/relay/src/regional-rehome-store.test.ts @@ -626,7 +626,7 @@ describe('regional rehome assignment state', () => { await context.store.releaseActivity(identity, sourceControl) } probe.reset() - probe.failNoWaitOnce = true + probe.failNoWaitTimes = 1 const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy') let completed: number @@ -647,6 +647,57 @@ describe('regional rehome assignment state', () => { await context.database.close() }) + // Why: with `continue` replaced by `break` a single contended candidate drops + // the rest of the page. Two in a row prove the sweep resumes, not just that it + // survived one, and that the summary counts both. + it('completes a candidate behind two contended ones', async () => { + const probe = new CellInventoryLockProbe() + const context = await setup({ wrap: (database) => probe.wrap(database) }) + const identities = [ + { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }, + { userId: 'user-2', relayHostId: 'ponmlkjihgfedcba' }, + { userId: 'user-3', relayHostId: 'aaaabbbbccccdddd' } + ] + for (const identity of identities) { + // Dispatch is rate limited, so each claim needs its own interval. + context.advance(60_000) + await freshHeartbeats(context) + const sourceControl = await activatePreferredSource(context, identity) + const attempt = await context.store.claimRegionalRehome() + await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted') + await context.store.activateControl(identity, { + cellId: target.id, + assignmentEpoch: 2, + generation: 1 + }) + await context.store.markMigrationTargetRegistered(identity, { + cellId: target.id, + assignmentEpoch: 2 + }) + await context.store.releaseActivity(identity, sourceControl) + } + probe.reset() + probe.failNoWaitTimes = 2 + const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy') + + let completed: number + try { + completed = await context.store.completeReadyRegionalRehomes() + } finally { + busy.restore() + } + + expect(completed).toBe(1) + expect(busy.entries).toEqual([ + { + event: 'orca_relay_sweep_cell_inventory_busy', + sweep: 'complete-ready-regional-rehomes', + skipped: 2 + } + ]) + await context.database.close() + }) + // Why: only inventory contention is ordinary. Every other failure must keep its // existing propagation and its dispatch-failure accounting. it('propagates a claim failure that is not inventory contention', async () => { @@ -1692,8 +1743,8 @@ async function heartbeat( class CellInventoryLockProbe { readonly locks: (RelayLockOptions | undefined)[] = [] failNoWait = false - // Contends one candidate only, so the sweep must carry on to the next. - failNoWaitOnce = false + // Contends the first N candidates only, so the sweep must carry on past them. + failNoWaitTimes = 0 failWith: Error | null = null reset(): void { @@ -1708,8 +1759,8 @@ class CellInventoryLockProbe { if (sql.trim() === 'SELECT * FROM relay_cells ORDER BY cell_id ASC') { probe.locks.push(options) if (probe.failWith) throw probe.failWith - if (options?.failIfUnavailable && probe.failNoWaitOnce) { - probe.failNoWaitOnce = false + if (options?.failIfUnavailable && probe.failNoWaitTimes > 0) { + probe.failNoWaitTimes-- throw new Error('database_lock_unavailable') } if (probe.failNoWait && options?.failIfUnavailable) { From f35015d0c8974e5c94e4701fe8f1e9bcedc94fac Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:44:32 -0700 Subject: [PATCH 17/76] fix(ssh): measure pane idleness in the unit the sweep's kill operates on (#18415) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The orphan-relay-PTY sweep authorizes `pty.shutdown { immediate: true }`, which runs `forceKillPosixPtyProcessGroups`: collect every process group on the pane's tty, then `killpg` each one. The blast radius is therefore (groups on the tty) x (members of those groups, wherever they are). The idleness evidence measured only the first factor, so three shapes read as idle and were SIGKILLed: - with job control off (`set +m`) a background job keeps the SHELL's pgid, so the tty carries exactly one process group and that group is running the user's build; - a child that drops the controlling terminal (`ioctl(TIOCNOTTY)` without `setsid`) keeps the pgid, reports `tpgid == -1`, and never appears in `ps -t `; - a double-forked grandchild keeps the pgid and tty but reparents to pid 1, so the `ppid` walk cannot reach it and the named-process backstop never fires. `shellOwnsEveryTtyProcessGroup` now also requires the shell's own process group to hold no other member anywhere in the table, indexed in the same single pass. A pids-per-tty set would catch the first and third but not the second, which is why the count is pgid-wide rather than tty-scoped. The wire field keeps its tty-shaped name: the value only ever became stricter, so an old client skips more, never less. Second, unrelated-in-mechanism but same file family: `foregroundSkipReason` summed `capturedAgeMs + evidenceAgeSinceListingMs` without validating either. A non-numeric `capturedAgeMs` makes the sum `NaN`, and `NaN > 5000` is false, so a malformed record PASSED the freshness gate and proceeded toward the stop — the one place in the file that defaulted toward kill. Nothing validated it on this path (`mapSshPtyProcessList` checks the ownership fields and spreads the rest through; `PtyProcessListAdmission` is not on the sweep path). It now runs `isForegroundProcessEvidence` and fails closed. Verified on real Linux, not only in mocks: a container drives `bash -i` on a real pty, builds each construction, runs the real publisher and planner, and then calls the real `forceKillPosixPtyProcessGroups`. Before, all three published `shellOwnsEveryTtyProcessGroup: true`, planned SWEEP, and the planted pid was gone after the signal. After, all three skip and survive, and an idle shell is still reclaimed. Residuals are written down at the predicate and in ssh-execution-boundary.md: the capture is a snapshot (bounded by the evidence-age budget, not removed), and a process the host's own `ps` cannot enumerate stays unobservable while `killpg` still reaches it. --- docs/reference/ssh-execution-boundary.md | 15 +++ .../agent-foreground-process-batch.ts | 91 ++++++++++++----- ...h-orphan-sweep-pane-state-verdicts.test.ts | 99 +++++++++++++++++++ src/shared/foreground-process-evidence.ts | 16 ++- .../ssh-relay-pty-ownership-proof.test.ts | 52 ++++++++++ src/shared/ssh-relay-pty-ownership-proof.ts | 34 +++++-- 6 files changed, 271 insertions(+), 36 deletions(-) diff --git a/docs/reference/ssh-execution-boundary.md b/docs/reference/ssh-execution-boundary.md index cc88cf39a17..070aae61d66 100644 --- a/docs/reference/ssh-execution-boundary.md +++ b/docs/reference/ssh-execution-boundary.md @@ -70,6 +70,21 @@ A verdict needs evidence from the host that owns the process. Apply these tests Anything short of positive host evidence is `unverifiable`. Reporting it as `exited` is the error this document exists to prevent: it orphans live work and can cold-start a duplicate over the same worktree. +## Deciding a remote pane is idle + +The orphan-PTY sweep is the one flow that turns an observation into a SIGKILL, so its idleness evidence has to be measured against the same thing the signal reaches. It is not the terminal. + +`forceKillPosixPtyProcessGroups` (`src/main/pty/posix-pty-process-groups.ts`) collects every process group on the pane's tty and `killpg`s each one. The blast radius is therefore _(process groups on the tty) × (members of those groups, wherever they are)_, and the second factor is not bounded by the terminal at all. Two facts make that gap reachable: + +- **Job control can be off.** With `set +m` a background job does not get its own process group — it keeps the shell's. `ps` then shows one process group on the tty, running a build. Nothing in a tty-shaped predicate can see it. +- **A group member can leave the terminal.** `ioctl(TIOCNOTTY)` without `setsid` drops the controlling terminal but keeps the pgid, so the process reports `tpgid == -1`, never appears in `ps -t `, and is still killed by `killpg(shellPgid)`. A double-forked grandchild similarly keeps the pgid while reparenting to pid 1, so no walk by `ppid` from the PTY root can name it either. + +So `shellOwnsEveryTtyProcessGroup` (`src/main/providers/agent-foreground-process-batch.ts`) requires both measurements: every process group on the tty is the shell's own with none stopped, **and** the shell's own process group has no other member anywhere in the host's process table. The name is tty-shaped for wire-compatibility reasons only. + +Two residuals remain, and neither is removable here. The capture is a snapshot, so work started between the `ps` and the signal is invisible — bounded by `RELAY_PTY_SWEEP_MAX_EVIDENCE_AGE_MS` on the reading side, not eliminated. And a process the host's own `ps` cannot enumerate (another PID namespace, `hidepid=2`, a table truncated by a permission boundary) is unobservable while `killpg` still reaches it. + +The general rule this instantiates: **evidence must be measured in the unit the destructive action operates on.** Evidence in a different unit is `unverifiable` no matter how precise it looks. + ## Reading artifacts instead of process state Artifacts are stronger evidence than liveness signals, but they answer a narrower question than they appear to. diff --git a/src/main/providers/agent-foreground-process-batch.ts b/src/main/providers/agent-foreground-process-batch.ts index 12b60164446..414e57afcb3 100644 --- a/src/main/providers/agent-foreground-process-batch.ts +++ b/src/main/providers/agent-foreground-process-batch.ts @@ -29,7 +29,10 @@ export type BatchedForegroundProcessResult = { processName: string | null reason?: string /** Set only when the table was readable: every process group attached to this PTY's terminal is - * the shell's own, and none of them is stopped. Left absent when we could not observe it. */ + * the shell's own, none of them is stopped, AND that group's only member is the shell itself. + * Left absent when we could not observe it. Keeps the tty-shaped name because it is on the wire + * (`ForegroundProcessEvidence`); the value only ever got stricter, so an old client reading it + * skips more, never less. */ shellOwnsEveryTtyProcessGroup?: boolean } @@ -62,30 +65,45 @@ export type BatchedForegroundProcessOptions = { stats?: ProcessTableIndexStats } -/** Which process groups occupy each controlling terminal, and which terminals hold a stopped - * process. */ -type TtyOccupancy = { +/** The two units a forced stop can reach, indexed from one capture: which process groups occupy + * each controlling terminal (which terminals hold a stopped process), and how many rows belong to + * each process group anywhere on the host. */ +type PaneOccupancy = { processGroupsByTty: ReadonlyMap> stoppedTtys: ReadonlySet + /** Rows per `pgid`, counted over the WHOLE table with no tty filter — that is the point of it. + * A member that shares the shell's group but has no controlling terminal is reachable by + * `killpg` and invisible to every tty-shaped index. */ + rowsByProcessGroup: ReadonlyMap + /** True when some row carried no `pgid`, so the group counts are incomplete and cannot support + * an idleness claim. */ + processGroupsIncomplete: boolean } -const ttyOccupancyByCapture = new WeakMap() +const paneOccupancyByCapture = new WeakMap() -/** Index the capture by controlling terminal. +/** Index the capture by controlling terminal and by process group. * - * Keyed on `tpgid` because the snapshot carries no tty column and does not need one: a process - * group belongs to exactly one session, a session to at most one controlling terminal, so two - * rows reporting the same live `tpgid` are on the same tty. Memoized per capture, since the - * per-pane cadence poll and `pty.listProcesses` share one TTL-cached table. */ -function getTtyOccupancy(rows: readonly ProcessTableRow[]): TtyOccupancy { - const cached = ttyOccupancyByCapture.get(rows) + * The tty half is keyed on `tpgid` because the snapshot carries no tty column and does not need + * one: a process group belongs to exactly one session, a session to at most one controlling + * terminal, so two rows reporting the same live `tpgid` are on the same tty. Memoized per capture, + * since the per-pane cadence poll and `pty.listProcesses` share one TTL-cached table. */ +function getPaneOccupancy(rows: readonly ProcessTableRow[]): PaneOccupancy { + const cached = paneOccupancyByCapture.get(rows) if (cached) { return cached } const processGroupsByTty = new Map>() const stoppedTtys = new Set() + const rowsByProcessGroup = new Map() + let processGroupsIncomplete = false for (const row of rows) { - if (row.pgid === undefined || row.tpgid === undefined || row.tpgid <= 0) { + if (row.pgid === undefined) { + processGroupsIncomplete = true + continue + } + rowsByProcessGroup.set(row.pgid, (rowsByProcessGroup.get(row.pgid) ?? 0) + 1) + if (row.tpgid === undefined || row.tpgid <= 0) { continue } let groups = processGroupsByTty.get(row.tpgid) @@ -99,8 +117,13 @@ function getTtyOccupancy(rows: readonly ProcessTableRow[]): TtyOccupancy { stoppedTtys.add(row.tpgid) } } - const occupancy: TtyOccupancy = { processGroupsByTty, stoppedTtys } - ttyOccupancyByCapture.set(rows, occupancy) + const occupancy: PaneOccupancy = { + processGroupsByTty, + stoppedTtys, + rowsByProcessGroup, + processGroupsIncomplete + } + paneOccupancyByCapture.set(rows, occupancy) return occupancy } @@ -161,7 +184,7 @@ export function resolveAgentForegroundProcessesFromIndex( } } - const occupancy = getTtyOccupancy(index.rows) + const occupancy = getPaneOccupancy(index.rows) return requests.map((request) => { const root = lookupProcessTableIndex(index, (value) => value.byPid.get(request.rootPid)) if (!root) { @@ -185,20 +208,40 @@ export function resolveAgentForegroundProcessesFromIndex( reason: 'no_controlling_tty' } } - // The only host-observable "nothing is running here" signal, and it has to be read off the - // whole tty rather than off `tpgid === pgid`. A backgrounded `pnpm build &` and a Ctrl-Z'd - // editor both leave the shell owning the foreground group, byte-identical to an idle prompt; - // what separates them is a second process group attached to the pane's terminal. That is also - // exactly the blast radius of the stop this attests to — `forceKillPosixPtyProcessGroups` - // SIGKILLs every process group on the tty — so the evidence and the kill now measure the same - // thing. A reader may treat `false` as "busy" and must never treat absence as "idle". + // The only host-observable "nothing is running here" signal, and it takes TWO measurements + // because the stop it authorizes has two units. `forceKillPosixPtyProcessGroups` collects every + // process group on the pane's tty and then `killpg`s each one, so the blast radius is + // (groups on the tty) x (members of those groups, wherever they are). Neither half implies the + // other, so both are required: + // + // tty: a backgrounded `pnpm build &` and a Ctrl-Z'd editor both hand the terminal back, so + // the shell's row is byte-identical to an idle prompt. What separates them is a second + // process group attached to the pane's terminal. + // group: with job control off (`set +m`, common in non-interactive and dumb-terminal shells, + // and settable by the user at the prompt) a background job KEEPS the shell's pgid, so + // the tty shows one group and that group is running a build. Same for a child that + // drops the controlling terminal without `setsid` (`tpgid == -1`, absent from every + // tty index, still reachable by `killpg`) and for a double-forked grandchild that + // reparents to pid 1 and so never appears in the ppid walk below. + // + // Residual after both, written down because the predicate cannot see it: the capture is a + // snapshot, so work started between the `ps` and the signal is invisible — bounded, not + // removed, by RELAY_PTY_SWEEP_MAX_EVIDENCE_AGE_MS on the reading side; and a process the host's + // own `ps` cannot enumerate (another PID namespace, `hidepid=2`, a table truncated by a + // permission boundary) is unobservable here while `killpg` still reaches it. + // + // A reader may treat `false` as "busy" and must never treat absence as "idle". const ttyProcessGroups = occupancy.processGroupsByTty.get(root.tpgid) const shellOwnsEveryTtyProcessGroup = root.tpgid === root.pgid && ttyProcessGroups !== undefined && ttyProcessGroups.size === 1 && ttyProcessGroups.has(root.pgid) && - !occupancy.stoppedTtys.has(root.tpgid) + !occupancy.stoppedTtys.has(root.tpgid) && + !occupancy.processGroupsIncomplete && + // The root always counts itself, so exactly one row in its group means the group IS the + // shell — no separate leader check, and no set of pids retained per capture. + occupancy.rowsByProcessGroup.get(root.pgid) === 1 const allCandidates = rowsByOwner.get(root.pid) ?? [] const foregroundCandidates = allCandidates.filter((row) => row.pgid === root.tpgid) const fallbackProcess = request.fallbackProcess diff --git a/src/main/ssh/ssh-orphan-sweep-pane-state-verdicts.test.ts b/src/main/ssh/ssh-orphan-sweep-pane-state-verdicts.test.ts index ab069240681..560d18b8b62 100644 --- a/src/main/ssh/ssh-orphan-sweep-pane-state-verdicts.test.ts +++ b/src/main/ssh/ssh-orphan-sweep-pane-state-verdicts.test.ts @@ -68,6 +68,44 @@ const CAPTURES = { ' 3159 3158 3159 3158 T sleep 300', ' 3160 1 1 -1 R ps -axo pid=,ppid=,pgid=,tpgid=,stat=,command=' ] + }, + /** `set +m; sleep 300 &`. With job control OFF the job does not get its own process group — it + * keeps the SHELL's pgid. So the tty carries exactly one process group, and that group is + * running a build. Reproduced independently on a real Ubuntu host through an Orca pane. */ + setMinusMBackground: { + rootPid: 12, + table: [ + ' 1 0 1 -1 Ss /bin/bash /work/run.sh', + ' 11 1 1 -1 S python3 /work/pty-scenario.py setm_background', + ' 12 11 12 12 Ss+ bash -i', + ' 13 12 12 12 S+ sleep 300', + ' 14 11 1 -1 R ps -axo pid=,ppid=,pgid=,tpgid=,stat=,command=' + ] + }, + /** A `set +m` job that drops its controlling terminal (`ioctl(TIOCNOTTY)` with no `setsid`). It + * keeps the shell's pgid, reports `tpgid == -1`, and is absent from `ps -t ` and from every + * tty-keyed index — while `killpg(shellPgid)` still reaches it. */ + nottyGroupMember: { + rootPid: 16, + table: [ + ' 1 0 1 -1 Ss /bin/bash /work/run.sh', + ' 15 1 1 -1 S python3 /work/pty-scenario.py notty_member', + ' 16 15 16 16 Ss+ bash -i', + ' 17 16 16 -1 S python3 -c import fcntl,os,time;fd=os.open("/dev/tty",os.O_RDWR);fcntl.ioctl(fd,0x5422);os.close(fd);time.sleep(300)', + ' 18 15 1 -1 R ps -axo pid=,ppid=,pgid=,tpgid=,stat=,command=' + ] + }, + /** A `set +m` job that double-forks. pid 22 keeps the shell's pgid and tty but reparented to pid + * 1, so the ppid walk from `rootPid` never reaches it and it can never be named. */ + doubleForkedGroupMember: { + rootPid: 20, + table: [ + ' 1 0 1 -1 Ss /bin/bash /work/run.sh', + ' 19 1 1 -1 S python3 /work/pty-scenario.py double_fork', + ' 20 19 20 20 Ss+ bash -i', + ' 22 1 20 20 S+ python3 -c import os,sys,time;p=os.fork() if p: print("GRANDCHILD:%d"%p);sys.stdout.flush();os._exit(0) time.sleep(300)', + ' 23 19 1 -1 R ps -axo pid=,ppid=,pgid=,tpgid=,stat=,command=' + ] } } as const @@ -147,6 +185,15 @@ describe('what the host publishes about a pane, read by the sweep', () => { expect(shellShape(CAPTURES.background)).toBe(shellShape(CAPTURES.idle)) expect(shellShape(CAPTURES.ctrlz)).toBe(shellShape(CAPTURES.idle)) expect(shellShape(CAPTURES.foreground)).not.toBe(shellShape(CAPTURES.idle)) + + // Same premise for the `set +m` captures, minus `ppid`: their harness keeps its parent alive + // rather than reparenting the shell to init, and the ppid is the one field of the shape the + // predicate never reads. + const paneShape = (capture: { rootPid: number; table: readonly string[] }): string => + shellShape(capture).split(' ').slice(1).join(' ') + expect(paneShape(CAPTURES.setMinusMBackground)).toBe(paneShape(CAPTURES.idle)) + expect(paneShape(CAPTURES.nottyGroupMember)).toBe(paneShape(CAPTURES.idle)) + expect(paneShape(CAPTURES.doubleForkedGroupMember)).toBe(paneShape(CAPTURES.idle)) }) it('sweeps an idle shell', async () => { @@ -191,6 +238,58 @@ describe('what the host publishes about a pane, read by the sweep', () => { expect(skipReason(plan)).toBe('host does not attest an idle shell') }) + // The tty is not the unit the stop operates on. `forceKillPosixPtyProcessGroups` collects the + // groups on the tty and then `killpg`s each one, so anything sharing the shell's pgid dies with + // it — including members the tty index cannot see at all. All three captures below reproduce on + // real Linux: before the group-membership half of the predicate they published + // `shellOwnsEveryTtyProcessGroup: true`, planned a SWEEP, and the planted pid was GONE after the + // real `forceKillPosixPtyProcessGroups` call. + it('never sweeps a pane whose background job shares the shell pgid under `set +m`', async () => { + // pid 13 is `sleep 300` — stand in `pnpm build`. Its pgid IS the shell's, so the tty carries + // exactly one process group and the tty half of the predicate reads the pane as idle. + const rows = parseStrictProcessTableRows(CAPTURES.setMinusMBackground.table.join('\n')) + const tty = rows.filter((row) => row.tpgid === CAPTURES.setMinusMBackground.rootPid) + expect(new Set(tty.map((row) => row.pgid))).toEqual(new Set([12])) + expect(tty.map((row) => row.pid)).toEqual([12, 13]) + + const evidence = await publish(CAPTURES.setMinusMBackground) + expect(evidence).toMatchObject({ shellOwnsEveryTtyProcessGroup: false }) + + const plan = await planFor(CAPTURES.setMinusMBackground) + expect(plan.sweep).toEqual([]) + expect(skipReason(plan)).toBe('host does not attest an idle shell') + }) + + it('never sweeps a pane whose group member dropped the controlling terminal', async () => { + // pid 17 kept the shell's pgid and called `ioctl(TIOCNOTTY)`, so it reports `tpgid == -1`, + // never appears in `ps -t `, and no tty-shaped index — not process groups, not pids — + // can observe it. `killpg(16)` reaches it regardless. + const rows = parseStrictProcessTableRows(CAPTURES.nottyGroupMember.table.join('\n')) + expect(rows.filter((row) => row.tpgid === 16).map((row) => row.pid)).toEqual([16]) + expect(rows.filter((row) => row.pgid === 16).map((row) => row.pid)).toEqual([16, 17]) + + const evidence = await publish(CAPTURES.nottyGroupMember) + expect(evidence).toMatchObject({ shellOwnsEveryTtyProcessGroup: false }) + + const plan = await planFor(CAPTURES.nottyGroupMember) + expect(plan.sweep).toEqual([]) + expect(skipReason(plan)).toBe('host does not attest an idle shell') + }) + + it('never sweeps a pane whose group member double-forked away from the shell', async () => { + // pid 22 reparented to pid 1, so the ppid walk from rootPid cannot reach it and the named- + // process backstop can never fire. It still holds the shell's pgid. + const rows = parseStrictProcessTableRows(CAPTURES.doubleForkedGroupMember.table.join('\n')) + expect(rows.find((row) => row.pid === 22)).toMatchObject({ ppid: 1, pgid: 20, tpgid: 20 }) + + const evidence = await publish(CAPTURES.doubleForkedGroupMember) + expect(evidence).toMatchObject({ processName: null, shellOwnsEveryTtyProcessGroup: false }) + + const plan = await planFor(CAPTURES.doubleForkedGroupMember) + expect(plan.sweep).toEqual([]) + expect(skipReason(plan)).toBe('host does not attest an idle shell') + }) + it('refuses an observation older than the pass it would authorize', async () => { // Same idle capture that sweeps above; only its age differs. Staleness degrades to "leave it // running", never to "stop it". diff --git a/src/shared/foreground-process-evidence.ts b/src/shared/foreground-process-evidence.ts index a9d36fe557c..975fbe388ff 100644 --- a/src/shared/foreground-process-evidence.ts +++ b/src/shared/foreground-process-evidence.ts @@ -17,14 +17,20 @@ export type ForegroundProcessEvidence = | ({ verdict: 'live' processName: string | null - /** True only when the host observed every process group attached to this PTY's terminal to be - * the shell's own, with none of them stopped — i.e. nothing is running in the pane, in the - * foreground OR the background, and nothing sits suspended. + /** True only when the host observed BOTH units a forced stop can reach to hold nothing but + * the shell: every process group attached to this PTY's terminal is the shell's own with + * none of them stopped, AND the shell's own process group has no other member anywhere on + * the host. I.e. nothing is running in the pane, in the foreground OR the background, and + * nothing sits suspended. * * Deliberately not `tpgid === pgid`: a job the user backgrounded with `&` and a job the user * suspended with Ctrl-Z both hand the terminal back to the shell, so a foreground-only - * predicate reads them as idle. This one is measured against the same set of process groups - * a forced stop would SIGKILL. + * predicate reads them as idle. Deliberately not the tty alone either: with job control off + * (`set +m`) a background job keeps the shell's pgid, and a child that drops the controlling + * terminal leaves every tty index entirely — both are still inside `killpg`'s reach. + * + * The name is tty-shaped for wire reasons only. It shipped that way and old clients read it; + * the value has only ever become stricter, which makes an old client skip more, never less. * * False means something IS running, named or not. Absent from a host that predates the * field, which is neither: a reader deciding whether the pane is idle must require `true` diff --git a/src/shared/ssh-relay-pty-ownership-proof.test.ts b/src/shared/ssh-relay-pty-ownership-proof.test.ts index 1abfc0ef7f5..b17f96808f9 100644 --- a/src/shared/ssh-relay-pty-ownership-proof.test.ts +++ b/src/shared/ssh-relay-pty-ownership-proof.test.ts @@ -146,6 +146,58 @@ describe('planRelayPtySweep', () => { expect(reasonFor(plan, 'pty-1')).toBe('host attests another client created it') }) + // The age gate is the one comparison in the file that a malformed field defaults toward the + // kill: the sum goes `NaN`, and `NaN > budget` is FALSE, so the entry PASSES the freshness gate + // and proceeds toward the stop. Nothing validated this record on the sweep path — + // `mapSshPtyProcessList` checks the ownership fields and spreads the rest through. + it.each([ + ['missing', undefined], + ['a string', '0' as unknown], + ['NaN', Number.NaN], + ['Infinity', Number.POSITIVE_INFINITY], + ['negative', -1], + ['fractional', 1.5] + ])('never sweeps when the host stamped capturedAgeMs %s', (_label, capturedAgeMs) => { + const plan = planRelayPtySweep( + [ + orphan({ + foregroundProcessEvidence: { + ...idleShell(), + capturedAgeMs + } as unknown as ForegroundProcessEvidence + }) + ], + context() + ) + + expect(plan.sweep).toEqual([]) + expect(reasonFor(plan, 'pty-1')).toBe('host foreground observation is malformed') + }) + + it('never sweeps on an evidence record whose other host stamps are malformed', () => { + const plan = planRelayPtySweep( + [ + orphan({ + foregroundProcessEvidence: { + ...idleShell(), + authorityGeneration: '' + } as ForegroundProcessEvidence + }) + ], + context() + ) + + expect(plan.sweep).toEqual([]) + expect(reasonFor(plan, 'pty-1')).toBe('host foreground observation is malformed') + }) + + it('never sweeps when this client cannot compute an age budget', () => { + const plan = planRelayPtySweep([orphan()], context({ evidenceAgeSinceListingMs: Number.NaN })) + + expect(plan.sweep).toEqual([]) + expect(reasonFor(plan, 'pty-1')).toBe('sweep has no usable evidence-age budget') + }) + it('never sweeps a PTY younger than the floor', () => { const plan = planRelayPtySweep( [orphan({ hostAgeMs: RELAY_PTY_SWEEP_MIN_AGE_MS - 1 })], diff --git a/src/shared/ssh-relay-pty-ownership-proof.ts b/src/shared/ssh-relay-pty-ownership-proof.ts index ed87be13610..866adbfb3e8 100644 --- a/src/shared/ssh-relay-pty-ownership-proof.ts +++ b/src/shared/ssh-relay-pty-ownership-proof.ts @@ -1,4 +1,7 @@ -import type { ForegroundProcessEvidence } from './foreground-process-evidence' +import { + isForegroundProcessEvidence, + type ForegroundProcessEvidence +} from './foreground-process-evidence' /** Which relay PTYs a client may prove it orphaned, and therefore may stop (#9819). * @@ -104,9 +107,10 @@ export const RELAY_PTY_SWEEP_MAX_PER_PASS = 8 export const RELAY_PTY_SWEEP_MAX_EVIDENCE_AGE_MS = 5_000 /** The host's own answer to "is anything running in this pane?". Only a positive "no" clears the - * sweep; every other shape — an older host, an unreadable process table, an observation too old to - * describe now, a named foreground process, any other process group on the pane's terminal — is a - * reason to leave the process alone. */ + * sweep; every other shape — an older host, a malformed record, an unreadable process table, an + * observation too old to describe now, a named foreground process, any other process group on the + * pane's terminal, any other member of the shell's own process group — is a reason to leave the + * process alone. */ function foregroundSkipReason( evidence: ForegroundProcessEvidence | undefined, context: RelayPtySweepContext @@ -116,6 +120,20 @@ function foregroundSkipReason( // observation is not the observation of absence. return 'host published no foreground-process observation' } + // The record reaches this decision straight off the wire — `mapSshPtyProcessList` validates the + // ownership fields and spreads the rest through, and `PtyProcessListAdmission` is not on the + // sweep path. Shape-check it here, because the age gate below is the one comparison in this file + // that a malformed field defaults toward the kill: a non-numeric `capturedAgeMs` makes the sum + // `NaN`, and `NaN > budget` is FALSE, so the entry would pass the freshness gate. + if (!isForegroundProcessEvidence(evidence)) { + return 'host foreground observation is malformed' + } + if ( + !Number.isFinite(context.evidenceAgeSinceListingMs) || + !Number.isFinite(context.maximumEvidenceAgeMs) + ) { + return 'sweep has no usable evidence-age budget' + } // Before anything is read out of it: an observation is only a claim about the instant it was // taken. Age is checked on both verdicts because a stale `unverifiable` is no better. if (evidence.capturedAgeMs + context.evidenceAgeSinceListingMs > context.maximumEvidenceAgeMs) { @@ -130,9 +148,11 @@ function foregroundSkipReason( return 'host observes a named foreground process' } if (evidence.shellOwnsEveryTtyProcessGroup !== true) { - // Something other than the shell's own process group is attached to the pane's terminal — a - // foreground command, a job backgrounded with `&`, a Ctrl-Z'd editor — or this host predates - // the field. The stop would SIGKILL that group, so none of those is a pane to reclaim. + // The host saw work inside the stop's blast radius: another process group on the pane's + // terminal (a foreground command, a job backgrounded with `&`, a Ctrl-Z'd editor), or another + // member of the shell's OWN process group (a `set +m` background job, a child that dropped the + // controlling terminal) — or this host predates the field. `killpg` reaches all of it, so none + // of those is a pane to reclaim. return 'host does not attest an idle shell' } return null From b1186c6beb58adbf796d1352040c0d6877045926 Mon Sep 17 00:00:00 2001 From: Jinjing <6427696+AmethystLiang@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:45:58 -0700 Subject: [PATCH 18/76] Fix scope of workspace-creation-project tour target (#18502) * fix: scope workspace-creation-project tour target to project picker only The tour target was previously applied to a container that included both the project picker and the run target picker below it. Restructure the layout to scope the target to only the project-related section, and add a test to verify the tour target does not span into the run target picker. * fix: scope workspace-creation-project tour target to project picker only Move the tour target attribute from the outer project section to an inner wrapper around just the combobox and its messages, excluding the header label and "Add project" button. Update tests to verify the narrower scope. --- .../NewWorkspaceComposerCard.test.tsx | 153 ++++++++---------- .../NewWorkspaceComposerProjectSection.tsx | 112 ++++++------- 2 files changed, 129 insertions(+), 136 deletions(-) diff --git a/src/renderer/src/components/NewWorkspaceComposerCard.test.tsx b/src/renderer/src/components/NewWorkspaceComposerCard.test.tsx index 8206354f916..1456f7e30ad 100644 --- a/src/renderer/src/components/NewWorkspaceComposerCard.test.tsx +++ b/src/renderer/src/components/NewWorkspaceComposerCard.test.tsx @@ -104,7 +104,7 @@ vi.mock('@/components/new-workspace/ProjectCombobox', () => ({ value: string | null onValueChange: (value: string) => void }) => ( -
+
{options.map((option) => ( + + + {translate('auto.components.NewWorkspaceComposerCard.d6b0a96f32', 'Add project')} + + + ) : null} +
+
+ + {projectError ? ( +

+ {projectError} +

+ ) : projectOptions.length === 0 ? ( +

+ {emptyProjectMessage ?? + translate( + 'auto.components.NewWorkspaceComposerCard.addProjectBeforeWorkspace', + 'Add a project before creating a workspace.' )} - > - - - - - {translate('auto.components.NewWorkspaceComposerCard.d6b0a96f32', 'Add project')} - - - ) : null} +

+ ) : null} +
- - {projectError ? ( -

- {projectError} -

- ) : projectOptions.length === 0 ? ( -

- {emptyProjectMessage ?? - translate( - 'auto.components.NewWorkspaceComposerCard.addProjectBeforeWorkspace', - 'Add a project before creating a workspace.' - )} -

- ) : null} {shouldShowRunTargetPicker ? (