From 65e513b193261f8b71a4e1b34c538bfa7f1d0c81 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:28:34 -0700 Subject: [PATCH] fix(daemon): recover lost macOS worker wake-ups in the packaged terminal runtime (#29133) Builds the macOS terminal host's Node 24.22.1 from pinned source with a libuv patch: idle threadpool workers recheck for queued work every 30 s instead of waiting forever, so a lost Darwin condition-variable wake-up (libuv #5280) no longer strands terminal I/O. Bumps the daemon protocol to v52 so new terminals start on the patched runtime right after an update; older daemons stay attachable. Adds stalled/recovered daemon async-I/O telemetry, and lets release workflows rebuild older refs that predate the source build. Fixes STA-11709, #31926. --- .github/workflows/adhoc-mac-build.yml | 26 +- .github/workflows/daily-mac-build.yml | 31 ++- .github/workflows/hourly-mac-build.yml | 36 +-- .github/workflows/pr.yml | 34 +++ .github/workflows/release-cut.yml | 11 +- .github/workflows/release-mac-build.yml | 18 +- .../darwin-worker-wait.json | 7 + .../darwin-worker-wait.patch | 46 ++++ config/reliability-gates.jsonc | 73 ++++++ .../darwin-terminal-worker-wait.c | 128 ++++++++++ .../__fixtures__/darwin-threadpool-wait.h | 68 ++++++ .../adhoc-windows-parallelism.test.mjs | 13 +- config/scripts/build-darwin-terminal-node.mjs | 222 ++++++++++++++++++ .../build-darwin-terminal-node.test.mjs | 92 ++++++++ config/scripts/darwin-terminal-node-probe.cjs | 80 +++++++ .../darwin-terminal-node-workflow.test.mjs | 147 ++++++++++++ .../scripts/darwin-terminal-worker-stress.cjs | 82 +++++++ config/scripts/macos-terminal-host-bundle.cjs | 6 +- .../mobile-release-check-scope.test.mjs | 1 + ...package-electron-runtime-contract.test.mjs | 2 +- .../pr-auxiliary-preflight-admission.test.mjs | 1 + config/scripts/pr-code-change-scope.mjs | 40 ++-- .../scripts/pr-darwin-terminal-node-route.mjs | 20 ++ ...lease-mac-build-workflow-dispatch.test.mjs | 82 ++++++- .../release-windows-signing-gate.test.mjs | 34 ++- .../run-release-mac-build-workflow.mjs | 22 +- .../scripts/verify-darwin-terminal-node.mjs | 118 ++++++++++ .../win32-test-lane-registration.test.mjs | 36 ++- .../bug-reproductions/darwin-worker-wakeup.md | 181 ++++++++++++++ .../reference/macos-daemon-tcc-attribution.md | 5 +- .../daemon/daemon-adoption-telemetry-event.ts | 11 +- .../daemon-async-io-telemetry-event.test.ts | 50 ++++ .../daemon/daemon-async-io-telemetry-event.ts | 55 +++++ ...n-foreground-confirmation-protocol.test.ts | 4 +- src/main/daemon/daemon-protocol-version.ts | 5 +- ...daemon-pty-adapter-async-io-health.test.ts | 16 ++ src/main/daemon/daemon-pty-adapter.ts | 13 +- src/main/ipc/telemetry.ts | 1 + src/shared/daemon-adoption-telemetry.ts | 7 + .../local-build-compatibility-contract.json | 4 +- .../local-build-compatibility-contract.ts | 4 +- src/shared/telemetry-daemon-event-schemas.ts | 18 ++ src/shared/telemetry-event-registry.ts | 2 + 43 files changed, 1758 insertions(+), 94 deletions(-) create mode 100644 config/node-runtime-patches/darwin-worker-wait.json create mode 100644 config/node-runtime-patches/darwin-worker-wait.patch create mode 100644 config/scripts/__fixtures__/darwin-terminal-worker-wait.c create mode 100644 config/scripts/__fixtures__/darwin-threadpool-wait.h create mode 100644 config/scripts/build-darwin-terminal-node.mjs create mode 100644 config/scripts/build-darwin-terminal-node.test.mjs create mode 100644 config/scripts/darwin-terminal-node-probe.cjs create mode 100644 config/scripts/darwin-terminal-node-workflow.test.mjs create mode 100644 config/scripts/darwin-terminal-worker-stress.cjs create mode 100644 config/scripts/pr-darwin-terminal-node-route.mjs create mode 100644 config/scripts/verify-darwin-terminal-node.mjs create mode 100644 docs/bug-reproductions/darwin-worker-wakeup.md create mode 100644 src/main/daemon/daemon-async-io-telemetry-event.test.ts create mode 100644 src/main/daemon/daemon-async-io-telemetry-event.ts diff --git a/.github/workflows/adhoc-mac-build.yml b/.github/workflows/adhoc-mac-build.yml index 7cabc3667bb..ed1fad297d1 100644 --- a/.github/workflows/adhoc-mac-build.yml +++ b/.github/workflows/adhoc-mac-build.yml @@ -267,8 +267,8 @@ jobs: matrix: arch: [x64, arm64] runs-on: blacksmith-12vcpu-macos-15 - # Allow 150min for install 3×10 + publish 2×45 + ~25 setup/build/verify; outer timeout would prevent cleanup. - timeout-minutes: 150 + # Allow 360min: 180 for a cold patched-runtime build + 150 for install/publish retries and setup. + timeout-minutes: 360 env: JOBS: max NODE_OPTIONS: --max-old-space-size=4096 @@ -308,7 +308,7 @@ jobs: pnpm-lock.yaml mobile/pnpm-lock.yaml - # Cache bundled host Node by exact pin, without restore keys, so bumps cannot restore old archives. + # Cache the patched host Node by exact pin and patch, without restore keys. - name: Locate the pinned Node runtime cache shell: bash run: echo "ORCA_NODE_RUNTIME_CACHE_DIR=$HOME/Library/Caches/orca-node-runtime" >>"$GITHUB_ENV" @@ -317,7 +317,7 @@ jobs: uses: actions/cache@v5 with: path: ~/Library/Caches/orca-node-runtime - key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts') }} + key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts', 'config/node-runtime-patches/**', 'config/scripts/build-darwin-terminal-node.mjs', 'config/scripts/pinned-node-downloads.mjs', 'config/scripts/update-node-runtime-pin.mjs', 'config/scripts/macos-terminal-host-bundle.cjs') }} - name: Cache electron-builder downloads uses: actions/cache@v5 @@ -338,6 +338,15 @@ jobs: retry_wait_seconds: 30 command: pnpm install --frozen-lockfile --cpu=current,x64,arm64 + # Why: older refs predate the source build and package the pinned download instead. + - name: Build and verify the patched terminal runtimes + if: hashFiles('config/scripts/build-darwin-terminal-node.mjs') != '' + timeout-minutes: 180 + run: | + pnpm run build:packages + ORCA_BACKGROUND_LAUNCH=1 node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/verify-darwin-terminal-node.mjs --arch=arm64 + ORCA_BACKGROUND_LAUNCH=1 node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/verify-darwin-terminal-node.mjs --arch=x64 + # Build required out/mobile-web after installing its React Native/Expo dependencies. - uses: ./.github/actions/install-mobile-dependencies if: needs.release-javascript.outputs.supported != 'true' @@ -423,7 +432,7 @@ jobs: timeout_minutes: 45 max_attempts: 2 retry_wait_seconds: 30 - command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_ADHOC=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac dmg zip --${{ matrix.arch }} --publish never + command: if [ ! -f config/scripts/build-darwin-terminal-node.mjs ]; then node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64; fi && node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_ADHOC=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac dmg zip --${{ matrix.arch }} --publish never env: ORCA_ADHOC_BUILD_VERSION: ${{ needs.orcad-template-support.outputs.version }} ORCA_BUILD_COMMIT: ${{ needs.orcad-template-support.outputs.commit }} @@ -656,12 +665,12 @@ jobs: fast_release: true javascript_source_sha: ${{ needs.release-javascript.outputs.supported == 'true' && needs.release-javascript.outputs.source_sha || '' }} - # Prefetch the installer during notarization; the publisher still waits for Mac success. + # Wait outside the publisher runner so cold runtime builds cannot exhaust its timeout. publish-adhoc-win: - needs: [orcad-template-support, build-adhoc-win] + needs: [orcad-template-support, build-adhoc-win, build-adhoc-mac] if: >- !cancelled() && needs.orcad-template-support.result == 'success' && - needs.build-adhoc-win.result == 'success' + needs.build-adhoc-win.result == 'success' && needs.build-adhoc-mac.result == 'success' permissions: contents: read actions: read @@ -670,4 +679,3 @@ jobs: with: channel: adhoc tag: v${{ needs.orcad-template-support.outputs.version }} - wait_for_mac_job: build-adhoc-mac diff --git a/.github/workflows/daily-mac-build.yml b/.github/workflows/daily-mac-build.yml index 38e20094ad6..dfec074a0dd 100644 --- a/.github/workflows/daily-mac-build.yml +++ b/.github/workflows/daily-mac-build.yml @@ -69,8 +69,8 @@ jobs: head_sha: ${{ steps.freshness.outputs.head_sha }} published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }} runs-on: blacksmith-6vcpu-macos-15 - # Allow 150min for install 3×10 + publish 2×45 + ~25 setup/build/verify, so outer timeout cannot strand cleanup mid-retry. - timeout-minutes: 150 + # Allow 360min: 180 for a cold patched-runtime build + 150 for install/publish retries and setup. + timeout-minutes: 360 env: NODE_OPTIONS: --max-old-space-size=4096 steps: @@ -143,7 +143,7 @@ jobs: pnpm-lock.yaml mobile/pnpm-lock.yaml - # Cache bundled host Node by exact pin without restore keys. + # Cache the patched host Node by exact pin and patch, without restore keys. - name: Locate the pinned Node runtime cache if: steps.freshness.outputs.should_build == 'true' shell: bash @@ -154,7 +154,7 @@ jobs: uses: actions/cache@v5 with: path: ~/Library/Caches/orca-node-runtime - key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts') }} + key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts', 'config/node-runtime-patches/**', 'config/scripts/build-darwin-terminal-node.mjs', 'config/scripts/pinned-node-downloads.mjs', 'config/scripts/update-node-runtime-pin.mjs', 'config/scripts/macos-terminal-host-bundle.cjs') }} - name: Cache electron-builder downloads if: steps.freshness.outputs.should_build == 'true' @@ -177,6 +177,14 @@ jobs: retry_wait_seconds: 30 command: pnpm install --frozen-lockfile --cpu=current,x64,arm64 + - name: Build and verify the patched terminal runtimes + if: steps.freshness.outputs.should_build == 'true' + timeout-minutes: 180 + run: | + pnpm run build:packages + ORCA_BACKGROUND_LAUNCH=1 node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/verify-darwin-terminal-node.mjs --arch=arm64 + ORCA_BACKGROUND_LAUNCH=1 node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/verify-darwin-terminal-node.mjs --arch=x64 + # Build required out/mobile-web using mobile’s React Native/Expo install. - uses: ./.github/actions/install-mobile-dependencies if: steps.freshness.outputs.should_build == 'true' @@ -193,12 +201,23 @@ jobs: APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }} APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} + # Why: App tokens last one hour, and the cold runtime build alone can outlast that. + - name: Re-mint daily repo token for versioning + id: app_token_version + if: steps.freshness.outputs.should_build == 'true' + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ secrets.HOURLY_RELEASE_APP_ID }} + private-key: ${{ secrets.HOURLY_RELEASE_APP_PRIVATE_KEY }} + owner: stablyai + repositories: orca-daily + - name: Compute daily version id: daily if: steps.freshness.outputs.should_build == 'true' shell: bash env: - GH_TOKEN: ${{ steps.app_token.outputs.token }} + GH_TOKEN: ${{ steps.app_token_version.outputs.token }} MAIN_REPO_TOKEN: ${{ github.token }} run: | set -euo pipefail @@ -303,7 +322,7 @@ jobs: timeout_minutes: 45 max_attempts: 2 retry_wait_seconds: 30 - command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && if [ -f config/scripts/build-mac-notarization-overlap.mjs ]; then ORCA_MAC_DAILY=1 node config/scripts/build-mac-notarization-overlap.mjs --publish always; else ORCA_MAC_DAILY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always; fi + command: node config/scripts/ensure-native-runtime.mjs --runtime=electron && if [ -f config/scripts/build-mac-notarization-overlap.mjs ]; then ORCA_MAC_DAILY=1 node config/scripts/build-mac-notarization-overlap.mjs --publish always; else ORCA_MAC_DAILY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always; fi env: # Give the publisher App write access to its configured orca-daily target. GH_TOKEN: ${{ steps.app_token_publish.outputs.token }} diff --git a/.github/workflows/hourly-mac-build.yml b/.github/workflows/hourly-mac-build.yml index ca4ee37aa76..1e28bf7db0a 100644 --- a/.github/workflows/hourly-mac-build.yml +++ b/.github/workflows/hourly-mac-build.yml @@ -105,8 +105,8 @@ jobs: head_sha: ${{ needs.preflight.outputs.head_sha }} published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }} runs-on: blacksmith-6vcpu-macos-15 - # Allow 150min for install 3×10 + publish 2×45 + ~25 setup/build/verify; outer timeout otherwise interrupts cleanup. - timeout-minutes: 150 + # Allow 360min: 180 for a cold patched-runtime build + 150 for install/publish retries and setup. + timeout-minutes: 360 env: NODE_OPTIONS: --max-old-space-size=4096 steps: @@ -119,15 +119,6 @@ jobs: # Read this repo without persisted credentials; write orca-hourly via env App token (zizmor: artipacked). persist-credentials: false - - name: Mint hourly repo token - id: app_token - uses: actions/create-github-app-token@v2 - with: - app-id: ${{ secrets.HOURLY_RELEASE_APP_ID }} - private-key: ${{ secrets.HOURLY_RELEASE_APP_PRIVATE_KEY }} - owner: stablyai - repositories: orca-hourly - - name: Setup pnpm uses: ./.github/actions/setup-pnpm @@ -141,7 +132,7 @@ jobs: pnpm-lock.yaml mobile/pnpm-lock.yaml - # Cache bundled host Node by exact pin without restore keys. + # Cache the patched host Node by exact pin and patch, without restore keys. - name: Locate the pinned Node runtime cache shell: bash run: echo "ORCA_NODE_RUNTIME_CACHE_DIR=$HOME/Library/Caches/orca-node-runtime" >>"$GITHUB_ENV" @@ -150,7 +141,7 @@ jobs: uses: actions/cache@v5 with: path: ~/Library/Caches/orca-node-runtime - key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts') }} + key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts', 'config/node-runtime-patches/**', 'config/scripts/build-darwin-terminal-node.mjs', 'config/scripts/pinned-node-downloads.mjs', 'config/scripts/update-node-runtime-pin.mjs', 'config/scripts/macos-terminal-host-bundle.cjs') }} - name: Cache electron-builder downloads uses: actions/cache@v5 @@ -171,6 +162,13 @@ jobs: retry_wait_seconds: 30 command: pnpm install --frozen-lockfile --cpu=current,x64,arm64 + - name: Build and verify the patched terminal runtimes + timeout-minutes: 180 + run: | + pnpm run build:packages + ORCA_BACKGROUND_LAUNCH=1 node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/verify-darwin-terminal-node.mjs --arch=arm64 + ORCA_BACKGROUND_LAUNCH=1 node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/verify-darwin-terminal-node.mjs --arch=x64 + # Build required out/mobile-web from mobile’s React Native/Expo install. - uses: ./.github/actions/install-mobile-dependencies @@ -184,6 +182,16 @@ jobs: APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }} APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} + # Why here: App tokens last one hour, and the cold runtime build alone can outlast that. + - name: Mint hourly repo token + id: app_token + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ secrets.HOURLY_RELEASE_APP_ID }} + private-key: ${{ secrets.HOURLY_RELEASE_APP_PRIVATE_KEY }} + owner: stablyai + repositories: orca-hourly + - name: Compute hourly version id: hourly shell: bash @@ -282,7 +290,7 @@ jobs: timeout_minutes: 45 max_attempts: 2 retry_wait_seconds: 30 - command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && if [ -f config/scripts/build-mac-notarization-overlap.mjs ]; then ORCA_MAC_HOURLY=1 node config/scripts/build-mac-notarization-overlap.mjs --publish always; else ORCA_MAC_HOURLY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always; fi + command: node config/scripts/ensure-native-runtime.mjs --runtime=electron && if [ -f config/scripts/build-mac-notarization-overlap.mjs ]; then ORCA_MAC_HOURLY=1 node config/scripts/build-mac-notarization-overlap.mjs --publish always; else ORCA_MAC_HOURLY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always; fi env: # Give publisher token write access to configured orca-hourly repo. GH_TOKEN: ${{ steps.app_token_publish.outputs.token }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8245654acae..c2a88e2a909 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -50,6 +50,7 @@ jobs: windows_process_probe: ${{ steps.filter.outputs.windows_process_probe }} package_windows: ${{ steps.readiness.outputs.reused != 'true' && steps.filter.outputs.package_windows }} layout_oracle: ${{ steps.readiness.outputs.reused != 'true' && steps.filter.outputs.layout_oracle }} + darwin_terminal_node: ${{ steps.readiness.outputs.reused != 'true' && steps.filter.outputs.darwin_terminal_node }} dependency_isolation: ${{ steps.readiness.outputs.reused != 'true' && steps.filter.outputs.dependency_isolation }} e2e_should_run: ${{ steps.e2e_filter.outputs.should_run }} test_files: ${{ steps.e2e_filter.outputs.test_files }} @@ -76,6 +77,7 @@ jobs: /config/.bun-version /config/scripts/check-readme-local-links.mjs /config/scripts/pr-code-change-scope.mjs + /config/scripts/pr-darwin-terminal-node-route.mjs /.oxfmtrc.json /config/scripts/pr-package-test-paths.mjs /config/scripts/pr-layout-oracle-route.mjs @@ -1494,6 +1496,34 @@ jobs: *) echo "node_modules/zod is not in the isolated store"; exit 1 ;; esac + darwin_terminal_node: + name: mac terminal worker wakeup (${{ matrix.os }}) + needs: [code_paths, preflight] + if: >- + !cancelled() && needs.preflight.result == 'success' && + needs.code_paths.outputs.darwin_terminal_node == 'true' + strategy: + fail-fast: false + matrix: + os: [macos-15, macos-15-intel] + runs-on: ${{ matrix.os }} + timeout-minutes: 180 + env: + ORCA_BACKGROUND_LAUNCH: '1' + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: ./.github/actions/install-node-dependencies + with: + native-runtime: node + - uses: actions/cache@v5 + with: + path: out/node-runtime-cache + key: darwin-terminal-node-${{ runner.arch }}-${{ hashFiles('src/shared/node-runtime-pin.ts', 'config/node-runtime-patches/**', 'config/scripts/build-darwin-terminal-node.mjs', 'config/scripts/pinned-node-downloads.mjs', 'config/scripts/update-node-runtime-pin.mjs', 'config/scripts/macos-terminal-host-bundle.cjs') }} + - name: Build and verify the shipping terminal runtime + run: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/verify-darwin-terminal-node.mjs + verify: name: PR checks passed # Publish an explicit result after every required dependency settles. @@ -1511,6 +1541,7 @@ jobs: - managed_hook_node18 - package - package_windows + - darwin_terminal_node - layout_oracle - dependency_isolation # Report settled results without competing with the GitHub-hosted test queue. @@ -1547,6 +1578,8 @@ jobs: PACKAGE_SHOULD_RUN: ${{ needs.code_paths.outputs.package }} PACKAGE_WINDOWS: ${{ needs.package_windows.result }} PACKAGE_WINDOWS_SHOULD_RUN: ${{ needs.code_paths.outputs.package_windows }} + DARWIN_TERMINAL_NODE: ${{ needs.darwin_terminal_node.result }} + DARWIN_TERMINAL_NODE_SHOULD_RUN: ${{ needs.code_paths.outputs.darwin_terminal_node }} LAYOUT_ORACLE: ${{ needs.layout_oracle.result }} LAYOUT_ORACLE_SHOULD_RUN: ${{ needs.code_paths.outputs.layout_oracle }} DEPENDENCY_ISOLATION: ${{ needs.dependency_isolation.result }} @@ -1592,6 +1625,7 @@ jobs: check_job managed_hook_node18 "$MANAGED_HOOK_NODE18" "$MANAGED_HOOK_NODE18_SHOULD_RUN" check_job package "$PACKAGE" "$PACKAGE_SHOULD_RUN" check_job package_windows "$PACKAGE_WINDOWS" "$PACKAGE_WINDOWS_SHOULD_RUN" + check_job darwin_terminal_node "$DARWIN_TERMINAL_NODE" "$DARWIN_TERMINAL_NODE_SHOULD_RUN" check_job layout_oracle "$LAYOUT_ORACLE" "$LAYOUT_ORACLE_SHOULD_RUN" check_job dependency_isolation "$DEPENDENCY_ISOLATION" "$DEPENDENCY_ISOLATION_SHOULD_RUN" exit "$failed" diff --git a/.github/workflows/release-cut.yml b/.github/workflows/release-cut.yml index 7ae1f1d613e..f126320066d 100644 --- a/.github/workflows/release-cut.yml +++ b/.github/workflows/release-cut.yml @@ -1071,6 +1071,7 @@ jobs: - release-preflight - release-javascript - build-mac + - verify-mac-build # Why not the implicit success(): a tag without the orcad template skips that job on purpose. if: >- !cancelled() && @@ -1080,6 +1081,7 @@ jobs: needs.release-preflight.result == 'success' && needs.release-javascript.result == 'success' && needs.build-mac.result == 'success' && + needs.verify-mac-build.result == 'success' && (needs.orcad-template.result == 'success' || (needs.orcad-template.result == 'skipped' && needs.cut.outputs.ships_orcad_template == 'false')) env: @@ -1467,7 +1469,7 @@ jobs: path: signing-stage/** if-no-files-found: error - # Unsigned packaging overlaps macOS; only successful macOS builds may consume signing quota. + # Recheck the exact successful macOS run before consuming signing quota. - name: Require macOS success before Windows signing id: mac-signing-gate if: matrix.platform == 'win' && github.run_attempt == 1 @@ -2078,6 +2080,7 @@ jobs: (needs.orcad-template.result == 'skipped' && needs.cut.outputs.ships_orcad_template == 'false')) # Keep every SignPath workflow job GitHub-hosted; separate release-mac-build allows Blacksmith outside Windows provenance. runs-on: ubuntu-latest + timeout-minutes: 185 permissions: actions: write contents: read @@ -2106,20 +2109,20 @@ jobs: RELEASE_MAC_BUILD_JAVASCRIPT_SOURCE_SHA: ${{ needs.release-javascript.outputs.supported == 'true' && needs.release-javascript.outputs.source_sha || '' }} RELEASE_MAC_BUILD_TAG: ${{ needs.cut.outputs.tag }} RELEASE_MAC_BUILD_WORKFLOW: release-mac-build.yml + RELEASE_MAC_BUILD_OBSERVE_MINUTES: '180' verify-mac-build: needs: - cut - build-mac - - build - # Qualify macOS even when a sibling build fails; publication still requires every build. + # Finish the shared deadline before starting the Windows signing job's six-hour budget. if: >- !cancelled() && needs.cut.result == 'success' && needs.cut.outputs.should_release == 'true' && needs.build-mac.result == 'success' runs-on: ubuntu-latest - timeout-minutes: 95 + timeout-minutes: 215 permissions: actions: read contents: read diff --git a/.github/workflows/release-mac-build.yml b/.github/workflows/release-mac-build.yml index b6a752f6177..e02ae146e5e 100644 --- a/.github/workflows/release-mac-build.yml +++ b/.github/workflows/release-mac-build.yml @@ -49,7 +49,8 @@ jobs: if: ${{ !cancelled() && (!inputs.qualification_only || needs.qualification-template.result == 'success') && github.repository == 'stablyai/orca' }} # Keep Blacksmith mac notarization outside SignPath’s GitHub-hosted Windows provenance. runs-on: blacksmith-6vcpu-macos-15 - timeout-minutes: 60 + # Includes the 180-minute cold runtime build and the existing packaging retry budget. + timeout-minutes: 360 # actions: read fetches the release-cut run's artifacts. permissions: actions: read @@ -95,7 +96,7 @@ jobs: pnpm-lock.yaml mobile/pnpm-lock.yaml - # Cache bundled host Node by exact pin, without restore keys. + # Cache the patched host Node by exact pin and patch, without restore keys. - name: Locate the pinned Node runtime cache shell: bash run: echo "ORCA_NODE_RUNTIME_CACHE_DIR=$HOME/Library/Caches/orca-node-runtime" >>"$GITHUB_ENV" @@ -104,7 +105,7 @@ jobs: uses: actions/cache@v5 with: path: ~/Library/Caches/orca-node-runtime - key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts') }} + key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts', 'config/node-runtime-patches/**', 'config/scripts/build-darwin-terminal-node.mjs', 'config/scripts/pinned-node-downloads.mjs', 'config/scripts/update-node-runtime-pin.mjs', 'config/scripts/macos-terminal-host-bundle.cjs') }} # Cache Electron/builder tool downloads to save ~30–90s. - name: Cache electron-builder downloads @@ -126,6 +127,15 @@ jobs: retry_wait_seconds: 30 command: pnpm install --frozen-lockfile --cpu=current,x64,arm64 + # Why: older refs predate the source build and package the pinned download instead. + - name: Build and verify the patched terminal runtimes + if: hashFiles('config/scripts/build-darwin-terminal-node.mjs') != '' + timeout-minutes: 180 + run: | + pnpm run build:packages + ORCA_BACKGROUND_LAUNCH=1 node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/verify-darwin-terminal-node.mjs --arch=arm64 + ORCA_BACKGROUND_LAUNCH=1 node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/verify-darwin-terminal-node.mjs --arch=x64 + # Build required out/mobile-web using mobile’s React Native/Expo install. - uses: ./.github/actions/install-mobile-dependencies @@ -225,7 +235,7 @@ jobs: timeout_minutes: 45 max_attempts: 3 retry_wait_seconds: 30 - command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && if [ -f config/scripts/build-mac-notarization-overlap.mjs ]; then ORCA_MAC_RELEASE=1 node config/scripts/build-mac-notarization-overlap.mjs --publish "$ORCA_MAC_PUBLISH" --draft; else ORCA_MAC_RELEASE=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish "$ORCA_MAC_PUBLISH" -c.publish.releaseType=draft; fi + command: if [ ! -f config/scripts/build-darwin-terminal-node.mjs ]; then node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64; fi && node config/scripts/ensure-native-runtime.mjs --runtime=electron && if [ -f config/scripts/build-mac-notarization-overlap.mjs ]; then ORCA_MAC_RELEASE=1 node config/scripts/build-mac-notarization-overlap.mjs --publish "$ORCA_MAC_PUBLISH" --draft; else ORCA_MAC_RELEASE=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish "$ORCA_MAC_PUBLISH" -c.publish.releaseType=draft; fi env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} ORCA_REQUIRE_ORCAD_TEMPLATE: '1' diff --git a/config/node-runtime-patches/darwin-worker-wait.json b/config/node-runtime-patches/darwin-worker-wait.json new file mode 100644 index 00000000000..47aedc804bc --- /dev/null +++ b/config/node-runtime-patches/darwin-worker-wait.json @@ -0,0 +1,7 @@ +{ + "version": "24.22.1", + "sourceArchive": "node-v24.22.1.tar.gz", + "sourceSha256": "001773d9bd53fbd27a25efd42aa6a9693ebfe500998ca9bec635e9b6ee79bc91", + "recipeVersion": 1, + "workerSourceSha256": "06ab0be8159c22cfb5f4a715d776693825835f1369c7c82d13a22e607947b4fd" +} diff --git a/config/node-runtime-patches/darwin-worker-wait.patch b/config/node-runtime-patches/darwin-worker-wait.patch new file mode 100644 index 00000000000..f2a29bd4f60 --- /dev/null +++ b/config/node-runtime-patches/darwin-worker-wait.patch @@ -0,0 +1,46 @@ +--- a/deps/uv/src/threadpool.c ++++ b/deps/uv/src/threadpool.c +@@ -26,6 +26,9 @@ + #endif + + #include ++#if defined(__APPLE__) && defined(__MACH__) ++# include ++#endif + + #define MAX_THREADPOOL_SIZE 1024 + +@@ -51,6 +54,24 @@ + } + + ++static void wait_for_work(void) { ++#if defined(__APPLE__) && defined(__MACH__) ++ /* Bound idle waits so Darwin psynch lost wakeups can recover. */ ++ const struct timespec timeout = { 30, 0 }; ++ int r; ++ ++ errno = 0; ++ r = pthread_cond_timedwait_relative_np(&cond, &mutex, &timeout); ++ /* Preserve uv_cond_wait's EINVAL/EBUSY workaround for Darwin. */ ++ if (r == 0 || r == ETIMEDOUT || (r == EINVAL && errno == EBUSY)) ++ return; ++ abort(); ++#else ++ uv_cond_wait(&cond, &mutex); ++#endif ++} ++ ++ + /* To avoid deadlock with uv_cancel() it's crucial that the worker + * never holds the global mutex and the loop-local mutex at the same time. + */ +@@ -74,7 +95,7 @@ + uv__queue_next(&run_slow_work_message) == &wq && + slow_io_work_running >= slow_work_thread_threshold())) { + idle_threads += 1; +- uv_cond_wait(&cond, &mutex); ++ wait_for_work(); + idle_threads -= 1; + } + diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index dfe4db1bcde..3c9a7c44c6e 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -484,6 +484,79 @@ ], "demotionRule": "Keep experimental or demote if recovery signals an unverified owner, treats loss of contact as exit, reports success before descendant exit, adds background scanning, or lets a recovery failure block the current-daemon restart." }, + { + "id": "terminal-runtime.darwin-worker-wakeup", + "title": "Idle Darwin workers refresh their waits and continue delivering asynchronous terminal work", + "maturity": "experimental", + "protection": "partial", + "owner": "terminal-runtime", + "layer": "native-runtime", + "surfaces": ["packaged macOS terminal helper", "terminal input", "terminal creation"], + "platforms": ["macos"], + "providers": ["daemon", "local", "ssh", "remote-runtime", "mobile-relay"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["daemon"], + "coverageNotes": "The repository arm64 build and x64 build under Rosetta passed real PTY input, filesystem/DNS/crypto and idle-wait generation checks; physical ARM and Intel CI passed on Node 24.21.0. Node 24.22.1 arm64 passed the same native oracle locally; its official unpatched runtime failed idle refresh. Fresh physical ARM/Intel CI remains required. An isolated signed installed helper with patched Node also served a PTY. A complete package build and older macOS remain unverified. Standalone/SSH servers and the Electron fallback do not receive the patch; mobile/relay inherit only their execution host runtime.", + "motivatingLinks": [ + "docs/bug-reproductions/darwin-worker-wakeup.md", + "https://github.com/libuv/libuv/pull/5351" + ], + "invariant": "A packaged Darwin terminal worker must not remain indefinitely parked on a stale condition generation while asynchronous work is queued; process and session ownership must remain unchanged.", + "oracle": "Four real idle workers each advance their low 32-bit psynch wait generation within the 31-second observation interval, with at most sixteen generation steps, and real PTY input plus filesystem/DNS/crypto work completes. The upstream half-range witness stalls unpatched workers with queued jobs and completes with the patch.", + "commands": ["ORCA_BACKGROUND_LAUNCH=1 node config/scripts/verify-darwin-terminal-node.mjs"], + "testFiles": ["config/scripts/verify-darwin-terminal-node.mjs"], + "assertionRefs": [ + { + "file": "config/scripts/verify-darwin-terminal-node.mjs", + "assertions": ["Runtime proof failed"] + } + ], + "evidenceRuns": [ + { + "date": "2026-10-23", + "runner": "local", + "platform": "macos", + "command": "ORCA_BACKGROUND_LAUNCH=1 node config/scripts/verify-darwin-terminal-node.mjs", + "result": "passed", + "durationSeconds": 2231, + "summary": "Cold source build of pinned Node 24.22.1 arm64 passed architecture/minimum-OS checks, real PTY input, fs/DNS/crypto and all four bounded idle-wait generation checks. Official Node 24.22.1 failed the same idle-refresh assertion after passing the initial PTY and I/O checks. Timing includes cold compilation and source cleanup." + }, + { + "date": "2026-10-15", + "runner": "manual", + "platform": "macos", + "command": "ORCA_BACKGROUND_LAUNCH=1 node config/scripts/verify-darwin-terminal-node.mjs", + "result": "passed", + "durationSeconds": 32, + "summary": "Repository-built Node 24.21.0 arm64 passed real PTY input, fs/DNS/crypto and all four bounded idle-generation checks. Cached verifier execution took about 32 seconds; this is not full package-build proof." + } + ], + "runtimeBudget": { + "p95Seconds": 10800, + "scope": "Cold pinned Node source compilation dominates; Intel exceeded the initial 90-minute job allowance, now 180 minutes. Cached native runtime probe takes about 32 seconds. No CI soak yet." + }, + "performanceBudget": { + "required": true, + "evidence": "One relative timeout per idle worker per 30 seconds, at most eight wakes/minute for the default pool; immediate signals remain. Generation advancement is bounded by the native probe. No application polling or session scans added." + }, + "flakeHistory": { + "status": "not-started", + "evidence": "Focused native and manual Node checks passed on arm64; both physical architectures passed on Node 24.21.0; the updated recipe and repeated CI soak remain pending." + }, + "redGreenEvidence": { + "status": "complete", + "evidence": "Upstream real-worker half-range witness: baseline four idle workers with zero jobs/callbacks and queued work; patch four jobs/callbacks and empty queue. An ordinary sequential workload also reproduced the exact half-range signature, zero fresh stat callbacks and no real PTY echo after 15 seconds. The patched runtime completed 30 million sequential jobs, a fresh stat and real PTY echo. Idle-generation regression fails without the patch and passes with it." + }, + "promotionCriteria": [ + "Collect physical arm64/Intel CI, signed helper launch and older macOS evidence, then repeated soak before promotion." + ], + "knownGaps": [ + "Ordinary sequential jobs reproduced half-range waiter starvation and PTY input loss; scheduling is intermittent and the exact Orca job-producer sequence remains unknown.", + "No older macOS, full package build, launchd/TCC attribution or battery measurement yet.", + "Existing daemons require deliberate restart; standalone/SSH macOS servers and Electron fallback remain unpatched." + ], + "demotionRule": "Keep experimental until native runtime and signed packaging evidence is stable; investigate failures without weakening generation bounds or silently using an official unpatched binary." + }, { "id": "agent-session.delayed-terminal-launch", "title": "Delayed terminal launches retain their harness preparation and recover process discovery", diff --git a/config/scripts/__fixtures__/darwin-terminal-worker-wait.c b/config/scripts/__fixtures__/darwin-terminal-worker-wait.c new file mode 100644 index 00000000000..449489369f1 --- /dev/null +++ b/config/scripts/__fixtures__/darwin-terminal-worker-wait.c @@ -0,0 +1,128 @@ +#include +#include +#include +#include "darwin-threadpool-wait.h" + +static uv_work_t requests[4]; +static uv_thread_t workers[4]; +static uv_sem_t arrived; +static uv_sem_t release_workers; +static unsigned callbacks; +static int started; +static uv_work_t stress_request; +static uv_loop_t* stress_loop; +static unsigned stress_completed; +static const unsigned stress_target = 30000000; + +static void stress_work(uv_work_t* request) { + (void) request; +} + +static void stress_done(uv_work_t* request, int status) { + if (status != 0) + abort(); + if (++stress_completed < stress_target && + uv_queue_work(stress_loop, request, stress_work, stress_done) != 0) + abort(); +} + +static napi_value start_stress(napi_env env, napi_callback_info info) { + napi_value result; + (void) info; + if (stress_loop != NULL || + napi_get_uv_event_loop(env, &stress_loop) != napi_ok || + uv_queue_work(stress_loop, &stress_request, stress_work, stress_done) != 0) + abort(); + napi_get_undefined(env, &result); + return result; +} + +static napi_value stress_progress(napi_env env, napi_callback_info info) { + napi_value result; + (void) info; + napi_create_uint32(env, stress_completed, &result); + return result; +} + +static void enroll_worker(uv_work_t* request) { + workers[request - requests] = uv_thread_self(); + uv_sem_post(&arrived); + uv_sem_wait(&release_workers); +} + +static void complete_work(uv_work_t* request, int status) { + (void) request; + if (status != 0) + abort(); + if (++callbacks == 4) { + uv_sem_destroy(&arrived); + uv_sem_destroy(&release_workers); + } +} + +static napi_value enroll(napi_env env, napi_callback_info info) { + uv_loop_t* loop; + napi_value result; + unsigned i; + (void) info; + if (started) { + napi_throw_error(env, NULL, "Workers already enrolled"); + return NULL; + } + started = 1; + if (napi_get_uv_event_loop(env, &loop) != napi_ok || + uv_sem_init(&arrived, 0) != 0 || + uv_sem_init(&release_workers, 0) != 0) + abort(); + for (i = 0; i < 4; i++) + if (uv_queue_work(loop, requests + i, enroll_worker, complete_work) != 0) + abort(); + for (i = 0; i < 4; i++) + uv_sem_wait(&arrived); + for (i = 0; i < 4; i++) + uv_sem_post(&release_workers); + napi_get_undefined(env, &result); + return result; +} + +static napi_value snapshot(napi_env env, napi_callback_info info) { + char output[2048]; + uintptr_t condition; + uint64_t generation; + napi_value result; + unsigned i; + int at; + int parked; + (void) info; + if (!started) { + napi_throw_error(env, NULL, "Enroll workers first"); + return NULL; + } + at = snprintf(output, sizeof(output), + "{\"callbacks\":%u,\"waits\":[", callbacks); + for (i = 0; i < 4; i++) { + generation = 0; + parked = read_wait_generation(workers[i], &condition, &generation); + at += snprintf(output + at, sizeof(output) - at, + "%s{\"parked\":%d,\"generation\":\"%llx\"}", + i ? "," : "", parked, (unsigned long long) generation); + } + snprintf(output + at, sizeof(output) - at, "]}"); + napi_create_string_utf8(env, output, NAPI_AUTO_LENGTH, &result); + return result; +} + +NAPI_MODULE_INIT() { + napi_value function; + napi_create_function(env, "enroll", NAPI_AUTO_LENGTH, enroll, NULL, &function); + napi_set_named_property(env, exports, "enroll", function); + napi_create_function(env, "snapshot", NAPI_AUTO_LENGTH, snapshot, NULL, &function); + napi_set_named_property(env, exports, "snapshot", function); + napi_create_function(env, "startStress", NAPI_AUTO_LENGTH, + start_stress, NULL, &function); + napi_set_named_property(env, exports, "startStress", function); + napi_create_function(env, "stressProgress", NAPI_AUTO_LENGTH, + stress_progress, NULL, &function); + napi_set_named_property(env, exports, "stressProgress", function); + return exports; +} diff --git a/config/scripts/__fixtures__/darwin-threadpool-wait.h b/config/scripts/__fixtures__/darwin-threadpool-wait.h new file mode 100644 index 00000000000..e7abe2c2260 --- /dev/null +++ b/config/scripts/__fixtures__/darwin-threadpool-wait.h @@ -0,0 +1,68 @@ +/* Copyright Joyent, Inc. and other Node contributors. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#ifndef THREADPOOL_DARWIN_WAIT_H_ +#define THREADPOOL_DARWIN_WAIT_H_ + +#include "uv.h" +#include +#include +#include +#include +#include +#include + +static int read_wait_generation(uv_thread_t worker, + uintptr_t* condition, + uint64_t* generation) { + mach_msg_type_number_t count; + kern_return_t result; + uintptr_t pc; + Dl_info symbol; +# if defined(__aarch64__) + arm_thread_state64_t state = {0}; + count = ARM_THREAD_STATE64_COUNT; + result = thread_get_state(pthread_mach_thread_np(worker), + ARM_THREAD_STATE64, + (thread_state_t) &state, + &count); + pc = arm_thread_state64_get_pc(state); + *generation = state.__x[1]; + *condition = state.__x[0]; +# else + x86_thread_state64_t state = {0}; + count = x86_THREAD_STATE64_COUNT; + result = thread_get_state(pthread_mach_thread_np(worker), + x86_THREAD_STATE64, + (thread_state_t) &state, + &count); + pc = state.__rip; + *generation = state.__rsi; + *condition = state.__rdi; +# endif + if (result != KERN_SUCCESS) + return 0; + if (!dladdr((void*) pc, &symbol) || symbol.dli_sname == NULL) + return 0; + return strcmp(symbol.dli_sname, "__psynch_cvwait") == 0; +} + +#endif diff --git a/config/scripts/adhoc-windows-parallelism.test.mjs b/config/scripts/adhoc-windows-parallelism.test.mjs index a1096645624..a797d42c52d 100644 --- a/config/scripts/adhoc-windows-parallelism.test.mjs +++ b/config/scripts/adhoc-windows-parallelism.test.mjs @@ -117,15 +117,20 @@ describe('adhoc Windows compilation overlaps Mac without publishing early', () = }) it.each(['success', 'failure', 'cancelled', 'skipped', ''])( - 'prefetches only after Windows compilation succeeds: %s', + 'publishes only after both Windows compilation and macOS publication succeed: %s', (result) => { const run = compileAggregateExpression(publish.if) const needs = { 'orcad-template-support': { result: 'success' }, - 'build-adhoc-win': { result: 'success' } + 'build-adhoc-win': { result: 'success' }, + 'build-adhoc-mac': { result: 'success' } } - expect(publish.needs).toEqual(['orcad-template-support', 'build-adhoc-win']) - expect(publish.with.wait_for_mac_job).toBe('build-adhoc-mac') + expect(publish.needs).toEqual([ + 'orcad-template-support', + 'build-adhoc-win', + 'build-adhoc-mac' + ]) + expect(publish.with.wait_for_mac_job).toBeUndefined() expect(run({ needs }).coerceString()).toBe('true') expect(run({ needs }, true).coerceString()).toBe('false') for (const job of publish.needs) { diff --git a/config/scripts/build-darwin-terminal-node.mjs b/config/scripts/build-darwin-terminal-node.mjs new file mode 100644 index 00000000000..5bfe022e18a --- /dev/null +++ b/config/scripts/build-darwin-terminal-node.mjs @@ -0,0 +1,222 @@ +import { createHash } from 'node:crypto' +import { + chmodSync, + copyFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + renameSync, + rmSync, + writeFileSync +} from 'node:fs' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { lock } from 'proper-lockfile' +import { runProcess } from '@orca/process-host' +import { + NODE_RUNTIME_DARWIN_MINIMUM_OS, + NODE_RUNTIME_PIN +} from '../../src/shared/node-runtime-pin.ts' +import { fetchPinnedFile, pinnedNodeCacheDir } from './pinned-node-downloads.mjs' +import { sha256File } from './update-node-runtime-pin.mjs' +import { getTarProgram } from './zip-extractor-command.mjs' +import { isDirectInvocation } from './script-entry-detection.mjs' + +const root = resolve(import.meta.dirname, '../..') +const patchPath = join(root, 'config/node-runtime-patches/darwin-worker-wait.patch') +const recipe = JSON.parse( + readFileSync(join(root, 'config/node-runtime-patches/darwin-worker-wait.json'), 'utf8') +) +const require = createRequire(import.meta.url) +const { lipoArchsToNodeArchs, parseBuildVersionMinos } = require('./macos-terminal-host-bundle.cjs') +const builds = new Map() + +export function darwinTerminalNodeInputs(arch, version = NODE_RUNTIME_PIN.version) { + if (arch !== 'arm64' && arch !== 'x64') { + throw new Error(`Unsupported macOS terminal Node architecture: ${arch}`) + } + if (version !== recipe.version) { + throw new Error(`Review the Darwin worker-wait patch before moving Node to ${version}`) + } + return { + ...recipe, + arch, + ...(arch !== process.arch + ? { + crossBuildMode: arch === 'x64' ? 'target-tools-under-rosetta' : 'separate-host-tools' + } + : {}), + patchSha256: createHash('sha256').update(readFileSync(patchPath)).digest('hex'), + buildScriptsSha256: Object.fromEntries( + [ + 'build-darwin-terminal-node.mjs', + 'pinned-node-downloads.mjs', + 'update-node-runtime-pin.mjs', + 'macos-terminal-host-bundle.cjs' + ].map((file) => [ + file, + createHash('sha256') + .update(readFileSync(join(import.meta.dirname, file))) + .digest('hex') + ]) + ), + minimumOs: NODE_RUNTIME_DARWIN_MINIMUM_OS + } +} + +export async function readBuiltDarwinTerminalNode(directory, inputs) { + try { + const receipt = JSON.parse(readFileSync(join(directory, 'receipt.json'), 'utf8')) + if (JSON.stringify(receipt.inputs) !== JSON.stringify(inputs)) { + return null + } + const executable = join(directory, 'node') + return (await sha256File(executable)) === receipt.executableSha256 ? executable : null + } catch { + return null + } +} + +async function runBuildCommand(program, args, options = {}) { + const result = await runProcess( + { + program, + args, + timeoutMs: 180 * 60_000, + maxOutputBytes: 1024 * 1024, + ...options + }, + 'tail' + ) + if (result.code !== 0) { + throw new Error(`${program} failed (${result.code}):\n${result.stderr}\n${result.stdout}`) + } + return result.stdout +} + +async function compileDarwinTerminalNode(directory, inputs, cacheDir, scratch) { + const archivePath = join(cacheDir, inputs.sourceArchive) + const releaseArchive = await lock(archivePath, { + realpath: false, + stale: 60_000, + retries: { retries: 2160, minTimeout: 5_000, maxTimeout: 5_000 } + }) + let archive + try { + archive = await fetchPinnedFile({ + url: `https://nodejs.org/dist/v${inputs.version}/${inputs.sourceArchive}`, + destination: archivePath, + sha256: inputs.sourceSha256 + }) + } finally { + await releaseArchive() + } + const source = join(scratch, `node-v${inputs.version}`) + await runBuildCommand(getTarProgram(), [ + '-xzf', + archive, + '-C', + scratch, + `node-v${inputs.version}` + ]) + await runBuildCommand('patch', ['--batch', '--forward', '--fuzz=0', '-p1', '-i', patchPath], { + cwd: source + }) + if ((await sha256File(join(source, 'deps/uv/src/threadpool.c'))) !== inputs.workerSourceSha256) { + throw new Error('The Node worker source does not match the reviewed patch') + } + const env = { + ...process.env, + ORCA_BACKGROUND_LAUNCH: '1', + MACOSX_DEPLOYMENT_TARGET: inputs.minimumOs + } + const args = ['--without-npm', '--without-corepack', `--dest-cpu=${inputs.arch}`] + if (process.arch !== inputs.arch) { + const compilerArch = inputs.arch === 'x64' ? 'x86_64' : 'arm64' + env.CC = `clang -arch ${compilerArch}` + env.CXX = `clang++ -arch ${compilerArch}` + if (inputs.arch === 'x64') { + await runBuildCommand('/usr/bin/arch', ['-x86_64', '/usr/bin/true']) + env.CC_host = env.CC + env.CXX_host = env.CXX + args.push('--no-cross-compiling') + } else { + env.CC_host = 'clang -arch x86_64' + env.CXX_host = 'clang++ -arch x86_64' + args.push('--cross-compiling') + } + } + await runBuildCommand(join(source, 'configure'), args, { cwd: source, env }) + console.log(`[terminal-node] Building patched Node ${inputs.version} (${inputs.arch})`) + await runBuildCommand('make', ['-j4', 'node'], { cwd: source, env }) + const executable = join(source, 'out/Release/node') + const architectures = lipoArchsToNodeArchs(await runBuildCommand('lipo', ['-archs', executable])) + const minimumOs = parseBuildVersionMinos(await runBuildCommand('otool', ['-l', executable])) + if ( + architectures.length !== 1 || + architectures[0] !== inputs.arch || + minimumOs !== inputs.minimumOs + ) { + throw new Error('Patched terminal Node has the wrong architecture or macOS floor') + } + const staged = join(directory, `node.${process.pid}.partial`) + copyFileSync(executable, staged) + chmodSync(staged, 0o755) + const executableSha256 = await sha256File(staged) + renameSync(staged, join(directory, 'node')) + const receipt = join(directory, `receipt.${process.pid}.partial`) + writeFileSync(receipt, `${JSON.stringify({ inputs, executableSha256 }, null, 2)}\n`) + renameSync(receipt, join(directory, 'receipt.json')) + return join(directory, 'node') +} + +export function ensureDarwinTerminalNode({ arch, cacheDir = pinnedNodeCacheDir() }) { + if (process.platform !== 'darwin') { + throw new Error('Build the patched macOS terminal runtime on a macOS host') + } + const inputs = darwinTerminalNodeInputs(arch) + const key = createHash('sha256').update(JSON.stringify(inputs)).digest('hex') + const directory = join(cacheDir, 'darwin-terminal-node', key) + const pending = builds.get(directory) + if (pending) { + return pending + } + const build = (async () => { + mkdirSync(directory, { recursive: true }) + const cached = await readBuiltDarwinTerminalNode(directory, inputs) + if (cached) { + return cached + } + const release = await lock(directory, { + realpath: false, + stale: 60_000, + retries: { retries: 2160, minTimeout: 5_000, maxTimeout: 5_000 } + }) + let scratch + try { + const cached = await readBuiltDarwinTerminalNode(directory, inputs) + if (cached) { + return cached + } + scratch = mkdtempSync(join(tmpdir(), 'orca-darwin-terminal-node-build-')) + return await compileDarwinTerminalNode(directory, inputs, cacheDir, scratch) + } finally { + try { + // Release before synchronous cleanup can starve the lease heartbeat. + await release() + } finally { + if (scratch) { + rmSync(scratch, { recursive: true, force: true }) + } + } + } + })() + builds.set(directory, build) + void build.finally(() => builds.delete(directory)).catch(() => {}) + return build +} + +if (isDirectInvocation(import.meta.url, process.argv[1])) { + console.log(await ensureDarwinTerminalNode({ arch: process.argv[2] ?? process.arch })) +} diff --git a/config/scripts/build-darwin-terminal-node.test.mjs b/config/scripts/build-darwin-terminal-node.test.mjs new file mode 100644 index 00000000000..da72c569c2a --- /dev/null +++ b/config/scripts/build-darwin-terminal-node.test.mjs @@ -0,0 +1,92 @@ +import { createHash } from 'node:crypto' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, it } from 'vitest' +import { + darwinTerminalNodeInputs, + readBuiltDarwinTerminalNode +} from './build-darwin-terminal-node.mjs' + +const directories = [] +afterEach(() => { + for (const directory of directories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +function artifact() { + const directory = mkdtempSync(join(tmpdir(), 'darwin-terminal-node-cache-')) + directories.push(directory) + const inputs = darwinTerminalNodeInputs('arm64') + const bytes = Buffer.from('built Node artifact') + const executable = join(directory, 'node') + writeFileSync(executable, bytes) + writeFileSync( + join(directory, 'receipt.json'), + JSON.stringify({ + inputs, + executableSha256: createHash('sha256').update(bytes).digest('hex') + }) + ) + return { directory, inputs, executable } +} + +it('requires review when the Node pin moves instead of silently dropping the fix', () => { + expect(() => darwinTerminalNodeInputs('arm64', '99.0.0')).toThrow('Review the Darwin') + expect(() => darwinTerminalNodeInputs('riscv64')).toThrow('Unsupported') +}) + +it('separates architecture and includes the exact patch and source identity', () => { + const arm = darwinTerminalNodeInputs('arm64') + const x64 = darwinTerminalNodeInputs('x64') + expect(arm.arch).not.toBe(x64.arch) + expect(arm.sourceSha256).toMatch(/^[0-9a-f]{64}$/) + expect(arm.workerSourceSha256).toMatch(/^[0-9a-f]{64}$/) + expect(arm.patchSha256).toBe( + createHash('sha256') + .update( + readFileSync(join(import.meta.dirname, '../node-runtime-patches/darwin-worker-wait.patch')) + ) + .digest('hex') + ) +}) + +it('reuses a built artifact only when its bytes and every build input match', async () => { + const { directory, inputs, executable } = artifact() + expect(await readBuiltDarwinTerminalNode(directory, inputs)).toBe(executable) + expect(await readBuiltDarwinTerminalNode(directory, { ...inputs, arch: 'x64' })).toBeNull() + expect( + await readBuiltDarwinTerminalNode(directory, { ...inputs, patchSha256: 'other' }) + ).toBeNull() +}) + +it('rejects a tampered cached executable', async () => { + const { directory, inputs, executable } = artifact() + writeFileSync(executable, 'different bytes') + expect(await readBuiltDarwinTerminalNode(directory, inputs)).toBeNull() +}) + +it('rejects unchanged executable bytes built with an older build script', async () => { + const { directory, inputs, executable } = artifact() + const receiptPath = join(directory, 'receipt.json') + const receipt = JSON.parse(readFileSync(receiptPath, 'utf8')) + expect(inputs.buildScriptsSha256['build-darwin-terminal-node.mjs']).toMatch(/^[0-9a-f]{64}$/) + expect(await readBuiltDarwinTerminalNode(directory, inputs)).toBe(executable) + receipt.inputs.buildScriptsSha256['build-darwin-terminal-node.mjs'] = 'older build recipe' + writeFileSync(receiptPath, JSON.stringify(receipt)) + expect(await readBuiltDarwinTerminalNode(directory, inputs)).toBeNull() + delete receipt.inputs.buildScriptsSha256 + writeFileSync(receiptPath, JSON.stringify(receipt)) + expect(await readBuiltDarwinTerminalNode(directory, inputs)).toBeNull() +}) + +it('rejects a partial build, invalid receipt, or non-file executable', async () => { + const { directory, inputs, executable } = artifact() + rmSync(executable) + expect(await readBuiltDarwinTerminalNode(directory, inputs)).toBeNull() + mkdirSync(executable) + expect(await readBuiltDarwinTerminalNode(directory, inputs)).toBeNull() + writeFileSync(join(directory, 'receipt.json'), '{') + expect(await readBuiltDarwinTerminalNode(directory, inputs)).toBeNull() +}) diff --git a/config/scripts/darwin-terminal-node-probe.cjs b/config/scripts/darwin-terminal-node-probe.cjs new file mode 100644 index 00000000000..ec3ad408b94 --- /dev/null +++ b/config/scripts/darwin-terminal-node-probe.cjs @@ -0,0 +1,80 @@ +const assert = require('node:assert/strict') +const { setTimeout: delay } = require('node:timers/promises') +const fs = require('node:fs/promises') +const crypto = require('node:crypto') +const dns = require('node:dns/promises') +const pty = require(process.argv[3]) +const canary = require(process.argv[2]) + +const deadline = setTimeout(() => process.exit(2), 45_000) +async function checkIo() { + await Promise.all([ + fs.stat(process.execPath), + dns.lookup('localhost'), + new Promise((resolve, reject) => { + crypto.randomBytes(32, (error) => (error ? reject(error) : resolve())) + }) + ]) +} + +async function checkPty() { + const terminal = pty.spawn('/bin/sh', ['-c', 'read line; printf "ECHO:%s\\n" "$line"'], { + cwd: process.argv[4], + env: process.env + }) + let output = '' + terminal.onData((chunk) => { + output += chunk + }) + const exited = new Promise((resolve, reject) => { + terminal.onExit(({ exitCode }) => + exitCode === 0 ? resolve() : reject(new Error('PTY failed')) + ) + }) + terminal.write('ORCA_WORKER_WAKEUP_TEST\r') + await exited + assert.ok(output.includes('ECHO:ORCA_WORKER_WAKEUP_TEST')) +} + +async function main() { + assert.equal(process.version, process.argv[5]) + await checkIo() + await checkPty() + canary.enroll() + let before + for (let i = 0; i < 100; i++) { + await delay(10) + before = JSON.parse(canary.snapshot()) + if (before.callbacks === 4 && before.waits.every((wait) => wait.parked)) { + break + } + } + assert.equal(before.callbacks, 4) + assert.ok(before.waits.every((wait) => wait.parked)) + await delay(31_000) + const after = JSON.parse(canary.snapshot()) + assert.ok(after.waits.every((wait) => wait.parked)) + for (let i = 0; i < 4; i++) { + const previous = BigInt(`0x${before.waits[i].generation}`) + const current = BigInt(`0x${after.waits[i].generation}`) + const advances = BigInt.asUintN(32, current - previous) + assert.ok(advances > 0n) + assert.ok(advances <= 16n * 256n) + } + await checkIo() + clearTimeout(deadline) + console.log( + JSON.stringify({ + node: process.version, + uv: process.versions.uv, + arch: process.arch, + fsDnsCrypto: 'passed', + ptyInput: 'passed', + idleWaitRefresh: 'passed' + }) + ) +} +main().catch((error) => { + console.error(error) + process.exit(1) +}) diff --git a/config/scripts/darwin-terminal-node-workflow.test.mjs b/config/scripts/darwin-terminal-node-workflow.test.mjs new file mode 100644 index 00000000000..0d51d3c6eb1 --- /dev/null +++ b/config/scripts/darwin-terminal-node-workflow.test.mjs @@ -0,0 +1,147 @@ +import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { runProcess } from '@orca/process-host' +import { expect, it } from 'vitest' +import { parse } from 'yaml' +import { readDarwinVerificationArch } from './verify-darwin-terminal-node.mjs' +import { classifyPrJobs } from './pr-code-change-scope.mjs' + +function workflow(name) { + return parse( + readFileSync(join(import.meta.dirname, '../../.github/workflows', `${name}.yml`), 'utf8') + ) +} + +it.each([ + 'config/node-runtime-patches/darwin-worker-wait.patch', + 'config/scripts/build-darwin-terminal-node.mjs', + 'config/scripts/pr-darwin-terminal-node-route.mjs', + 'config/scripts/verify-darwin-terminal-node.mjs', + 'config/scripts/darwin-terminal-node-probe.cjs', + 'config/scripts/__fixtures__/darwin-threadpool-wait.h', + 'config/scripts/macos-terminal-host-bundle.cjs', + 'config/scripts/pinned-node-downloads.mjs', + 'config/scripts/update-node-runtime-pin.mjs', + 'config/scripts/orcad-prebuild-node-gyp.mjs', + 'config/scripts/orcad-prebuild-slot-contents.mjs', + 'src/shared/node-runtime-pin.ts' +])('qualifies the actual macOS runtime when %s changes', (file) => { + expect(classifyPrJobs([file]).darwin_terminal_node).toBe(true) +}) + +it('does not source-build Node for unrelated renderer changes', () => { + expect(classifyPrJobs(['src/renderer/src/App.tsx']).darwin_terminal_node).toBe(false) +}) + +it.each(['adhoc', 'daily', 'hourly', 'release'])( + 'builds both %s runtime architectures after installing build dependencies', + (channel) => { + const jobs = Object.values(workflow(`${channel}-mac-build`).jobs) + const steps = jobs.find((job) => + job.steps?.some((step) => step.name === 'Build and verify the patched terminal runtimes') + ).steps + const build = steps.findIndex( + (step) => step.name === 'Build and verify the patched terminal runtimes' + ) + expect(build).toBeGreaterThan(steps.findIndex((step) => step.name === 'Install dependencies')) + expect(steps[build].run).toContain('pnpm run build:packages') + expect(steps[build].run).toContain('verify-darwin-terminal-node.mjs --arch=arm64') + expect(steps[build].run).toContain('verify-darwin-terminal-node.mjs --arch=x64') + const packaging = steps.findIndex((step) => step.with?.command?.includes('--runtime=electron')) + expect(build).toBeLessThan(packaging) + const cache = steps.find((step) => step.with?.key?.startsWith('orca-node-runtime-mac-')) + for (const dependency of [ + 'pinned-node-downloads.mjs', + 'update-node-runtime-pin.mjs', + 'macos-terminal-host-bundle.cjs' + ]) { + expect(cache.with.key).toContain(dependency) + } + if (channel === 'daily') { + expect(steps[build].if).toBe("steps.freshness.outputs.should_build == 'true'") + } + if (channel === 'adhoc' || channel === 'release') { + // Older refs and tags have no source-build recipe and must keep the pinned download. + expect(steps[build].if).toBe( + "hashFiles('config/scripts/build-darwin-terminal-node.mjs') != ''" + ) + expect(steps[packaging].with.command).toMatch( + /^if \[ ! -f config\/scripts\/build-darwin-terminal-node\.mjs \]; then node .*pinned-node-downloads\.mjs darwin-arm64 darwin-x64; fi && / + ) + } + } +) + +it.skipIf(process.platform === 'win32').each([ + ['success', 'true', 0], + ['failure', 'true', 1], + ['cancelled', 'true', 1], + ['skipped', 'true', 1], + ['', 'true', 1], + ['skipped', 'false', 0] +])('aggregate handles runtime result %s (required=%s)', async (result, required, code) => { + const gate = workflow('pr').jobs.verify.steps.find( + (step) => step.name === 'Require successful checks' + ) + const env = Object.fromEntries( + Object.keys(gate.env).map((key) => [key, key.endsWith('_SHOULD_RUN') ? 'true' : 'success']) + ) + const checked = await runProcess({ + program: 'bash', + args: ['-c', gate.run], + env: { + ...process.env, + ...env, + SHOULD_RUN: 'true', + DARWIN_TERMINAL_NODE: result, + DARWIN_TERMINAL_NODE_SHOULD_RUN: required + }, + timeoutMs: 10_000 + }) + expect(checked.code).toBe(code) +}) + +it('classifies runtime changes from the dependency-free CI sparse checkout', async () => { + const root = join(import.meta.dirname, '../..') + const directory = mkdtempSync(join(tmpdir(), 'orca-runtime-sparse-checkout-')) + try { + const checkout = workflow('pr').jobs.code_paths.steps.find( + (step) => step.with?.['sparse-checkout'] + ) + for (const entry of checkout.with['sparse-checkout'].trim().split('\n')) { + const path = entry.trim().replace(/^\//, '') + const source = join(root, path) + if (statSync(source).isFile()) { + const destination = join(directory, path) + mkdirSync(dirname(destination), { recursive: true }) + copyFileSync(source, destination) + } + } + const result = await runProcess({ + program: process.execPath, + args: ['config/scripts/pr-code-change-scope.mjs'], + cwd: directory, + input: 'config/node-runtime-patches/darwin-worker-wait.patch\n', + timeoutMs: 10_000 + }) + expect(result.code, result.stderr).toBe(0) + expect(result.stdout).toContain('darwin_terminal_node=true') + expect(result.stdout).toContain('package_windows=false') + } finally { + rmSync(directory, { recursive: true, force: true }) + } +}) + +it('selects the requested runtime architecture and retains the native default', () => { + expect(readDarwinVerificationArch([], 'arm64')).toBe('arm64') + expect(readDarwinVerificationArch(['--arch=x64'], 'arm64')).toBe('x64') + expect(readDarwinVerificationArch(['--arch=arm64'], 'x64')).toBe('arm64') +}) + +it.each([['--arch=riscv64'], ['--arch'], ['--unknown'], ['--arch=x64', '--arch=arm64']])( + 'rejects invalid verifier arguments %j before building a runtime', + (...args) => { + expect(() => readDarwinVerificationArch(args, 'arm64')).toThrow('Usage:') + } +) diff --git a/config/scripts/darwin-terminal-worker-stress.cjs b/config/scripts/darwin-terminal-worker-stress.cjs new file mode 100644 index 00000000000..e7a335d5c57 --- /dev/null +++ b/config/scripts/darwin-terminal-worker-stress.cjs @@ -0,0 +1,82 @@ +const { setTimeout: delay } = require('node:timers/promises') +const fs = require('node:fs/promises') +const pty = require(process.argv[3]) +const canary = require(process.argv[2]) + +const target = 30_000_000 +const began = Date.now() +const terminal = pty.spawn( + '/bin/sh', + ['-c', 'while IFS= read -r line; do printf "ECHO:%s\\n" "$line"; done'], + { cwd: process.argv[4], env: process.env } +) +let output = '' +terminal.onData((chunk) => { + output = (output + chunk).slice(-4096) +}) +const deadline = setTimeout(() => { + console.error('Stress run exceeded one hour') + terminal.kill() + process.kill(process.pid, 'SIGKILL') +}, 60 * 60_000) + +async function echo(marker, timeoutMs = 10_000) { + terminal.write(`${marker}\r`) + const echoDeadline = Date.now() + timeoutMs + while (Date.now() < echoDeadline) { + if (output.includes(`ECHO:${marker}`)) { + return + } + await delay(10) + } + throw new Error(`No PTY echo for ${marker}`) +} + +async function main() { + await echo('ORCA_BEFORE') + canary.startStress() + let previous = 0 + let lastProgress = Date.now() + let lastPrint = 0 + while (canary.stressProgress() < target) { + await delay(100) + const completed = canary.stressProgress() + if (completed !== previous) { + previous = completed + lastProgress = Date.now() + } + if (Date.now() - lastPrint > 10_000) { + console.log(JSON.stringify({ elapsedMs: Date.now() - began, completed })) + lastPrint = Date.now() + } + if (Date.now() - lastProgress > 3000) { + const marker = `ORCA_AFTER_STALL_${completed}` + await Promise.race([ + Promise.all([ + ...Array.from({ length: 4 }, () => fs.stat(process.execPath)), + echo(marker, 40_000), + (async () => { + while (canary.stressProgress() <= completed) { + await delay(100) + } + })() + ]), + delay(40_000, undefined, { ref: false }).then(() => { + throw new Error(`Workers stranded after ${completed} jobs: I/O recovery timed out`) + }) + ]) + lastProgress = Date.now() + } + } + await fs.stat(process.execPath) + await echo('ORCA_AFTER') + console.log(JSON.stringify({ result: 'passed', arch: process.arch, completed: target })) + clearTimeout(deadline) + terminal.kill() +} +main().catch((error) => { + console.error(error) + terminal.kill() + // Stalled libuv cleanup can block process.exit(). + process.kill(process.pid, 'SIGKILL') +}) diff --git a/config/scripts/macos-terminal-host-bundle.cjs b/config/scripts/macos-terminal-host-bundle.cjs index 7339c7e3e52..64a1902b34e 100644 --- a/config/scripts/macos-terminal-host-bundle.cjs +++ b/config/scripts/macos-terminal-host-bundle.cjs @@ -294,10 +294,10 @@ async function loadNodeRuntimePin() { } async function pinnedDarwinNode(arch) { - const { ensurePinnedNodeExecutable } = await import( - pathToFileURL(resolve(__dirname, 'pinned-node-downloads.mjs')).href + const { ensureDarwinTerminalNode } = await import( + pathToFileURL(resolve(__dirname, 'build-darwin-terminal-node.mjs')).href ) - return ensurePinnedNodeExecutable({ target: `darwin-${arch}` }) + return ensureDarwinTerminalNode({ arch }) } /** diff --git a/config/scripts/mobile-release-check-scope.test.mjs b/config/scripts/mobile-release-check-scope.test.mjs index 7195be14d45..6940f17e91e 100644 --- a/config/scripts/mobile-release-check-scope.test.mjs +++ b/config/scripts/mobile-release-check-scope.test.mjs @@ -145,6 +145,7 @@ function fixture(detectorStep = detector) { for (const file of [ 'package.json', 'config/scripts/pr-code-change-scope.mjs', + 'config/scripts/pr-darwin-terminal-node-route.mjs', 'config/scripts/pr-package-test-paths.mjs', 'config/scripts/pr-layout-oracle-route.mjs', 'config/scripts/mobile-release-check-scope.mjs', diff --git a/config/scripts/package-electron-runtime-contract.test.mjs b/config/scripts/package-electron-runtime-contract.test.mjs index 5921f7f6727..728d14d9a21 100644 --- a/config/scripts/package-electron-runtime-contract.test.mjs +++ b/config/scripts/package-electron-runtime-contract.test.mjs @@ -141,7 +141,7 @@ describe('Electron runtime package contract', () => { expect(releaseWorkflow.jobs['publish-release'].needs).toContain('build-mac') expect(releaseWorkflow.jobs['publish-release'].needs).toContain('verify-mac-build') const macVerification = releaseWorkflow.jobs['verify-mac-build'] - expect(macVerification.needs).toEqual(['cut', 'build-mac', 'build']) + expect(macVerification.needs).toEqual(['cut', 'build-mac']) expect(macVerification.if).toContain('!cancelled()') expect(macVerification.if).not.toContain('needs.build.result') expect(macVerification.permissions).toEqual({ actions: 'read', contents: 'read' }) diff --git a/config/scripts/pr-auxiliary-preflight-admission.test.mjs b/config/scripts/pr-auxiliary-preflight-admission.test.mjs index a0abbcee847..cd573eacaf8 100644 --- a/config/scripts/pr-auxiliary-preflight-admission.test.mjs +++ b/config/scripts/pr-auxiliary-preflight-admission.test.mjs @@ -383,6 +383,7 @@ function createMobileCiScopeRepository( for (const file of [ 'package.json', 'config/scripts/pr-code-change-scope.mjs', + 'config/scripts/pr-darwin-terminal-node-route.mjs', 'config/scripts/pr-package-test-paths.mjs', 'config/scripts/pr-layout-oracle-route.mjs', 'config/scripts/git-pull-request-diff-base.mjs', diff --git a/config/scripts/pr-code-change-scope.mjs b/config/scripts/pr-code-change-scope.mjs index 8550dcc6bcc..59f37ef6241 100644 --- a/config/scripts/pr-code-change-scope.mjs +++ b/config/scripts/pr-code-change-scope.mjs @@ -1,6 +1,7 @@ import { readFileSync } from 'node:fs' import { join } from 'node:path' import { pathToFileURL } from 'node:url' +import { isDarwinTerminalNodeSource } from './pr-darwin-terminal-node-route.mjs' import { isLayoutOracleSource } from './pr-layout-oracle-route.mjs' import { isCloudTestRunnerInput, @@ -37,6 +38,7 @@ export const PR_CHECK_JOBS = [ 'managed_hook_node18', 'package', 'package_windows', + 'darwin_terminal_node', 'layout_oracle', 'dependency_isolation' ] @@ -557,10 +559,26 @@ export function classifyPrJobs( } } +const FILE_ROUTED_JOBS = new Map([ + ['git_compatibility', (file) => matchesPrefix(file, GIT_COMPAT_PREFIXES)], + ['xterm_patch_sync', (file) => matchesPrefix(file, XTERM_PREFIXES)], + ['shell_contracts', (file) => matchesPrefix(file, SHELL_PREFIXES)], + ['orcad_browser', (file) => matchesPrefix(file, ORCAD_BROWSER_PREFIXES)], + ['cross-version-wire', (file) => matchesPrefix(file, CROSS_VERSION_WIRE_PREFIXES)], + ['managed_hook_node18', (file) => matchesPrefix(file, MANAGED_HOOK_PREFIXES)], + ['package', isLinuxPackagePath], + ['package_windows', isWindowsPackagePath], + ['layout_oracle', isLayoutOracleSource], + ['dependency_isolation', (file) => matchesPrefix(file, DEPENDENCY_ISOLATION_PREFIXES)], + ['darwin_terminal_node', isDarwinTerminalNodeSource] +]) + function jobDetector(job) { + const routesFile = FILE_ROUTED_JOBS.get(job) + if (routesFile) { + return (files) => files.some(routesFile) + } switch (job) { - case 'git_compatibility': - return (files) => files.some((file) => matchesPrefix(file, GIT_COMPAT_PREFIXES)) case 'codex_index_heal_contract': return (files) => files.some( @@ -568,27 +586,9 @@ function jobDetector(job) { !/^src\/main\/codex\/codex-app-server-(client|session)\.test\.ts(?!.)/s.test(file) && matchesPrefix(file, CODEX_INDEX_HEAL_CONTRACT_PREFIXES) ) - case 'xterm_patch_sync': - return (files) => files.some((file) => matchesPrefix(file, XTERM_PREFIXES)) - case 'shell_contracts': - return (files) => files.some((file) => matchesPrefix(file, SHELL_PREFIXES)) - case 'orcad_browser': - return (files) => files.some((file) => matchesPrefix(file, ORCAD_BROWSER_PREFIXES)) // Handle inside map too: default detector otherwise runs this job on every desktop-relevant PR. case 'mobile_web_app': return changesMobileWebApp - case 'cross-version-wire': - return (files) => files.some((file) => matchesPrefix(file, CROSS_VERSION_WIRE_PREFIXES)) - case 'managed_hook_node18': - return (files) => files.some((file) => matchesPrefix(file, MANAGED_HOOK_PREFIXES)) - case 'package': - return (files) => files.some(isLinuxPackagePath) - case 'package_windows': - return (files) => files.some(isWindowsPackagePath) - case 'layout_oracle': - return (files) => files.some(isLayoutOracleSource) - case 'dependency_isolation': - return (files) => files.some((file) => matchesPrefix(file, DEPENDENCY_ISOLATION_PREFIXES)) default: return () => true } diff --git a/config/scripts/pr-darwin-terminal-node-route.mjs b/config/scripts/pr-darwin-terminal-node-route.mjs new file mode 100644 index 00000000000..c290648e9b1 --- /dev/null +++ b/config/scripts/pr-darwin-terminal-node-route.mjs @@ -0,0 +1,20 @@ +const DARWIN_TERMINAL_NODE_PREFIXES = [ + 'config/node-runtime-patches/', + 'config/scripts/build-darwin-terminal-node', + 'config/scripts/verify-darwin-terminal-node', + 'config/scripts/darwin-terminal-node-', + 'config/scripts/darwin-terminal-worker-stress', + 'config/scripts/pr-darwin-terminal-node-route', + 'config/scripts/__fixtures__/darwin-terminal-worker-wait', + 'config/scripts/__fixtures__/darwin-threadpool-wait', + 'config/scripts/macos-terminal-host-bundle', + 'config/scripts/pinned-node-downloads', + 'config/scripts/update-node-runtime-pin', + 'config/scripts/orcad-prebuild-node-gyp', + 'config/scripts/orcad-prebuild-slot-contents', + 'src/shared/node-runtime-pin.ts' +] + +export function isDarwinTerminalNodeSource(file) { + return DARWIN_TERMINAL_NODE_PREFIXES.some((prefix) => file.startsWith(prefix)) +} diff --git a/config/scripts/release-mac-build-workflow-dispatch.test.mjs b/config/scripts/release-mac-build-workflow-dispatch.test.mjs index f6410b846be..e4d907e266a 100644 --- a/config/scripts/release-mac-build-workflow-dispatch.test.mjs +++ b/config/scripts/release-mac-build-workflow-dispatch.test.mjs @@ -164,7 +164,7 @@ describe('release mac build workflow dispatch', () => { expect(options.releaseRunId).toBe('987') expect(options.workflow).toBe('release-mac-build.yml') expect(options.pollSeconds).toBe(30) - expect(options.timeoutMinutes).toBe(90) + expect(options.timeoutMinutes).toBe(390) }) }) @@ -195,6 +195,84 @@ const splitRun = { } describe('release mac build split dispatch and verification', () => { + it('hands a pending exact run to verification after observing 180 minutes without renewing its deadline', async () => { + const startedAt = Date.parse('2026-06-30T12:00:00Z') + let currentTime = startedAt + const live = { ...splitRun, status: 'in_progress', conclusion: null } + const { fetch } = createGitHubFetch([ + jsonResponse(200, { workflow_run_id: 123 }), + jsonResponse(200, live), + jsonResponse(200, live), + jsonResponse(200, live) + ]) + const appendOutput = vi.fn() + const result = await executeReleaseMacBuildMode( + 'dispatch', + { ...splitEnvironment, RELEASE_MAC_BUILD_OBSERVE_MINUTES: '180' }, + { + fetch, + appendOutput, + now: () => currentTime, + sleep: async () => { + currentTime += 180 * 60_000 + } + } + ) + expect(result).toMatchObject({ id: 123, status: 'in_progress', conclusion: null }) + expect(appendOutput).toHaveBeenCalledOnce() + expect(appendOutput.mock.calls[0][1]).toContain( + `wait_deadline_ms=${startedAt + 390 * 60_000}\n` + ) + const { fetch: finalFetch } = createGitHubFetch([ + jsonResponse(200, { ...splitRun, updated_at: '2026-06-30T18:00:00Z' }) + ]) + await expect( + executeReleaseMacBuildMode( + 'wait', + { + ...splitEnvironment, + RELEASE_MAC_BUILD_WAIT_DEADLINE_MS: String(startedAt + 390 * 60_000) + }, + { fetch: finalFetch, now: () => startedAt + 360 * 60_000 } + ) + ).resolves.toMatchObject({ + status: 'completed', + conclusion: 'success' + }) + const { fetch: expiredFetch } = createGitHubFetch([jsonResponse(200, live)]) + await expect( + executeReleaseMacBuildMode( + 'wait', + { + ...splitEnvironment, + RELEASE_MAC_BUILD_WAIT_DEADLINE_MS: String(startedAt + 390 * 60_000) + }, + { fetch: expiredFetch, now: () => startedAt + 391 * 60_000 } + ) + ).rejects.toThrow(/Timed out/) + }) + + it.each([ + { ...splitRun, conclusion: 'failure' }, + { ...splitRun, head_sha: 'b'.repeat(40) } + ])('does not hand off failure or changed identity as pending: %j', async (observed) => { + const { fetch } = createGitHubFetch([ + jsonResponse(200, { workflow_run_id: 123 }), + jsonResponse(200, splitRun), + jsonResponse(200, observed) + ]) + await expect( + executeReleaseMacBuildMode( + 'dispatch', + { + ...splitEnvironment, + RELEASE_MAC_BUILD_OBSERVE_MINUTES: '180' + }, + { fetch, appendOutput: vi.fn(), now: () => Date.parse('2026-06-30T12:00:00Z') } + ) + ).rejects.toThrow() + }) + it('waits for the exact dispatched run metadata to become consistent', async () => { const { fetch, requests } = createGitHubFetch([ jsonResponse(200, { workflow_run_id: 123 }), @@ -249,7 +327,7 @@ describe('release mac build split dispatch and verification', () => { expect(requests[0].headers['X-GitHub-Api-Version']).toBe('2026-03-10') expect(appendOutput).toHaveBeenCalledWith( splitEnvironment.GITHUB_OUTPUT, - `run_id=123\nworkflow_sha=${splitRun.head_sha}\nrun_attempt=1\nwait_deadline_ms=${splitEnvironment.RELEASE_MAC_BUILD_WAIT_DEADLINE_MS}\n` + `run_id=123\nworkflow_sha=${splitRun.head_sha}\nrun_attempt=1\nwait_deadline_ms=${Date.parse('2026-06-30T18:30:00Z')}\n` ) }) diff --git a/config/scripts/release-windows-signing-gate.test.mjs b/config/scripts/release-windows-signing-gate.test.mjs index 5a6d022d44a..20340b16742 100644 --- a/config/scripts/release-windows-signing-gate.test.mjs +++ b/config/scripts/release-windows-signing-gate.test.mjs @@ -5,6 +5,7 @@ import { runInNewContext } from 'node:vm' import { runProcess } from '@orca/process-host' import { describe, expect, it } from 'vitest' import { parse } from 'yaml' +import { readReleaseMacBuildWorkflowOptions } from './run-release-mac-build-workflow.mjs' const { jobs } = parse(readFileSync('.github/workflows/release-cut.yml', 'utf8')) const { steps } = jobs.build @@ -35,12 +36,39 @@ function signingAllowed(submission, outcome, attempt = 1, platform = 'win', uplo } describe('Windows release signing waits for macOS success', () => { - it('packages concurrently, then verifies the exact dispatched macOS run before signing', () => { + it('covers the cold runtime and packaging budget in two hosted observation jobs', () => { + const child = parse(readFileSync('.github/workflows/release-mac-build.yml', 'utf8')).jobs[ + 'build-mac' + ] + const dispatch = jobs['build-mac'].steps.find((step) => step.id === 'dispatch') + const options = readReleaseMacBuildWorkflowOptions({ + GITHUB_REPOSITORY: 'stablyai/orca', + GITHUB_RUN_ID: '1', + GITHUB_TOKEN: 'test', + RELEASE_MAC_BUILD_REF: 'main', + RELEASE_MAC_BUILD_TAG: 'v1.2.3', + ...dispatch.env + }) + const observationMinutes = Number(dispatch.env.RELEASE_MAC_BUILD_OBSERVE_MINUTES) + expect(child['timeout-minutes']).toBe(360) + expect(options.timeoutMinutes).toBeGreaterThanOrEqual(child['timeout-minutes'] + 30) + expect(observationMinutes).toBe(180) + expect(jobs['build-mac']['timeout-minutes']).toBeGreaterThan(observationMinutes) + expect(jobs['verify-mac-build']['timeout-minutes']).toBeGreaterThan( + options.timeoutMinutes - observationMinutes + ) + for (const job of [jobs['build-mac'], jobs['verify-mac-build'], jobs.build]) { + expect(job['timeout-minutes']).toBeLessThanOrEqual(360) + } + }) + + it('waits outside the Windows runner, then verifies the exact macOS run before signing', () => { expect(jobs.build.needs).toContain('build-mac') expect(jobs.build.if).toContain("needs.build-mac.result == 'success'") - expect(jobs.build.needs).not.toContain('verify-mac-build') + expect(jobs.build.needs).toContain('verify-mac-build') + expect(jobs.build.if).toContain("needs.verify-mac-build.result == 'success'") expect(jobs['build-mac'].needs).not.toContain('build') - expect(jobs['verify-mac-build'].needs).toContain('build') + expect(jobs['verify-mac-build'].needs).not.toContain('build') const windows = jobs.build.strategy.matrix.include.find((entry) => entry.platform === 'win') expect(windows.release_command).toMatch( /^node config\/scripts\/ensure-native-runtime\.mjs --runtime=electron; if \(\$LASTEXITCODE -ne 0\) \{ exit \$LASTEXITCODE \}; pnpm exec electron-builder / diff --git a/config/scripts/run-release-mac-build-workflow.mjs b/config/scripts/run-release-mac-build-workflow.mjs index 5123b81c327..92cea65c06d 100644 --- a/config/scripts/run-release-mac-build-workflow.mjs +++ b/config/scripts/run-release-mac-build-workflow.mjs @@ -2,7 +2,7 @@ import { appendFileSync } from 'node:fs' import { pathToFileURL } from 'node:url' const DEFAULT_API_VERSION = '2026-03-10' -const DEFAULT_TIMEOUT_MINUTES = 90 +const DEFAULT_TIMEOUT_MINUTES = 390 const DEFAULT_POLL_SECONDS = 30 const RUN_DISCOVERY_TIMEOUT_SECONDS = 120 @@ -100,6 +100,11 @@ export async function finishReleaseMacBuildWorkflow(options, workflowRunId, deps const api = createGitHubApiClient(options, deps) const completedRun = await waitForReleaseMacBuildRun(api, workflowRunId, options, deps) + if (completedRun.status !== 'completed' && deps.observationDeadlineMs != null) { + console.log(`Mac release build workflow remains pending: ${completedRun.html_url}`) + return completedRun + } + if (completedRun.conclusion !== 'success') { throw new Error( `Mac release build workflow ${completedRun.html_url ?? completedRun.id} concluded ${ @@ -190,9 +195,13 @@ export async function waitForReleaseMacBuildRun(api, workflowRunId, options, dep } return run } - if (now() > deadlineMs) { + const observedAtMs = now() + if (observedAtMs > deadlineMs) { break } + if (deps.observationDeadlineMs != null && observedAtMs >= deps.observationDeadlineMs) { + return run + } console.log( `Mac release build workflow is ${run.status}; polling again in ${options.pollSeconds}s` @@ -310,6 +319,15 @@ export async function executeReleaseMacBuildMode(mode, env = process.env, deps = output, `run_id=${run.id}\nworkflow_sha=${run.head_sha}\nrun_attempt=${run.run_attempt}\nwait_deadline_ms=${deadlineMs}\n` ) + if (env.RELEASE_MAC_BUILD_OBSERVE_MINUTES) { + const observationMinutes = readPositiveInteger(env.RELEASE_MAC_BUILD_OBSERVE_MINUTES, 180) + return await finishReleaseMacBuildWorkflow(options, run.id, { + ...deps, + deadlineMs, + observationDeadlineMs: now() + observationMinutes * 60 * 1000, + expectedIdentity: { id: run.id, headSha: run.head_sha, attempt: run.run_attempt } + }) + } return run } if (mode === 'wait') { diff --git a/config/scripts/verify-darwin-terminal-node.mjs b/config/scripts/verify-darwin-terminal-node.mjs new file mode 100644 index 00000000000..ed865305eeb --- /dev/null +++ b/config/scripts/verify-darwin-terminal-node.mjs @@ -0,0 +1,118 @@ +import { cpSync, mkdtempSync, rmSync } from 'node:fs' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { runProcess, runProcessSync } from '@orca/process-host' +import { ensureDarwinTerminalNode } from './build-darwin-terminal-node.mjs' +import { preparePinnedNodeDir } from './pinned-node-downloads.mjs' +import { nodeGypRebuild, stageNodeAddonApi } from './orcad-prebuild-node-gyp.mjs' +import { isDirectInvocation } from './script-entry-detection.mjs' +import { NODE_RUNTIME_PIN } from '../../src/shared/node-runtime-pin.ts' + +export function readDarwinVerificationArch(args, defaultArch = process.arch) { + if (args.length > 1 || (args.length === 1 && !/^--arch=(arm64|x64)$/.test(args[0]))) { + throw new Error('Usage: verify-darwin-terminal-node.mjs [--arch=arm64|x64]') + } + const arch = args.length ? args[0].slice('--arch='.length) : defaultArch + if (arch !== 'arm64' && arch !== 'x64') { + throw new Error(`Unsupported macOS terminal Node architecture: ${arch}`) + } + return arch +} + +async function verifyDarwinTerminalNode(arch) { + if (process.platform !== 'darwin') { + throw new Error('Verify the Darwin terminal runtime on a macOS host') + } + const executable = await ensureDarwinTerminalNode({ arch }) + const scratch = mkdtempSync(join(tmpdir(), 'orca-darwin-terminal-node-proof-')) + const require = createRequire(import.meta.url) + try { + const headers = await preparePinnedNodeDir({ + target: `darwin-${arch}`, + workDir: join(scratch, 'headers') + }) + const sourcePty = dirname(require.resolve('node-pty/package.json')) + const stagedPty = join(scratch, 'node-pty') + for (const entry of ['package.json', 'binding.gyp', 'src', 'lib']) { + cpSync(join(sourcePty, entry), join(stagedPty, entry), { recursive: true }) + } + stageNodeAddonApi(sourcePty, stagedPty) + await nodeGypRebuild({ + stagedDir: stagedPty, + workDir: scratch, + nodeDir: headers, + runProcess: (options) => + runProcessSync({ + ...options, + program: executable, + timeoutMs: 120_000, + env: { ...options.env, ORCA_BACKGROUND_LAUNCH: '1', npm_config_arch: arch } + }) + }) + const addon = join(scratch, 'worker-wait.node') + const compiled = await runProcess({ + program: 'clang', + args: [ + '-arch', + arch === 'x64' ? 'x86_64' : 'arm64', + '-O2', + '-shared', + '-fPIC', + '-undefined', + 'dynamic_lookup', + '-I', + join(headers, 'include/node'), + join(import.meta.dirname, '__fixtures__/darwin-terminal-worker-wait.c'), + '-o', + addon + ], + timeoutMs: 60_000 + }) + if (compiled.code !== 0) { + throw new Error(compiled.stderr) + } + const result = await runProcess({ + program: executable, + args: [ + join(import.meta.dirname, 'darwin-terminal-node-probe.cjs'), + addon, + stagedPty, + scratch, + `v${NODE_RUNTIME_PIN.version}` + ], + env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1', UV_THREADPOOL_SIZE: '4' }, + timeoutMs: 60_000 + }) + if (result.code !== 0) { + throw new Error(`Runtime proof failed (${result.code}): ${result.stderr}`) + } + if (JSON.parse(result.stdout.trim()).arch !== arch) { + throw new Error(`Runtime proof used the wrong architecture; expected ${arch}`) + } + console.log(result.stdout.trim()) + if (process.env.ORCA_DARWIN_WORKER_STRESS === '1') { + const stress = await runProcess({ + program: executable, + args: [ + join(import.meta.dirname, 'darwin-terminal-worker-stress.cjs'), + addon, + stagedPty, + scratch + ], + env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1', UV_THREADPOOL_SIZE: '4' }, + timeoutMs: 61 * 60_000 + }) + console.log(stress.stdout.trim()) + if (stress.code !== 0) { + throw new Error(`Sequential worker stress failed: ${stress.stderr}`) + } + } + } finally { + rmSync(scratch, { recursive: true, force: true }) + } +} + +if (isDirectInvocation(import.meta.url, process.argv[1])) { + await verifyDarwinTerminalNode(readDarwinVerificationArch(process.argv.slice(2))) +} diff --git a/config/scripts/win32-test-lane-registration.test.mjs b/config/scripts/win32-test-lane-registration.test.mjs index a191e2a9726..99ac812cd20 100644 --- a/config/scripts/win32-test-lane-registration.test.mjs +++ b/config/scripts/win32-test-lane-registration.test.mjs @@ -149,22 +149,35 @@ export function requiresEnvOptIn(source) { }) } -// Treat any runs-on label/array/group/matrix that may be Windows as a lane; unresolvable expression fails closed for review. -export function couldRunOnWindows(runsOn) { +// Treat any runs-on label/array/group/matrix that may be Windows as a lane; simple matrix axes resolve +// from their declared values and an unresolvable expression fails closed for review. +export function couldRunOnWindows(runsOn, matrix = {}) { const labels = typeof runsOn === 'string' ? [runsOn] : Array.isArray(runsOn) ? runsOn : [...(runsOn?.labels ?? []), runsOn?.group ?? ''].flat() - return labels.some((label) => /windows/i.test(String(label)) || String(label).includes('${{')) + return labels.some((label) => { + const axis = /^\$\{\{\s*matrix\.(\w+)\s*\}\}$/.exec(String(label))?.[1] + const values = axis && matrix[axis] + if (Array.isArray(values) && values.length > 0) { + const included = Array.isArray(matrix.include) + ? matrix.include.flatMap((entry) => (entry[axis] === undefined ? [] : [entry[axis]])) + : [] + return [...values, ...included].some((value) => couldRunOnWindows(value)) + } + return /windows/i.test(String(label)) || String(label).includes('${{') + }) } /** The Windows job's curated files: the boundary list plus any argv the lane steps spell out. */ function readWindowsWorkflow() { const workflow = parse(readFileSync(join(projectDir, '.github/workflows/pr.yml'), 'utf8')) const jobs = Object.entries(workflow.jobs ?? {}) - const windowsJobs = jobs.filter(([, job]) => couldRunOnWindows(job?.['runs-on'])) + const windowsJobs = jobs.filter(([, job]) => + couldRunOnWindows(job?.['runs-on'], job?.strategy?.matrix) + ) const steps = workflow.jobs?.[WINDOWS_LANE_JOB]?.steps ?? [] const runs = WINDOWS_LANE_STEPS.map((name) => { const step = steps.find((candidate) => candidate?.name === name) @@ -397,6 +410,21 @@ describe('manual opt-in classification', () => { }) describe('Windows runner detection', () => { + it('resolves explicit matrix axes while keeping unknown expressions conservative', () => { + expect(couldRunOnWindows('${{ matrix.os }}', { os: ['macos-15', 'macos-15-intel'] })).toBe( + false + ) + expect(couldRunOnWindows('${{ matrix.os }}', { os: ['macos-15', 'windows-2022'] })).toBe(true) + expect(couldRunOnWindows('${{ matrix.os }}', { os: [] })).toBe(true) + expect( + couldRunOnWindows('${{ matrix.os }}', { + os: ['macos-15'], + include: [{ os: 'windows-2022' }] + }) + ).toBe(true) + expect(couldRunOnWindows('${{ inputs.runner }}', { os: ['macos-15'] })).toBe(true) + }) + it('reads every runs-on spelling that could land on Windows', () => { expect(couldRunOnWindows('windows-2022')).toBe(true) // The spelling that would have slipped past an equality test. diff --git a/docs/bug-reproductions/darwin-worker-wakeup.md b/docs/bug-reproductions/darwin-worker-wakeup.md new file mode 100644 index 00000000000..b88bf3722eb --- /dev/null +++ b/docs/bug-reproductions/darwin-worker-wakeup.md @@ -0,0 +1,181 @@ +# macOS terminal daemon worker wakeup stall + +On 2026-10-15, a long-running packaged macOS daemon accepted RPC traffic and PTY +output but stopped delivering terminal input or creating terminals. Its Node +24.21.0/libuv 1.52.1 event loop remained responsive. All four worker threads were +idle in `__psynch_cvwait`, with 38 queued jobs and no completed work waiting for +the loop. Submitting one asynchronous stat grew the queue to 39 without a callback. +This distinguishes the incident from a busy or saturated worker pool. + +A read-only native snapshot found these pthread condition generations: + +```text +L = U = 0x953f6000 +S = 0x953f5c02 +worker wait L: 0x153f6000, 0x953f6000, 0x953f5f00, 0x953f5e00 +``` + +The oldest waiter was exactly `0x80000000` behind the current generation. This +matches the psynch half-range failure discussed in +[libuv #5280](https://github.com/libuv/libuv/issues/5280). Kernel waiter ordering +can turn wake signals into preposts; once the signal generation U catches L, +subsequent signals can return in userspace while workers remain asleep. + +## Recurrence on 2026-10-23 + +A preserved v50 daemon running the unpatched Node 24.21.0/libuv 1.52.1 +runtime stopped accepting input with 155 sessions. Authenticated control requests +still answered in 2–3 ms and terminal output continued; new terminal creation +timed out after 30 seconds. Asynchronous stat, directory and crypto callbacks +stalled, while all four workers slept in `__psynch_cvwait`, the global work queue +was nonempty and the completion queue was empty. The existing detector logged +`async-io-stalled`. A working Claude terminal belonged to a different, older +daemon, so the apparent Codex/Claude distinction was a daemon ownership difference. + +A one-off native diagnostic interrupted all four verified workers while holding +their pool mutex. Queued work drained, fresh stats completed in about 1 ms, and +input resumed with the same PTY incarnation and agent processes. The user +confirmed recovery. Inspector access was then closed. This validates recovery +without restarting agents, but private-memory access and signal injection are +not production recovery mechanisms; bounded worker waits are the runtime fix. + +## Reproduction and limits + +The [upstream candidate](https://github.com/libuv/libuv/pull/5351) includes a +manual witness using production libuv workers and their real queue. It parks one +worker, advances the condition through 8,388,605 timed waits on the **same mutex**, +enrolls three fresh workers, and submits four jobs. It changes no private pthread +counters and does not occupy the worker pool with blocking filesystem requests. + +| Build | Jobs | Callbacks | Queue | Idle workers | +| ------------------------------ | ---- | --------- | -------- | ------------ | +| Unpatched, half-range boundary | 0 | 0 | Nonempty | 4 | +| Patched, same aging witness | 4 | 4 | Empty | 4 | +| Short control, either build | 4 | 4 | Empty | 4 | + +The unpatched witness produced L = U = 2147483904, S = 2147482882, and oldest +wait L = 256: exactly the same half-range relationship as the live daemon. +The patched witness refreshed the old wait during aging and completed normally. +A separate idle-generation regression fails on unpatched libuv and passes with +the patch; nineteen focused pool, cancellation, fork and condition tests pass. + +The controlled witness proves the native mechanism; the later sequential-job +reproduction below establishes an ordinary scheduling path to it. The precise +Orca job-producer sequence is still unproven. Stress runs with up to 60 million jobs, stop/continue cycles, +signals and PTY churn did not independently reproduce that enrollment sequence. +The earlier PTY fault injection established symptoms only. + +## Ordinary-workload reproduction + +A later run on the exact unpatched packaged Node also reproduced the failure +without condition aging or fault injection: queue one empty native job, and queue +its successor from the completion callback. Four workers service one outstanding +job. An initial run stalled after 11,032,951 completed jobs and hung in worker +cleanup on normal exit. A second instrumented run started with a successful real +PTY echo and stalled after 11,886,884 completed jobs: + +```text +L = 3043043584; oldest worker wait L = 895559936 +L - oldest wait L = 2147483648 = 0x80000000 +four idle workers; one queued native job +``` + +Four fresh async stats and a PTY write produced zero stat callbacks and no PTY +echo after 15 seconds; six jobs remained queued and U caught L. One worker had +stayed parked while millions of sequential jobs ran through the others. This +provides an ordinary scheduling path to the exact live-daemon signature. An +otherwise similar 30-million-job run completed, so the scheduling is intermittent. +The patched runtime completed all 30 million jobs, a fresh filesystem request +and real PTY echo. This still does not identify the exact sequence of Orca's own +job producers. + +The repository now includes a portable-to-both-macOS-architectures N-API workload +and PTY oracle with no private pthread fields. Run it after the normal check: + +```sh +ORCA_BACKGROUND_LAUNCH=1 ORCA_DARWIN_WORKER_STRESS=1 node config/scripts/verify-darwin-terminal-node.mjs +``` + +The workload runs 30 million sequential native jobs and requires PTY echo before +and after. After three seconds without progress, it allows forty seconds for +the thirty-second worker timeout and scheduling margin, requiring all four fresh +stat callbacks, a new PTY echo and resumed native work. It kills only its own disposable test +process if libuv cleanup is stranded. This probabilistic stress is opt-in; the +idle-refresh regression remains the regular CI oracle. + +## Runtime mitigation + +Only the private Darwin worker wait changes: it uses a 30-second relative timed +condition wait, retaining the existing queue predicate and Darwin error workaround. +Workers still wake immediately when signaled. An idle timeout rechecks queued +work and refreshes the wait generation, allowing recovery from lost wakeups. +The default four-worker pool adds at most eight timeout wakes per minute; the +regression also caps generation advancement to reject an accidental wake loop. +Public condition variables and other platforms are unchanged. + +`config/scripts/build-darwin-terminal-node.mjs` builds the pinned official Node +source with this patch. The recipe now follows Node 24.22.1: its source archive +matches the official SHA-256 manifest, the patch applies with zero fuzz, and the +reviewed patched worker source is byte-identical to the Node 24.21.0 recipe. The recipe pins the source archive hash, reviewed patched +source hash, architecture and minimum OS. Cache receipts verify the resulting +executable bytes and fingerprint the build scripts, so changes to build logic +invalidate cached binaries. A Node pin change fails until the recipe is reviewed. +This preserves the existing signed +helper layout and stable host copy rather than changing its launch or permissions. + +Run the shipping-runtime check on each macOS architecture: + +```sh +ORCA_BACKGROUND_LAUNCH=1 node config/scripts/verify-darwin-terminal-node.mjs +``` + +It builds the runtime, checks real PTY input and filesystem/DNS/crypto operations, +observes all four idle worker generations refresh, bounds their advancement, and +checks fresh asynchronous work. It inspects only its own test process. The Node 24.21.0 recipe passed on Apple silicon (x64 under Rosetta) and on +physical ARM and Intel CI ([run](https://github.com/stablyai/orca/actions/runs/38795005777)). +On 2026-10-23, the repository-built Node 24.22.1 arm64 runtime also passed: + +```json +{ + "node": "v24.22.1", + "uv": "1.52.1", + "arch": "arm64", + "fsDnsCrypto": "passed", + "ptyInput": "passed", + "idleWaitRefresh": "passed" +} +``` + +The official unpatched 24.22.1 runtime passed the same initial PTY and I/O checks, +then failed the idle-generation `advances > 0n` assertion. This supplies a current-pin +red/green regression. ARM CI qualification passed on PR commit `6f006d865193` in +[run 39922624279](https://github.com/stablyai/orca/actions/runs/39922624279); +Intel qualification was still running at review time. The cold +build also exposed multi-minute synchronous source cleanup; the cache lease is +now released after atomic publication and before cleanup, so a blocked heartbeat +cannot let a waiting builder steal the completed producer's lock. Cold source compilation is +substantial and is cached separately from packaging retries. + +## Coverage and remaining risks + +The packaged macOS terminal daemon receives this runtime, including local folder +workspaces and worktrees, and clients attached to that daemon. Existing daemons +keep their executable until deliberately restarted. The detection/restart prompt +is tracked separately in [Orca #29087](https://github.com/stablyai/orca/pull/29087). +No automatic restart or queued-input replay is part of this patch. + +Linux/Windows and WSL are unaffected by the Darwin-only change. Standalone/SSH +server runtimes and the development Electron fallback do not receive this patch. +Mobile/relay clients benefit only when their execution host uses the patched +helper. An isolated Developer ID signed copy of the installed helper, with the +repository-built runtime substituted, served a PTY under hardened runtime. Its +protocol 45 required a matching smoke-test handshake; the original patch checkout used 46. A +repeat on October 23 with Node 24.22.1 passed strict signature verification, +real PTY input, filesystem/DNS/crypto and all four idle worker refreshes under +hardened runtime. This disposable copy used the installed helper's native +dependencies and an untimestamped Developer ID signature; it does not establish +notarization or qualification of a complete current package. A complete package +build, launchd/TCC attribution, older macOS releases, battery +impact and extended soak remain verification gaps. Do not infer remote process +death from missing callbacks or loss of contact. The experimental reliability gate records these +limits; an upstream merge and a Node release can eventually replace this backport. diff --git a/docs/reference/macos-daemon-tcc-attribution.md b/docs/reference/macos-daemon-tcc-attribution.md index fe3d9b61ad7..f4a559ea4e1 100644 --- a/docs/reference/macos-daemon-tcc-attribution.md +++ b/docs/reference/macos-daemon-tcc-attribution.md @@ -109,8 +109,9 @@ The helper declares no links, documents or types, has a lower `CFBundleVersion` launchd leaves no record, but another process running the copy's executable directly can make macOS register it, so retirement runs `lsregister -u` on the bundle before deleting it. -The helper's main-executable UUID is Node's, shared by every official Node 24.21.0 binary of that -architecture, so it is not unique to Orca. Local Network access still passed in the signed spike: +The original helper used the official Node UUID, shared by that release and architecture. +The worker-wakeup backport builds Node from source and has a different UUID; its launchd/TCC +attribution still needs verification. Local Network access passed in the original signed spike: macOS matched the client by its code signature, and the job's `AssociatedBundleIdentifiers` ties it to Orca. Do not register copies explicitly (`lsregister -f` or `LSRegisterURL`). diff --git a/src/main/daemon/daemon-adoption-telemetry-event.ts b/src/main/daemon/daemon-adoption-telemetry-event.ts index 4a66e2a3926..ce36c34a063 100644 --- a/src/main/daemon/daemon-adoption-telemetry-event.ts +++ b/src/main/daemon/daemon-adoption-telemetry-event.ts @@ -3,10 +3,10 @@ import { homedir } from 'node:os' import { join } from 'node:path' import { getAppEnvironment } from '../../shared/app-environment' import { + classifyDaemonAppVersionMatch, classifyDaemonPtyCwd, classifyDaemonSpawnerPath, isMacTccFolderClass, - type DaemonAdoptedAppVersionMatch, type DaemonSpawnerPathClass } from '../../shared/daemon-adoption-telemetry' import { bucketDaemonLiveSessionCount } from '../../shared/daemon-lifecycle-telemetry' @@ -43,11 +43,10 @@ function stableCopyPrefix(): string | null { export async function classifyDaemonAdoptionOrigin( pidRecord: ParsedDaemonPid | null ): Promise { - const appVersionMatch: DaemonAdoptedAppVersionMatch = !pidRecord?.appVersion - ? 'unknown' - : pidRecord.appVersion === getAppEnvironment().getVersion() - ? 'same' - : 'different' + const appVersionMatch = classifyDaemonAppVersionMatch( + pidRecord?.appVersion, + getAppEnvironment().getVersion() + ) const spawnerPathClass: DaemonSpawnerPathClass = classifyDaemonSpawnerPath( pidRecord?.spawnerExecPath ?? null, existsSync, diff --git a/src/main/daemon/daemon-async-io-telemetry-event.test.ts b/src/main/daemon/daemon-async-io-telemetry-event.test.ts new file mode 100644 index 00000000000..f42ff35846a --- /dev/null +++ b/src/main/daemon/daemon-async-io-telemetry-event.test.ts @@ -0,0 +1,50 @@ +import { expect, it, vi } from 'vitest' +import { validate } from '../telemetry/validator' +import { observeDaemonAsyncIoHealth } from './daemon-async-io-telemetry-event' + +const { trackMock } = vi.hoisted(() => ({ + trackMock: vi.fn((_name: 'daemon_async_io_health', _props: unknown) => true) +})) +vi.mock('../telemetry/client', () => ({ track: trackMock })) +vi.mock('../../shared/app-environment', () => ({ + getAppEnvironment: () => ({ getVersion: () => '1.4.300' }) +})) + +it('emits one stalled and one recovered per daemon incarnation across repeated polls', () => { + const stalled = { state: 'stalled', pendingMs: 5000 } as const + const healthy = { state: 'healthy', pendingMs: 0 } as const + const adopted = { pid: 1, startedAtMs: 1, launchNonce: 'a', appVersion: '1.4.299' } + const forked = { pid: 2, startedAtMs: 2, launchNonce: 'b', appVersion: '1.4.300' } + + observeDaemonAsyncIoHealth(adopted, 51, healthy, 0) + observeDaemonAsyncIoHealth(adopted, 51, stalled, 1_000) + observeDaemonAsyncIoHealth({ ...adopted }, 51, stalled, 31_000) + observeDaemonAsyncIoHealth(forked, 51, stalled, 31_000) + observeDaemonAsyncIoHealth(adopted, 51, healthy, 121_000) + observeDaemonAsyncIoHealth(adopted, 51, stalled, 151_000) + observeDaemonAsyncIoHealth(adopted, 51, healthy, 181_000) + + const calls = trackMock.mock.calls + expect(calls).toEqual([ + [ + 'daemon_async_io_health', + { transition: 'stalled', protocol_generation: 51, app_version_match: 'different' } + ], + [ + 'daemon_async_io_health', + { transition: 'stalled', protocol_generation: 51, app_version_match: 'same' } + ], + [ + 'daemon_async_io_health', + { + transition: 'recovered', + protocol_generation: 51, + app_version_match: 'different', + stalled_for_bucket: '1-5m' + } + ] + ]) + for (const [name, props] of calls) { + expect(validate(name, props).ok).toBe(true) + } +}) diff --git a/src/main/daemon/daemon-async-io-telemetry-event.ts b/src/main/daemon/daemon-async-io-telemetry-event.ts new file mode 100644 index 00000000000..fc3f47a4f5b --- /dev/null +++ b/src/main/daemon/daemon-async-io-telemetry-event.ts @@ -0,0 +1,55 @@ +// Telemetry must never fail a health read, so every error stops here. + +import { getAppEnvironment } from '../../shared/app-environment' +import { classifyDaemonAppVersionMatch } from '../../shared/daemon-adoption-telemetry' +import type { DaemonAsyncIoHealth } from '../../shared/daemon-async-io-health' +import type { DAEMON_ASYNC_IO_STALL_DURATION_BUCKETS } from '../../shared/telemetry-daemon-event-schemas' +import { track } from '../telemetry/client' +import type { DaemonEndpointIdentity } from './daemon-hello-protocol' + +type StallObservation = { stalledAtMs: number; recovered: boolean } + +// Keyed by launch nonce so reconnects and multiple polling windows share one record per incarnation. +const observedStalls = new Map() + +function bucketStallDuration(ms: number): (typeof DAEMON_ASYNC_IO_STALL_DURATION_BUCKETS)[number] { + return ms < 60_000 ? '<1m' : ms < 300_000 ? '1-5m' : ms < 1_800_000 ? '5-30m' : '30m+' +} + +/** Emits at most one `stalled` and one later `recovered` per daemon incarnation. */ +export function observeDaemonAsyncIoHealth( + identity: DaemonEndpointIdentity, + protocolVersion: number, + health: DaemonAsyncIoHealth, + nowMs: number = Date.now() +): void { + try { + const seen = observedStalls.get(identity.launchNonce) + const isTransition = health.state === 'stalled' ? !seen : seen?.recovered === false + if (!isTransition) { + return + } + const base = { + protocol_generation: protocolVersion, + app_version_match: classifyDaemonAppVersionMatch( + identity.appVersion, + getAppEnvironment().getVersion() + ) + } + if (!seen) { + // Record only a delivered stall so a pre-init poll can report it later. + if (track('daemon_async_io_health', { transition: 'stalled', ...base })) { + observedStalls.set(identity.launchNonce, { stalledAtMs: nowMs, recovered: false }) + } + return + } + seen.recovered = true + track('daemon_async_io_health', { + transition: 'recovered', + ...base, + stalled_for_bucket: bucketStallDuration(nowMs - seen.stalledAtMs) + }) + } catch { + // Telemetry is best-effort; a dropped event must not fail a health read. + } +} diff --git a/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts b/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts index 7e9c560ccc0..13ade9d2952 100644 --- a/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts +++ b/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts @@ -3,7 +3,7 @@ import { PREVIOUS_DAEMON_PROTOCOL_VERSIONS, PROTOCOL_VERSION } from './types' describe('foreground-confirmation daemon protocol', () => { it('rejects daemons from before the fresh-confirmation RPC', () => { - expect(PROTOCOL_VERSION).toBe(51) + expect(PROTOCOL_VERSION).toBe(52) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(19) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(22) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(23) @@ -23,6 +23,6 @@ describe('foreground-confirmation daemon protocol', () => { expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(37) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(38) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(39) - expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(50) + expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(51) }) }) diff --git a/src/main/daemon/daemon-protocol-version.ts b/src/main/daemon/daemon-protocol-version.ts index 0416fcc0f03..da94db289e3 100644 --- a/src/main/daemon/daemon-protocol-version.ts +++ b/src/main/daemon/daemon-protocol-version.ts @@ -1,4 +1,5 @@ // Why: daemons survive app updates, so wire behavior must be version-gated. +// v52 starts new terminals on a fresh daemon whose patched Node recovers lost macOS worker wake-ups. // v51 checks terminal identity, ownership and attachments before reserving teardown. // v50 makes the `codex` function keep Orca's terminal identity under a filtering shell_environment_policy. // v49 makes the `claude` function follow the selection over the user's own CLAUDE_CONFIG_DIR. @@ -9,7 +10,7 @@ // v43 dropped it with the revert); v41 stages long startup commands as sourced scripts; v40 rolled // the #25130/#24636 shell-wrapper changes and the wider agent list into a fresh daemon; older owners // stay attachable. -export const PROTOCOL_VERSION = 51 +export const PROTOCOL_VERSION = 52 export const FENCED_TERMINATION_PROTOCOL_VERSION = 51 export const CLAUDE_ACCOUNT_FUNCTION_DAEMON_PROTOCOL_VERSION = 42 export const CLAUDE_ACCOUNT_FUNCTION_REVERTED_DAEMON_PROTOCOL_VERSION = 43 @@ -44,7 +45,7 @@ export const CLEAN_DISCONNECT_PROTOCOL_VERSION = 24 export const MODE_2031_UNSUBSCRIBE_FACT_PROTOCOL_VERSION = 29 export const PREVIOUS_DAEMON_PROTOCOL_VERSIONS = [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, - 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50 + 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 ] as const export function supportsColorQueryReplyColors(protocolVersion: number): boolean { diff --git a/src/main/daemon/daemon-pty-adapter-async-io-health.test.ts b/src/main/daemon/daemon-pty-adapter-async-io-health.test.ts index 469ed5af79b..01dc266cf37 100644 --- a/src/main/daemon/daemon-pty-adapter-async-io-health.test.ts +++ b/src/main/daemon/daemon-pty-adapter-async-io-health.test.ts @@ -2,6 +2,9 @@ import { afterEach, expect, it, vi } from 'vitest' import { DaemonClient } from './client' import { DaemonPtyAdapter } from './daemon-pty-adapter' +const { observeMock } = vi.hoisted(() => ({ observeMock: vi.fn() })) +vi.mock('./daemon-async-io-telemetry-event', () => ({ observeDaemonAsyncIoHealth: observeMock })) + afterEach(() => vi.restoreAllMocks()) it.each([ @@ -43,3 +46,16 @@ it('never reconnects or restarts a daemon to read health', async () => { expect(request).not.toHaveBeenCalled() expect(connect).not.toHaveBeenCalled() }) + +it('reports health for the same incarnation even though identity reads return copies', async () => { + const identity = { pid: 7, startedAtMs: 1, launchNonce: 'n' } + vi.spyOn(DaemonClient.prototype, 'isConnected').mockReturnValue(true) + vi.spyOn(DaemonClient.prototype, 'getDaemonIdentity').mockImplementation(() => ({ ...identity })) + vi.spyOn(DaemonClient.prototype, 'request').mockResolvedValue({ + state: 'stalled', + pendingMs: 5000 + }) + const adapter = new DaemonPtyAdapter({ socketPath: '/unused', tokenPath: '/unused' }) + await adapter.getAsyncIoHealth() + expect(observeMock).toHaveBeenCalledOnce() +}) diff --git a/src/main/daemon/daemon-pty-adapter.ts b/src/main/daemon/daemon-pty-adapter.ts index 889621e47b1..f6ddd69467f 100644 --- a/src/main/daemon/daemon-pty-adapter.ts +++ b/src/main/daemon/daemon-pty-adapter.ts @@ -19,7 +19,9 @@ import { } from '../../shared/host-spawn-admission' import { isUnknownRequestTypeError } from './daemon-endpoint-errors' import type { AutomaticWorkRequestType } from './daemon-automatic-work-protocol' -import type { DaemonAsyncIoStatus } from '../../shared/daemon-async-io-health' +import type { DaemonAsyncIoHealth, DaemonAsyncIoStatus } from '../../shared/daemon-async-io-health' +import { observeDaemonAsyncIoHealth } from './daemon-async-io-telemetry-event' +import { sameEndpointIdentity } from './daemon-endpoint-incarnation' export class DaemonPtyAdapter extends DaemonPtyDaemonRecovery implements IPtyProvider { constructor(opts: DaemonPtyAdapterOptions) { @@ -73,6 +75,7 @@ export class DaemonPtyAdapter extends DaemonPtyDaemonRecovery implements IPtyPro if (!this.client.isConnected()) { return { state: 'unverifiable' } } + const identity = this.client.getDaemonIdentity() try { const health = await this.client.request('asyncIoHealth', undefined, 7_000) if ( @@ -85,7 +88,13 @@ export class DaemonPtyAdapter extends DaemonPtyDaemonRecovery implements IPtyPro Number.isFinite(health.pendingMs) && health.pendingMs >= 0 ) { - return { state: health.state, pendingMs: health.pendingMs } + const result: DaemonAsyncIoHealth = { state: health.state, pendingMs: health.pendingMs } + // A reply that crossed a reconnect cannot be attributed to either incarnation. + const current = this.client.getDaemonIdentity() + if (identity && current && sameEndpointIdentity(identity, current)) { + observeDaemonAsyncIoHealth(identity, this.protocolVersion, result) + } + return result } } catch (error) { if (isUnknownRequestTypeError(error)) { diff --git a/src/main/ipc/telemetry.ts b/src/main/ipc/telemetry.ts index c58e345066e..625343321a7 100644 --- a/src/main/ipc/telemetry.ts +++ b/src/main/ipc/telemetry.ts @@ -23,6 +23,7 @@ const MAIN_OWNED_TELEMETRY_EVENTS = new Set([ 'agent_token_usage', 'app_starred_orca', 'daemon_adopted', + 'daemon_async_io_health', 'daemon_audit_eligibility', 'daemon_pty_cwd_denied', 'daemon_pty_cwd_readable', diff --git a/src/shared/daemon-adoption-telemetry.ts b/src/shared/daemon-adoption-telemetry.ts index 2ce4d0f990c..76ba3f8d085 100644 --- a/src/shared/daemon-adoption-telemetry.ts +++ b/src/shared/daemon-adoption-telemetry.ts @@ -44,6 +44,13 @@ export function isMacTccFolderClass(cwdClass: DaemonPtyCwdClass): cwdClass is Ma return MAC_TCC_FOLDER_CLASSES.some((name) => name === cwdClass) } +export function classifyDaemonAppVersionMatch( + daemonAppVersion: string | null | undefined, + appVersion: string +): DaemonAdoptedAppVersionMatch { + return !daemonAppVersion ? 'unknown' : daemonAppVersion === appVersion ? 'same' : 'different' +} + export function classifyDaemonSpawnerPath( spawnerExecPath: string | null, exists: (path: string) => boolean, diff --git a/src/shared/local-build-compatibility-contract.json b/src/shared/local-build-compatibility-contract.json index 031ab7ce8dc..6732acba726 100644 --- a/src/shared/local-build-compatibility-contract.json +++ b/src/shared/local-build-compatibility-contract.json @@ -3,10 +3,10 @@ "appId": "com.stablyai.orca", "stateSchemaVersion": 1, "readableStateSchemaVersions": [1], - "daemonProtocolVersion": 51, + "daemonProtocolVersion": 52, "attachableDaemonProtocolVersions": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, - 51 + 51, 52 ] } diff --git a/src/shared/local-build-compatibility-contract.ts b/src/shared/local-build-compatibility-contract.ts index f719d6759b8..ce1b8bd573a 100644 --- a/src/shared/local-build-compatibility-contract.ts +++ b/src/shared/local-build-compatibility-contract.ts @@ -3,10 +3,10 @@ export const LOCAL_BUILD_COMPATIBILITY_CONTRACT = { appId: 'com.stablyai.orca', stateSchemaVersion: 1, readableStateSchemaVersions: [1], - daemonProtocolVersion: 51, + daemonProtocolVersion: 52, attachableDaemonProtocolVersions: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, - 51 + 51, 52 ] } as const diff --git a/src/shared/telemetry-daemon-event-schemas.ts b/src/shared/telemetry-daemon-event-schemas.ts index 7f6ca4f0536..8de1a9cf86b 100644 --- a/src/shared/telemetry-daemon-event-schemas.ts +++ b/src/shared/telemetry-daemon-event-schemas.ts @@ -70,6 +70,24 @@ export const daemonAdoptedSchema = z }) .strict() +export const DAEMON_ASYNC_IO_STALL_DURATION_BUCKETS = ['<1m', '1-5m', '5-30m', '30m+'] as const + +// Why: measures whether #29133's runtime patch makes daemon worker-pool stalls recover. +const daemonAsyncIoHealthBase = { + protocol_generation: z.number().int().positive().max(1_000), + app_version_match: z.enum(DAEMON_ADOPTED_APP_VERSION_MATCH) +} +export const daemonAsyncIoHealthSchema = z.discriminatedUnion('transition', [ + z.object({ transition: z.literal('stalled'), ...daemonAsyncIoHealthBase }).strict(), + z + .object({ + transition: z.literal('recovered'), + ...daemonAsyncIoHealthBase, + stalled_for_bucket: z.enum(DAEMON_ASYNC_IO_STALL_DURATION_BUCKETS) + }) + .strict() +]) + // Denied means proven app/daemon cwd-access divergence; readable is the TCC-folder-only control (#17696). export const daemonPtyCwdVerdictSchema = z .object({ cwd_class: z.enum(DAEMON_PTY_CWD_CLASSES), ...daemonOriginProps }) diff --git a/src/shared/telemetry-event-registry.ts b/src/shared/telemetry-event-registry.ts index 989efcadf94..68d6061ae2e 100644 --- a/src/shared/telemetry-event-registry.ts +++ b/src/shared/telemetry-event-registry.ts @@ -26,6 +26,7 @@ import { agentHookUnattributedSchema, codexTrustGrantSchema, daemonAdoptedSchema, + daemonAsyncIoHealthSchema, daemonAuditEligibilitySchema, daemonFolderAccessNoticeSchema, daemonLifecycleSchema, @@ -146,6 +147,7 @@ export const eventSchemas = { main_thread_hang_detected: mainThreadHangDetectedSchema, daemon_lifecycle: daemonLifecycleSchema, daemon_adopted: daemonAdoptedSchema, + daemon_async_io_health: daemonAsyncIoHealthSchema, daemon_pty_cwd_denied: daemonPtyCwdVerdictSchema, daemon_pty_cwd_readable: daemonPtyCwdVerdictSchema, daemon_folder_access_notice: daemonFolderAccessNoticeSchema,