From a5f7da49288da5c2bada69298d3aa18f0a5fab03 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Sun, 25 Oct 2026 20:07:30 -0400 Subject: [PATCH] revert(daemon): ship official Node in the macOS terminal host again, keep stall telemetry (#33688) Reverts #29133 except its daemon async-I/O stall telemetry, so we can measure how often stalls happen on official Node before committing to a patched runtime. The macOS terminal host goes back to the official Node 24.22.1 download and the daemon protocol back to v51; no release or hourly build published v52. --- .github/workflows/adhoc-mac-build.yml | 26 +- .github/workflows/daily-mac-build.yml | 31 +-- .github/workflows/hourly-mac-build.yml | 36 ++- .github/workflows/pr.yml | 34 --- .github/workflows/release-cut.yml | 11 +- .github/workflows/release-mac-build.yml | 18 +- .../darwin-worker-wait.json | 7 - .../darwin-worker-wait.patch | 46 ---- config/reliability-gates.jsonc | 73 ------ .../darwin-terminal-worker-wait.c | 128 ---------- .../__fixtures__/darwin-threadpool-wait.h | 68 ------ .../adhoc-windows-parallelism.test.mjs | 13 +- config/scripts/build-darwin-terminal-node.mjs | 222 ------------------ .../build-darwin-terminal-node.test.mjs | 92 -------- config/scripts/darwin-terminal-node-probe.cjs | 80 ------- .../darwin-terminal-node-workflow.test.mjs | 147 ------------ .../scripts/darwin-terminal-worker-stress.cjs | 82 ------- config/scripts/macos-terminal-host-bundle.cjs | 6 +- .../mobile-release-check-scope.test.mjs | 1 - ...package-electron-runtime-contract.test.mjs | 2 +- .../pr-auxiliary-preflight-admission.test.mjs | 1 - config/scripts/pr-code-change-scope.mjs | 40 ++-- .../scripts/pr-darwin-terminal-node-route.mjs | 20 -- ...lease-mac-build-workflow-dispatch.test.mjs | 82 +------ .../release-windows-signing-gate.test.mjs | 34 +-- .../run-release-mac-build-workflow.mjs | 22 +- .../scripts/verify-darwin-terminal-node.mjs | 118 ---------- .../win32-test-lane-registration.test.mjs | 36 +-- .../bug-reproductions/darwin-worker-wakeup.md | 181 -------------- .../reference/macos-daemon-tcc-attribution.md | 5 +- ...n-foreground-confirmation-protocol.test.ts | 4 +- src/main/daemon/daemon-protocol-version.ts | 5 +- .../local-build-compatibility-contract.json | 4 +- .../local-build-compatibility-contract.ts | 4 +- src/shared/telemetry-daemon-event-schemas.ts | 2 +- 35 files changed, 87 insertions(+), 1594 deletions(-) delete mode 100644 config/node-runtime-patches/darwin-worker-wait.json delete mode 100644 config/node-runtime-patches/darwin-worker-wait.patch delete mode 100644 config/scripts/__fixtures__/darwin-terminal-worker-wait.c delete mode 100644 config/scripts/__fixtures__/darwin-threadpool-wait.h delete mode 100644 config/scripts/build-darwin-terminal-node.mjs delete mode 100644 config/scripts/build-darwin-terminal-node.test.mjs delete mode 100644 config/scripts/darwin-terminal-node-probe.cjs delete mode 100644 config/scripts/darwin-terminal-node-workflow.test.mjs delete mode 100644 config/scripts/darwin-terminal-worker-stress.cjs delete mode 100644 config/scripts/pr-darwin-terminal-node-route.mjs delete mode 100644 config/scripts/verify-darwin-terminal-node.mjs delete mode 100644 docs/bug-reproductions/darwin-worker-wakeup.md diff --git a/.github/workflows/adhoc-mac-build.yml b/.github/workflows/adhoc-mac-build.yml index ed1fad297d1..7cabc3667bb 100644 --- a/.github/workflows/adhoc-mac-build.yml +++ b/.github/workflows/adhoc-mac-build.yml @@ -267,8 +267,8 @@ jobs: matrix: arch: [x64, arm64] runs-on: blacksmith-12vcpu-macos-15 - # Allow 360min: 180 for a cold patched-runtime build + 150 for install/publish retries and setup. - timeout-minutes: 360 + # Allow 150min for install 3×10 + publish 2×45 + ~25 setup/build/verify; outer timeout would prevent cleanup. + timeout-minutes: 150 env: JOBS: max NODE_OPTIONS: --max-old-space-size=4096 @@ -308,7 +308,7 @@ jobs: pnpm-lock.yaml mobile/pnpm-lock.yaml - # Cache the patched host Node by exact pin and patch, without restore keys. + # Cache bundled host Node by exact pin, without restore keys, so bumps cannot restore old archives. - name: Locate the pinned Node runtime cache shell: bash run: echo "ORCA_NODE_RUNTIME_CACHE_DIR=$HOME/Library/Caches/orca-node-runtime" >>"$GITHUB_ENV" @@ -317,7 +317,7 @@ jobs: uses: actions/cache@v5 with: path: ~/Library/Caches/orca-node-runtime - key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts', 'config/node-runtime-patches/**', 'config/scripts/build-darwin-terminal-node.mjs', 'config/scripts/pinned-node-downloads.mjs', 'config/scripts/update-node-runtime-pin.mjs', 'config/scripts/macos-terminal-host-bundle.cjs') }} + key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts') }} - name: Cache electron-builder downloads uses: actions/cache@v5 @@ -338,15 +338,6 @@ jobs: retry_wait_seconds: 30 command: pnpm install --frozen-lockfile --cpu=current,x64,arm64 - # Why: older refs predate the source build and package the pinned download instead. - - name: Build and verify the patched terminal runtimes - if: hashFiles('config/scripts/build-darwin-terminal-node.mjs') != '' - timeout-minutes: 180 - run: | - pnpm run build:packages - ORCA_BACKGROUND_LAUNCH=1 node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/verify-darwin-terminal-node.mjs --arch=arm64 - ORCA_BACKGROUND_LAUNCH=1 node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/verify-darwin-terminal-node.mjs --arch=x64 - # Build required out/mobile-web after installing its React Native/Expo dependencies. - uses: ./.github/actions/install-mobile-dependencies if: needs.release-javascript.outputs.supported != 'true' @@ -432,7 +423,7 @@ jobs: timeout_minutes: 45 max_attempts: 2 retry_wait_seconds: 30 - command: if [ ! -f config/scripts/build-darwin-terminal-node.mjs ]; then node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64; fi && node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_ADHOC=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac dmg zip --${{ matrix.arch }} --publish never + command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_ADHOC=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac dmg zip --${{ matrix.arch }} --publish never env: ORCA_ADHOC_BUILD_VERSION: ${{ needs.orcad-template-support.outputs.version }} ORCA_BUILD_COMMIT: ${{ needs.orcad-template-support.outputs.commit }} @@ -665,12 +656,12 @@ jobs: fast_release: true javascript_source_sha: ${{ needs.release-javascript.outputs.supported == 'true' && needs.release-javascript.outputs.source_sha || '' }} - # Wait outside the publisher runner so cold runtime builds cannot exhaust its timeout. + # Prefetch the installer during notarization; the publisher still waits for Mac success. publish-adhoc-win: - needs: [orcad-template-support, build-adhoc-win, build-adhoc-mac] + needs: [orcad-template-support, build-adhoc-win] if: >- !cancelled() && needs.orcad-template-support.result == 'success' && - needs.build-adhoc-win.result == 'success' && needs.build-adhoc-mac.result == 'success' + needs.build-adhoc-win.result == 'success' permissions: contents: read actions: read @@ -679,3 +670,4 @@ jobs: with: channel: adhoc tag: v${{ needs.orcad-template-support.outputs.version }} + wait_for_mac_job: build-adhoc-mac diff --git a/.github/workflows/daily-mac-build.yml b/.github/workflows/daily-mac-build.yml index dfec074a0dd..38e20094ad6 100644 --- a/.github/workflows/daily-mac-build.yml +++ b/.github/workflows/daily-mac-build.yml @@ -69,8 +69,8 @@ jobs: head_sha: ${{ steps.freshness.outputs.head_sha }} published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }} runs-on: blacksmith-6vcpu-macos-15 - # Allow 360min: 180 for a cold patched-runtime build + 150 for install/publish retries and setup. - timeout-minutes: 360 + # Allow 150min for install 3×10 + publish 2×45 + ~25 setup/build/verify, so outer timeout cannot strand cleanup mid-retry. + timeout-minutes: 150 env: NODE_OPTIONS: --max-old-space-size=4096 steps: @@ -143,7 +143,7 @@ jobs: pnpm-lock.yaml mobile/pnpm-lock.yaml - # Cache the patched host Node by exact pin and patch, without restore keys. + # Cache bundled host Node by exact pin without restore keys. - name: Locate the pinned Node runtime cache if: steps.freshness.outputs.should_build == 'true' shell: bash @@ -154,7 +154,7 @@ jobs: uses: actions/cache@v5 with: path: ~/Library/Caches/orca-node-runtime - key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts', 'config/node-runtime-patches/**', 'config/scripts/build-darwin-terminal-node.mjs', 'config/scripts/pinned-node-downloads.mjs', 'config/scripts/update-node-runtime-pin.mjs', 'config/scripts/macos-terminal-host-bundle.cjs') }} + key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts') }} - name: Cache electron-builder downloads if: steps.freshness.outputs.should_build == 'true' @@ -177,14 +177,6 @@ jobs: retry_wait_seconds: 30 command: pnpm install --frozen-lockfile --cpu=current,x64,arm64 - - name: Build and verify the patched terminal runtimes - if: steps.freshness.outputs.should_build == 'true' - timeout-minutes: 180 - run: | - pnpm run build:packages - ORCA_BACKGROUND_LAUNCH=1 node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/verify-darwin-terminal-node.mjs --arch=arm64 - ORCA_BACKGROUND_LAUNCH=1 node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/verify-darwin-terminal-node.mjs --arch=x64 - # Build required out/mobile-web using mobile’s React Native/Expo install. - uses: ./.github/actions/install-mobile-dependencies if: steps.freshness.outputs.should_build == 'true' @@ -201,23 +193,12 @@ jobs: APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }} APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} - # Why: App tokens last one hour, and the cold runtime build alone can outlast that. - - name: Re-mint daily repo token for versioning - id: app_token_version - if: steps.freshness.outputs.should_build == 'true' - uses: actions/create-github-app-token@v2 - with: - app-id: ${{ secrets.HOURLY_RELEASE_APP_ID }} - private-key: ${{ secrets.HOURLY_RELEASE_APP_PRIVATE_KEY }} - owner: stablyai - repositories: orca-daily - - name: Compute daily version id: daily if: steps.freshness.outputs.should_build == 'true' shell: bash env: - GH_TOKEN: ${{ steps.app_token_version.outputs.token }} + GH_TOKEN: ${{ steps.app_token.outputs.token }} MAIN_REPO_TOKEN: ${{ github.token }} run: | set -euo pipefail @@ -322,7 +303,7 @@ jobs: timeout_minutes: 45 max_attempts: 2 retry_wait_seconds: 30 - command: node config/scripts/ensure-native-runtime.mjs --runtime=electron && if [ -f config/scripts/build-mac-notarization-overlap.mjs ]; then ORCA_MAC_DAILY=1 node config/scripts/build-mac-notarization-overlap.mjs --publish always; else ORCA_MAC_DAILY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always; fi + command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && if [ -f config/scripts/build-mac-notarization-overlap.mjs ]; then ORCA_MAC_DAILY=1 node config/scripts/build-mac-notarization-overlap.mjs --publish always; else ORCA_MAC_DAILY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always; fi env: # Give the publisher App write access to its configured orca-daily target. GH_TOKEN: ${{ steps.app_token_publish.outputs.token }} diff --git a/.github/workflows/hourly-mac-build.yml b/.github/workflows/hourly-mac-build.yml index 1e28bf7db0a..ca4ee37aa76 100644 --- a/.github/workflows/hourly-mac-build.yml +++ b/.github/workflows/hourly-mac-build.yml @@ -105,8 +105,8 @@ jobs: head_sha: ${{ needs.preflight.outputs.head_sha }} published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }} runs-on: blacksmith-6vcpu-macos-15 - # Allow 360min: 180 for a cold patched-runtime build + 150 for install/publish retries and setup. - timeout-minutes: 360 + # Allow 150min for install 3×10 + publish 2×45 + ~25 setup/build/verify; outer timeout otherwise interrupts cleanup. + timeout-minutes: 150 env: NODE_OPTIONS: --max-old-space-size=4096 steps: @@ -119,6 +119,15 @@ jobs: # Read this repo without persisted credentials; write orca-hourly via env App token (zizmor: artipacked). persist-credentials: false + - name: Mint hourly repo token + id: app_token + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ secrets.HOURLY_RELEASE_APP_ID }} + private-key: ${{ secrets.HOURLY_RELEASE_APP_PRIVATE_KEY }} + owner: stablyai + repositories: orca-hourly + - name: Setup pnpm uses: ./.github/actions/setup-pnpm @@ -132,7 +141,7 @@ jobs: pnpm-lock.yaml mobile/pnpm-lock.yaml - # Cache the patched host Node by exact pin and patch, without restore keys. + # Cache bundled host Node by exact pin without restore keys. - name: Locate the pinned Node runtime cache shell: bash run: echo "ORCA_NODE_RUNTIME_CACHE_DIR=$HOME/Library/Caches/orca-node-runtime" >>"$GITHUB_ENV" @@ -141,7 +150,7 @@ jobs: uses: actions/cache@v5 with: path: ~/Library/Caches/orca-node-runtime - key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts', 'config/node-runtime-patches/**', 'config/scripts/build-darwin-terminal-node.mjs', 'config/scripts/pinned-node-downloads.mjs', 'config/scripts/update-node-runtime-pin.mjs', 'config/scripts/macos-terminal-host-bundle.cjs') }} + key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts') }} - name: Cache electron-builder downloads uses: actions/cache@v5 @@ -162,13 +171,6 @@ jobs: retry_wait_seconds: 30 command: pnpm install --frozen-lockfile --cpu=current,x64,arm64 - - name: Build and verify the patched terminal runtimes - timeout-minutes: 180 - run: | - pnpm run build:packages - ORCA_BACKGROUND_LAUNCH=1 node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/verify-darwin-terminal-node.mjs --arch=arm64 - ORCA_BACKGROUND_LAUNCH=1 node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/verify-darwin-terminal-node.mjs --arch=x64 - # Build required out/mobile-web from mobile’s React Native/Expo install. - uses: ./.github/actions/install-mobile-dependencies @@ -182,16 +184,6 @@ jobs: APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }} APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} - # Why here: App tokens last one hour, and the cold runtime build alone can outlast that. - - name: Mint hourly repo token - id: app_token - uses: actions/create-github-app-token@v2 - with: - app-id: ${{ secrets.HOURLY_RELEASE_APP_ID }} - private-key: ${{ secrets.HOURLY_RELEASE_APP_PRIVATE_KEY }} - owner: stablyai - repositories: orca-hourly - - name: Compute hourly version id: hourly shell: bash @@ -290,7 +282,7 @@ jobs: timeout_minutes: 45 max_attempts: 2 retry_wait_seconds: 30 - command: node config/scripts/ensure-native-runtime.mjs --runtime=electron && if [ -f config/scripts/build-mac-notarization-overlap.mjs ]; then ORCA_MAC_HOURLY=1 node config/scripts/build-mac-notarization-overlap.mjs --publish always; else ORCA_MAC_HOURLY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always; fi + command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && if [ -f config/scripts/build-mac-notarization-overlap.mjs ]; then ORCA_MAC_HOURLY=1 node config/scripts/build-mac-notarization-overlap.mjs --publish always; else ORCA_MAC_HOURLY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always; fi env: # Give publisher token write access to configured orca-hourly repo. GH_TOKEN: ${{ steps.app_token_publish.outputs.token }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c2a88e2a909..8245654acae 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -50,7 +50,6 @@ jobs: windows_process_probe: ${{ steps.filter.outputs.windows_process_probe }} package_windows: ${{ steps.readiness.outputs.reused != 'true' && steps.filter.outputs.package_windows }} layout_oracle: ${{ steps.readiness.outputs.reused != 'true' && steps.filter.outputs.layout_oracle }} - darwin_terminal_node: ${{ steps.readiness.outputs.reused != 'true' && steps.filter.outputs.darwin_terminal_node }} dependency_isolation: ${{ steps.readiness.outputs.reused != 'true' && steps.filter.outputs.dependency_isolation }} e2e_should_run: ${{ steps.e2e_filter.outputs.should_run }} test_files: ${{ steps.e2e_filter.outputs.test_files }} @@ -77,7 +76,6 @@ jobs: /config/.bun-version /config/scripts/check-readme-local-links.mjs /config/scripts/pr-code-change-scope.mjs - /config/scripts/pr-darwin-terminal-node-route.mjs /.oxfmtrc.json /config/scripts/pr-package-test-paths.mjs /config/scripts/pr-layout-oracle-route.mjs @@ -1496,34 +1494,6 @@ jobs: *) echo "node_modules/zod is not in the isolated store"; exit 1 ;; esac - darwin_terminal_node: - name: mac terminal worker wakeup (${{ matrix.os }}) - needs: [code_paths, preflight] - if: >- - !cancelled() && needs.preflight.result == 'success' && - needs.code_paths.outputs.darwin_terminal_node == 'true' - strategy: - fail-fast: false - matrix: - os: [macos-15, macos-15-intel] - runs-on: ${{ matrix.os }} - timeout-minutes: 180 - env: - ORCA_BACKGROUND_LAUNCH: '1' - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - - uses: ./.github/actions/install-node-dependencies - with: - native-runtime: node - - uses: actions/cache@v5 - with: - path: out/node-runtime-cache - key: darwin-terminal-node-${{ runner.arch }}-${{ hashFiles('src/shared/node-runtime-pin.ts', 'config/node-runtime-patches/**', 'config/scripts/build-darwin-terminal-node.mjs', 'config/scripts/pinned-node-downloads.mjs', 'config/scripts/update-node-runtime-pin.mjs', 'config/scripts/macos-terminal-host-bundle.cjs') }} - - name: Build and verify the shipping terminal runtime - run: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/verify-darwin-terminal-node.mjs - verify: name: PR checks passed # Publish an explicit result after every required dependency settles. @@ -1541,7 +1511,6 @@ jobs: - managed_hook_node18 - package - package_windows - - darwin_terminal_node - layout_oracle - dependency_isolation # Report settled results without competing with the GitHub-hosted test queue. @@ -1578,8 +1547,6 @@ jobs: PACKAGE_SHOULD_RUN: ${{ needs.code_paths.outputs.package }} PACKAGE_WINDOWS: ${{ needs.package_windows.result }} PACKAGE_WINDOWS_SHOULD_RUN: ${{ needs.code_paths.outputs.package_windows }} - DARWIN_TERMINAL_NODE: ${{ needs.darwin_terminal_node.result }} - DARWIN_TERMINAL_NODE_SHOULD_RUN: ${{ needs.code_paths.outputs.darwin_terminal_node }} LAYOUT_ORACLE: ${{ needs.layout_oracle.result }} LAYOUT_ORACLE_SHOULD_RUN: ${{ needs.code_paths.outputs.layout_oracle }} DEPENDENCY_ISOLATION: ${{ needs.dependency_isolation.result }} @@ -1625,7 +1592,6 @@ jobs: check_job managed_hook_node18 "$MANAGED_HOOK_NODE18" "$MANAGED_HOOK_NODE18_SHOULD_RUN" check_job package "$PACKAGE" "$PACKAGE_SHOULD_RUN" check_job package_windows "$PACKAGE_WINDOWS" "$PACKAGE_WINDOWS_SHOULD_RUN" - check_job darwin_terminal_node "$DARWIN_TERMINAL_NODE" "$DARWIN_TERMINAL_NODE_SHOULD_RUN" check_job layout_oracle "$LAYOUT_ORACLE" "$LAYOUT_ORACLE_SHOULD_RUN" check_job dependency_isolation "$DEPENDENCY_ISOLATION" "$DEPENDENCY_ISOLATION_SHOULD_RUN" exit "$failed" diff --git a/.github/workflows/release-cut.yml b/.github/workflows/release-cut.yml index f126320066d..7ae1f1d613e 100644 --- a/.github/workflows/release-cut.yml +++ b/.github/workflows/release-cut.yml @@ -1071,7 +1071,6 @@ jobs: - release-preflight - release-javascript - build-mac - - verify-mac-build # Why not the implicit success(): a tag without the orcad template skips that job on purpose. if: >- !cancelled() && @@ -1081,7 +1080,6 @@ jobs: needs.release-preflight.result == 'success' && needs.release-javascript.result == 'success' && needs.build-mac.result == 'success' && - needs.verify-mac-build.result == 'success' && (needs.orcad-template.result == 'success' || (needs.orcad-template.result == 'skipped' && needs.cut.outputs.ships_orcad_template == 'false')) env: @@ -1469,7 +1467,7 @@ jobs: path: signing-stage/** if-no-files-found: error - # Recheck the exact successful macOS run before consuming signing quota. + # Unsigned packaging overlaps macOS; only successful macOS builds may consume signing quota. - name: Require macOS success before Windows signing id: mac-signing-gate if: matrix.platform == 'win' && github.run_attempt == 1 @@ -2080,7 +2078,6 @@ jobs: (needs.orcad-template.result == 'skipped' && needs.cut.outputs.ships_orcad_template == 'false')) # Keep every SignPath workflow job GitHub-hosted; separate release-mac-build allows Blacksmith outside Windows provenance. runs-on: ubuntu-latest - timeout-minutes: 185 permissions: actions: write contents: read @@ -2109,20 +2106,20 @@ jobs: RELEASE_MAC_BUILD_JAVASCRIPT_SOURCE_SHA: ${{ needs.release-javascript.outputs.supported == 'true' && needs.release-javascript.outputs.source_sha || '' }} RELEASE_MAC_BUILD_TAG: ${{ needs.cut.outputs.tag }} RELEASE_MAC_BUILD_WORKFLOW: release-mac-build.yml - RELEASE_MAC_BUILD_OBSERVE_MINUTES: '180' verify-mac-build: needs: - cut - build-mac - # Finish the shared deadline before starting the Windows signing job's six-hour budget. + - build + # Qualify macOS even when a sibling build fails; publication still requires every build. if: >- !cancelled() && needs.cut.result == 'success' && needs.cut.outputs.should_release == 'true' && needs.build-mac.result == 'success' runs-on: ubuntu-latest - timeout-minutes: 215 + timeout-minutes: 95 permissions: actions: read contents: read diff --git a/.github/workflows/release-mac-build.yml b/.github/workflows/release-mac-build.yml index e02ae146e5e..b6a752f6177 100644 --- a/.github/workflows/release-mac-build.yml +++ b/.github/workflows/release-mac-build.yml @@ -49,8 +49,7 @@ jobs: if: ${{ !cancelled() && (!inputs.qualification_only || needs.qualification-template.result == 'success') && github.repository == 'stablyai/orca' }} # Keep Blacksmith mac notarization outside SignPath’s GitHub-hosted Windows provenance. runs-on: blacksmith-6vcpu-macos-15 - # Includes the 180-minute cold runtime build and the existing packaging retry budget. - timeout-minutes: 360 + timeout-minutes: 60 # actions: read fetches the release-cut run's artifacts. permissions: actions: read @@ -96,7 +95,7 @@ jobs: pnpm-lock.yaml mobile/pnpm-lock.yaml - # Cache the patched host Node by exact pin and patch, without restore keys. + # Cache bundled host Node by exact pin, without restore keys. - name: Locate the pinned Node runtime cache shell: bash run: echo "ORCA_NODE_RUNTIME_CACHE_DIR=$HOME/Library/Caches/orca-node-runtime" >>"$GITHUB_ENV" @@ -105,7 +104,7 @@ jobs: uses: actions/cache@v5 with: path: ~/Library/Caches/orca-node-runtime - key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts', 'config/node-runtime-patches/**', 'config/scripts/build-darwin-terminal-node.mjs', 'config/scripts/pinned-node-downloads.mjs', 'config/scripts/update-node-runtime-pin.mjs', 'config/scripts/macos-terminal-host-bundle.cjs') }} + key: orca-node-runtime-mac-${{ hashFiles('src/shared/node-runtime-pin.ts') }} # Cache Electron/builder tool downloads to save ~30–90s. - name: Cache electron-builder downloads @@ -127,15 +126,6 @@ jobs: retry_wait_seconds: 30 command: pnpm install --frozen-lockfile --cpu=current,x64,arm64 - # Why: older refs predate the source build and package the pinned download instead. - - name: Build and verify the patched terminal runtimes - if: hashFiles('config/scripts/build-darwin-terminal-node.mjs') != '' - timeout-minutes: 180 - run: | - pnpm run build:packages - ORCA_BACKGROUND_LAUNCH=1 node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/verify-darwin-terminal-node.mjs --arch=arm64 - ORCA_BACKGROUND_LAUNCH=1 node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/verify-darwin-terminal-node.mjs --arch=x64 - # Build required out/mobile-web using mobile’s React Native/Expo install. - uses: ./.github/actions/install-mobile-dependencies @@ -235,7 +225,7 @@ jobs: timeout_minutes: 45 max_attempts: 3 retry_wait_seconds: 30 - command: if [ ! -f config/scripts/build-darwin-terminal-node.mjs ]; then node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64; fi && node config/scripts/ensure-native-runtime.mjs --runtime=electron && if [ -f config/scripts/build-mac-notarization-overlap.mjs ]; then ORCA_MAC_RELEASE=1 node config/scripts/build-mac-notarization-overlap.mjs --publish "$ORCA_MAC_PUBLISH" --draft; else ORCA_MAC_RELEASE=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish "$ORCA_MAC_PUBLISH" -c.publish.releaseType=draft; fi + command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && if [ -f config/scripts/build-mac-notarization-overlap.mjs ]; then ORCA_MAC_RELEASE=1 node config/scripts/build-mac-notarization-overlap.mjs --publish "$ORCA_MAC_PUBLISH" --draft; else ORCA_MAC_RELEASE=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish "$ORCA_MAC_PUBLISH" -c.publish.releaseType=draft; fi env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} ORCA_REQUIRE_ORCAD_TEMPLATE: '1' diff --git a/config/node-runtime-patches/darwin-worker-wait.json b/config/node-runtime-patches/darwin-worker-wait.json deleted file mode 100644 index 47aedc804bc..00000000000 --- a/config/node-runtime-patches/darwin-worker-wait.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "version": "24.22.1", - "sourceArchive": "node-v24.22.1.tar.gz", - "sourceSha256": "001773d9bd53fbd27a25efd42aa6a9693ebfe500998ca9bec635e9b6ee79bc91", - "recipeVersion": 1, - "workerSourceSha256": "06ab0be8159c22cfb5f4a715d776693825835f1369c7c82d13a22e607947b4fd" -} diff --git a/config/node-runtime-patches/darwin-worker-wait.patch b/config/node-runtime-patches/darwin-worker-wait.patch deleted file mode 100644 index f2a29bd4f60..00000000000 --- a/config/node-runtime-patches/darwin-worker-wait.patch +++ /dev/null @@ -1,46 +0,0 @@ ---- a/deps/uv/src/threadpool.c -+++ b/deps/uv/src/threadpool.c -@@ -26,6 +26,9 @@ - #endif - - #include -+#if defined(__APPLE__) && defined(__MACH__) -+# include -+#endif - - #define MAX_THREADPOOL_SIZE 1024 - -@@ -51,6 +54,24 @@ - } - - -+static void wait_for_work(void) { -+#if defined(__APPLE__) && defined(__MACH__) -+ /* Bound idle waits so Darwin psynch lost wakeups can recover. */ -+ const struct timespec timeout = { 30, 0 }; -+ int r; -+ -+ errno = 0; -+ r = pthread_cond_timedwait_relative_np(&cond, &mutex, &timeout); -+ /* Preserve uv_cond_wait's EINVAL/EBUSY workaround for Darwin. */ -+ if (r == 0 || r == ETIMEDOUT || (r == EINVAL && errno == EBUSY)) -+ return; -+ abort(); -+#else -+ uv_cond_wait(&cond, &mutex); -+#endif -+} -+ -+ - /* To avoid deadlock with uv_cancel() it's crucial that the worker - * never holds the global mutex and the loop-local mutex at the same time. - */ -@@ -74,7 +95,7 @@ - uv__queue_next(&run_slow_work_message) == &wq && - slow_io_work_running >= slow_work_thread_threshold())) { - idle_threads += 1; -- uv_cond_wait(&cond, &mutex); -+ wait_for_work(); - idle_threads -= 1; - } - diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 964dcaf9389..f4e7f8fa66b 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -484,79 +484,6 @@ ], "demotionRule": "Keep experimental or demote if recovery signals an unverified owner, treats loss of contact as exit, reports success before descendant exit, adds background scanning, or lets a recovery failure block the current-daemon restart." }, - { - "id": "terminal-runtime.darwin-worker-wakeup", - "title": "Idle Darwin workers refresh their waits and continue delivering asynchronous terminal work", - "maturity": "experimental", - "protection": "partial", - "owner": "terminal-runtime", - "layer": "native-runtime", - "surfaces": ["packaged macOS terminal helper", "terminal input", "terminal creation"], - "platforms": ["macos"], - "providers": ["daemon", "local", "ssh", "remote-runtime", "mobile-relay"], - "coveredPlatforms": ["macos"], - "coveredProviders": ["daemon"], - "coverageNotes": "The repository arm64 build and x64 build under Rosetta passed real PTY input, filesystem/DNS/crypto and idle-wait generation checks; physical ARM and Intel CI passed on Node 24.21.0. Node 24.22.1 arm64 passed the same native oracle locally; its official unpatched runtime failed idle refresh. Fresh physical ARM/Intel CI remains required. An isolated signed installed helper with patched Node also served a PTY. A complete package build and older macOS remain unverified. Standalone/SSH servers and the Electron fallback do not receive the patch; mobile/relay inherit only their execution host runtime.", - "motivatingLinks": [ - "docs/bug-reproductions/darwin-worker-wakeup.md", - "https://github.com/libuv/libuv/pull/5351" - ], - "invariant": "A packaged Darwin terminal worker must not remain indefinitely parked on a stale condition generation while asynchronous work is queued; process and session ownership must remain unchanged.", - "oracle": "Four real idle workers each advance their low 32-bit psynch wait generation within the 31-second observation interval, with at most sixteen generation steps, and real PTY input plus filesystem/DNS/crypto work completes. The upstream half-range witness stalls unpatched workers with queued jobs and completes with the patch.", - "commands": ["ORCA_BACKGROUND_LAUNCH=1 node config/scripts/verify-darwin-terminal-node.mjs"], - "testFiles": ["config/scripts/verify-darwin-terminal-node.mjs"], - "assertionRefs": [ - { - "file": "config/scripts/verify-darwin-terminal-node.mjs", - "assertions": ["Runtime proof failed"] - } - ], - "evidenceRuns": [ - { - "date": "2026-10-23", - "runner": "local", - "platform": "macos", - "command": "ORCA_BACKGROUND_LAUNCH=1 node config/scripts/verify-darwin-terminal-node.mjs", - "result": "passed", - "durationSeconds": 2231, - "summary": "Cold source build of pinned Node 24.22.1 arm64 passed architecture/minimum-OS checks, real PTY input, fs/DNS/crypto and all four bounded idle-wait generation checks. Official Node 24.22.1 failed the same idle-refresh assertion after passing the initial PTY and I/O checks. Timing includes cold compilation and source cleanup." - }, - { - "date": "2026-10-15", - "runner": "manual", - "platform": "macos", - "command": "ORCA_BACKGROUND_LAUNCH=1 node config/scripts/verify-darwin-terminal-node.mjs", - "result": "passed", - "durationSeconds": 32, - "summary": "Repository-built Node 24.21.0 arm64 passed real PTY input, fs/DNS/crypto and all four bounded idle-generation checks. Cached verifier execution took about 32 seconds; this is not full package-build proof." - } - ], - "runtimeBudget": { - "p95Seconds": 10800, - "scope": "Cold pinned Node source compilation dominates; Intel exceeded the initial 90-minute job allowance, now 180 minutes. Cached native runtime probe takes about 32 seconds. No CI soak yet." - }, - "performanceBudget": { - "required": true, - "evidence": "One relative timeout per idle worker per 30 seconds, at most eight wakes/minute for the default pool; immediate signals remain. Generation advancement is bounded by the native probe. No application polling or session scans added." - }, - "flakeHistory": { - "status": "not-started", - "evidence": "Focused native and manual Node checks passed on arm64; both physical architectures passed on Node 24.21.0; the updated recipe and repeated CI soak remain pending." - }, - "redGreenEvidence": { - "status": "complete", - "evidence": "Upstream real-worker half-range witness: baseline four idle workers with zero jobs/callbacks and queued work; patch four jobs/callbacks and empty queue. An ordinary sequential workload also reproduced the exact half-range signature, zero fresh stat callbacks and no real PTY echo after 15 seconds. The patched runtime completed 30 million sequential jobs, a fresh stat and real PTY echo. Idle-generation regression fails without the patch and passes with it." - }, - "promotionCriteria": [ - "Collect physical arm64/Intel CI, signed helper launch and older macOS evidence, then repeated soak before promotion." - ], - "knownGaps": [ - "Ordinary sequential jobs reproduced half-range waiter starvation and PTY input loss; scheduling is intermittent and the exact Orca job-producer sequence remains unknown.", - "No older macOS, full package build, launchd/TCC attribution or battery measurement yet.", - "Existing daemons require deliberate restart; standalone/SSH macOS servers and Electron fallback remain unpatched." - ], - "demotionRule": "Keep experimental until native runtime and signed packaging evidence is stable; investigate failures without weakening generation bounds or silently using an official unpatched binary." - }, { "id": "agent-session.delayed-terminal-launch", "title": "Delayed terminal launches retain their harness preparation and recover process discovery", diff --git a/config/scripts/__fixtures__/darwin-terminal-worker-wait.c b/config/scripts/__fixtures__/darwin-terminal-worker-wait.c deleted file mode 100644 index 449489369f1..00000000000 --- a/config/scripts/__fixtures__/darwin-terminal-worker-wait.c +++ /dev/null @@ -1,128 +0,0 @@ -#include -#include -#include -#include "darwin-threadpool-wait.h" - -static uv_work_t requests[4]; -static uv_thread_t workers[4]; -static uv_sem_t arrived; -static uv_sem_t release_workers; -static unsigned callbacks; -static int started; -static uv_work_t stress_request; -static uv_loop_t* stress_loop; -static unsigned stress_completed; -static const unsigned stress_target = 30000000; - -static void stress_work(uv_work_t* request) { - (void) request; -} - -static void stress_done(uv_work_t* request, int status) { - if (status != 0) - abort(); - if (++stress_completed < stress_target && - uv_queue_work(stress_loop, request, stress_work, stress_done) != 0) - abort(); -} - -static napi_value start_stress(napi_env env, napi_callback_info info) { - napi_value result; - (void) info; - if (stress_loop != NULL || - napi_get_uv_event_loop(env, &stress_loop) != napi_ok || - uv_queue_work(stress_loop, &stress_request, stress_work, stress_done) != 0) - abort(); - napi_get_undefined(env, &result); - return result; -} - -static napi_value stress_progress(napi_env env, napi_callback_info info) { - napi_value result; - (void) info; - napi_create_uint32(env, stress_completed, &result); - return result; -} - -static void enroll_worker(uv_work_t* request) { - workers[request - requests] = uv_thread_self(); - uv_sem_post(&arrived); - uv_sem_wait(&release_workers); -} - -static void complete_work(uv_work_t* request, int status) { - (void) request; - if (status != 0) - abort(); - if (++callbacks == 4) { - uv_sem_destroy(&arrived); - uv_sem_destroy(&release_workers); - } -} - -static napi_value enroll(napi_env env, napi_callback_info info) { - uv_loop_t* loop; - napi_value result; - unsigned i; - (void) info; - if (started) { - napi_throw_error(env, NULL, "Workers already enrolled"); - return NULL; - } - started = 1; - if (napi_get_uv_event_loop(env, &loop) != napi_ok || - uv_sem_init(&arrived, 0) != 0 || - uv_sem_init(&release_workers, 0) != 0) - abort(); - for (i = 0; i < 4; i++) - if (uv_queue_work(loop, requests + i, enroll_worker, complete_work) != 0) - abort(); - for (i = 0; i < 4; i++) - uv_sem_wait(&arrived); - for (i = 0; i < 4; i++) - uv_sem_post(&release_workers); - napi_get_undefined(env, &result); - return result; -} - -static napi_value snapshot(napi_env env, napi_callback_info info) { - char output[2048]; - uintptr_t condition; - uint64_t generation; - napi_value result; - unsigned i; - int at; - int parked; - (void) info; - if (!started) { - napi_throw_error(env, NULL, "Enroll workers first"); - return NULL; - } - at = snprintf(output, sizeof(output), - "{\"callbacks\":%u,\"waits\":[", callbacks); - for (i = 0; i < 4; i++) { - generation = 0; - parked = read_wait_generation(workers[i], &condition, &generation); - at += snprintf(output + at, sizeof(output) - at, - "%s{\"parked\":%d,\"generation\":\"%llx\"}", - i ? "," : "", parked, (unsigned long long) generation); - } - snprintf(output + at, sizeof(output) - at, "]}"); - napi_create_string_utf8(env, output, NAPI_AUTO_LENGTH, &result); - return result; -} - -NAPI_MODULE_INIT() { - napi_value function; - napi_create_function(env, "enroll", NAPI_AUTO_LENGTH, enroll, NULL, &function); - napi_set_named_property(env, exports, "enroll", function); - napi_create_function(env, "snapshot", NAPI_AUTO_LENGTH, snapshot, NULL, &function); - napi_set_named_property(env, exports, "snapshot", function); - napi_create_function(env, "startStress", NAPI_AUTO_LENGTH, - start_stress, NULL, &function); - napi_set_named_property(env, exports, "startStress", function); - napi_create_function(env, "stressProgress", NAPI_AUTO_LENGTH, - stress_progress, NULL, &function); - napi_set_named_property(env, exports, "stressProgress", function); - return exports; -} diff --git a/config/scripts/__fixtures__/darwin-threadpool-wait.h b/config/scripts/__fixtures__/darwin-threadpool-wait.h deleted file mode 100644 index e7abe2c2260..00000000000 --- a/config/scripts/__fixtures__/darwin-threadpool-wait.h +++ /dev/null @@ -1,68 +0,0 @@ -/* Copyright Joyent, Inc. and other Node contributors. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -#ifndef THREADPOOL_DARWIN_WAIT_H_ -#define THREADPOOL_DARWIN_WAIT_H_ - -#include "uv.h" -#include -#include -#include -#include -#include -#include - -static int read_wait_generation(uv_thread_t worker, - uintptr_t* condition, - uint64_t* generation) { - mach_msg_type_number_t count; - kern_return_t result; - uintptr_t pc; - Dl_info symbol; -# if defined(__aarch64__) - arm_thread_state64_t state = {0}; - count = ARM_THREAD_STATE64_COUNT; - result = thread_get_state(pthread_mach_thread_np(worker), - ARM_THREAD_STATE64, - (thread_state_t) &state, - &count); - pc = arm_thread_state64_get_pc(state); - *generation = state.__x[1]; - *condition = state.__x[0]; -# else - x86_thread_state64_t state = {0}; - count = x86_THREAD_STATE64_COUNT; - result = thread_get_state(pthread_mach_thread_np(worker), - x86_THREAD_STATE64, - (thread_state_t) &state, - &count); - pc = state.__rip; - *generation = state.__rsi; - *condition = state.__rdi; -# endif - if (result != KERN_SUCCESS) - return 0; - if (!dladdr((void*) pc, &symbol) || symbol.dli_sname == NULL) - return 0; - return strcmp(symbol.dli_sname, "__psynch_cvwait") == 0; -} - -#endif diff --git a/config/scripts/adhoc-windows-parallelism.test.mjs b/config/scripts/adhoc-windows-parallelism.test.mjs index a797d42c52d..a1096645624 100644 --- a/config/scripts/adhoc-windows-parallelism.test.mjs +++ b/config/scripts/adhoc-windows-parallelism.test.mjs @@ -117,20 +117,15 @@ describe('adhoc Windows compilation overlaps Mac without publishing early', () = }) it.each(['success', 'failure', 'cancelled', 'skipped', ''])( - 'publishes only after both Windows compilation and macOS publication succeed: %s', + 'prefetches only after Windows compilation succeeds: %s', (result) => { const run = compileAggregateExpression(publish.if) const needs = { 'orcad-template-support': { result: 'success' }, - 'build-adhoc-win': { result: 'success' }, - 'build-adhoc-mac': { result: 'success' } + 'build-adhoc-win': { result: 'success' } } - expect(publish.needs).toEqual([ - 'orcad-template-support', - 'build-adhoc-win', - 'build-adhoc-mac' - ]) - expect(publish.with.wait_for_mac_job).toBeUndefined() + expect(publish.needs).toEqual(['orcad-template-support', 'build-adhoc-win']) + expect(publish.with.wait_for_mac_job).toBe('build-adhoc-mac') expect(run({ needs }).coerceString()).toBe('true') expect(run({ needs }, true).coerceString()).toBe('false') for (const job of publish.needs) { diff --git a/config/scripts/build-darwin-terminal-node.mjs b/config/scripts/build-darwin-terminal-node.mjs deleted file mode 100644 index 5bfe022e18a..00000000000 --- a/config/scripts/build-darwin-terminal-node.mjs +++ /dev/null @@ -1,222 +0,0 @@ -import { createHash } from 'node:crypto' -import { - chmodSync, - copyFileSync, - mkdirSync, - mkdtempSync, - readFileSync, - renameSync, - rmSync, - writeFileSync -} from 'node:fs' -import { createRequire } from 'node:module' -import { tmpdir } from 'node:os' -import { join, resolve } from 'node:path' -import { lock } from 'proper-lockfile' -import { runProcess } from '@orca/process-host' -import { - NODE_RUNTIME_DARWIN_MINIMUM_OS, - NODE_RUNTIME_PIN -} from '../../src/shared/node-runtime-pin.ts' -import { fetchPinnedFile, pinnedNodeCacheDir } from './pinned-node-downloads.mjs' -import { sha256File } from './update-node-runtime-pin.mjs' -import { getTarProgram } from './zip-extractor-command.mjs' -import { isDirectInvocation } from './script-entry-detection.mjs' - -const root = resolve(import.meta.dirname, '../..') -const patchPath = join(root, 'config/node-runtime-patches/darwin-worker-wait.patch') -const recipe = JSON.parse( - readFileSync(join(root, 'config/node-runtime-patches/darwin-worker-wait.json'), 'utf8') -) -const require = createRequire(import.meta.url) -const { lipoArchsToNodeArchs, parseBuildVersionMinos } = require('./macos-terminal-host-bundle.cjs') -const builds = new Map() - -export function darwinTerminalNodeInputs(arch, version = NODE_RUNTIME_PIN.version) { - if (arch !== 'arm64' && arch !== 'x64') { - throw new Error(`Unsupported macOS terminal Node architecture: ${arch}`) - } - if (version !== recipe.version) { - throw new Error(`Review the Darwin worker-wait patch before moving Node to ${version}`) - } - return { - ...recipe, - arch, - ...(arch !== process.arch - ? { - crossBuildMode: arch === 'x64' ? 'target-tools-under-rosetta' : 'separate-host-tools' - } - : {}), - patchSha256: createHash('sha256').update(readFileSync(patchPath)).digest('hex'), - buildScriptsSha256: Object.fromEntries( - [ - 'build-darwin-terminal-node.mjs', - 'pinned-node-downloads.mjs', - 'update-node-runtime-pin.mjs', - 'macos-terminal-host-bundle.cjs' - ].map((file) => [ - file, - createHash('sha256') - .update(readFileSync(join(import.meta.dirname, file))) - .digest('hex') - ]) - ), - minimumOs: NODE_RUNTIME_DARWIN_MINIMUM_OS - } -} - -export async function readBuiltDarwinTerminalNode(directory, inputs) { - try { - const receipt = JSON.parse(readFileSync(join(directory, 'receipt.json'), 'utf8')) - if (JSON.stringify(receipt.inputs) !== JSON.stringify(inputs)) { - return null - } - const executable = join(directory, 'node') - return (await sha256File(executable)) === receipt.executableSha256 ? executable : null - } catch { - return null - } -} - -async function runBuildCommand(program, args, options = {}) { - const result = await runProcess( - { - program, - args, - timeoutMs: 180 * 60_000, - maxOutputBytes: 1024 * 1024, - ...options - }, - 'tail' - ) - if (result.code !== 0) { - throw new Error(`${program} failed (${result.code}):\n${result.stderr}\n${result.stdout}`) - } - return result.stdout -} - -async function compileDarwinTerminalNode(directory, inputs, cacheDir, scratch) { - const archivePath = join(cacheDir, inputs.sourceArchive) - const releaseArchive = await lock(archivePath, { - realpath: false, - stale: 60_000, - retries: { retries: 2160, minTimeout: 5_000, maxTimeout: 5_000 } - }) - let archive - try { - archive = await fetchPinnedFile({ - url: `https://nodejs.org/dist/v${inputs.version}/${inputs.sourceArchive}`, - destination: archivePath, - sha256: inputs.sourceSha256 - }) - } finally { - await releaseArchive() - } - const source = join(scratch, `node-v${inputs.version}`) - await runBuildCommand(getTarProgram(), [ - '-xzf', - archive, - '-C', - scratch, - `node-v${inputs.version}` - ]) - await runBuildCommand('patch', ['--batch', '--forward', '--fuzz=0', '-p1', '-i', patchPath], { - cwd: source - }) - if ((await sha256File(join(source, 'deps/uv/src/threadpool.c'))) !== inputs.workerSourceSha256) { - throw new Error('The Node worker source does not match the reviewed patch') - } - const env = { - ...process.env, - ORCA_BACKGROUND_LAUNCH: '1', - MACOSX_DEPLOYMENT_TARGET: inputs.minimumOs - } - const args = ['--without-npm', '--without-corepack', `--dest-cpu=${inputs.arch}`] - if (process.arch !== inputs.arch) { - const compilerArch = inputs.arch === 'x64' ? 'x86_64' : 'arm64' - env.CC = `clang -arch ${compilerArch}` - env.CXX = `clang++ -arch ${compilerArch}` - if (inputs.arch === 'x64') { - await runBuildCommand('/usr/bin/arch', ['-x86_64', '/usr/bin/true']) - env.CC_host = env.CC - env.CXX_host = env.CXX - args.push('--no-cross-compiling') - } else { - env.CC_host = 'clang -arch x86_64' - env.CXX_host = 'clang++ -arch x86_64' - args.push('--cross-compiling') - } - } - await runBuildCommand(join(source, 'configure'), args, { cwd: source, env }) - console.log(`[terminal-node] Building patched Node ${inputs.version} (${inputs.arch})`) - await runBuildCommand('make', ['-j4', 'node'], { cwd: source, env }) - const executable = join(source, 'out/Release/node') - const architectures = lipoArchsToNodeArchs(await runBuildCommand('lipo', ['-archs', executable])) - const minimumOs = parseBuildVersionMinos(await runBuildCommand('otool', ['-l', executable])) - if ( - architectures.length !== 1 || - architectures[0] !== inputs.arch || - minimumOs !== inputs.minimumOs - ) { - throw new Error('Patched terminal Node has the wrong architecture or macOS floor') - } - const staged = join(directory, `node.${process.pid}.partial`) - copyFileSync(executable, staged) - chmodSync(staged, 0o755) - const executableSha256 = await sha256File(staged) - renameSync(staged, join(directory, 'node')) - const receipt = join(directory, `receipt.${process.pid}.partial`) - writeFileSync(receipt, `${JSON.stringify({ inputs, executableSha256 }, null, 2)}\n`) - renameSync(receipt, join(directory, 'receipt.json')) - return join(directory, 'node') -} - -export function ensureDarwinTerminalNode({ arch, cacheDir = pinnedNodeCacheDir() }) { - if (process.platform !== 'darwin') { - throw new Error('Build the patched macOS terminal runtime on a macOS host') - } - const inputs = darwinTerminalNodeInputs(arch) - const key = createHash('sha256').update(JSON.stringify(inputs)).digest('hex') - const directory = join(cacheDir, 'darwin-terminal-node', key) - const pending = builds.get(directory) - if (pending) { - return pending - } - const build = (async () => { - mkdirSync(directory, { recursive: true }) - const cached = await readBuiltDarwinTerminalNode(directory, inputs) - if (cached) { - return cached - } - const release = await lock(directory, { - realpath: false, - stale: 60_000, - retries: { retries: 2160, minTimeout: 5_000, maxTimeout: 5_000 } - }) - let scratch - try { - const cached = await readBuiltDarwinTerminalNode(directory, inputs) - if (cached) { - return cached - } - scratch = mkdtempSync(join(tmpdir(), 'orca-darwin-terminal-node-build-')) - return await compileDarwinTerminalNode(directory, inputs, cacheDir, scratch) - } finally { - try { - // Release before synchronous cleanup can starve the lease heartbeat. - await release() - } finally { - if (scratch) { - rmSync(scratch, { recursive: true, force: true }) - } - } - } - })() - builds.set(directory, build) - void build.finally(() => builds.delete(directory)).catch(() => {}) - return build -} - -if (isDirectInvocation(import.meta.url, process.argv[1])) { - console.log(await ensureDarwinTerminalNode({ arch: process.argv[2] ?? process.arch })) -} diff --git a/config/scripts/build-darwin-terminal-node.test.mjs b/config/scripts/build-darwin-terminal-node.test.mjs deleted file mode 100644 index da72c569c2a..00000000000 --- a/config/scripts/build-darwin-terminal-node.test.mjs +++ /dev/null @@ -1,92 +0,0 @@ -import { createHash } from 'node:crypto' -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, expect, it } from 'vitest' -import { - darwinTerminalNodeInputs, - readBuiltDarwinTerminalNode -} from './build-darwin-terminal-node.mjs' - -const directories = [] -afterEach(() => { - for (const directory of directories.splice(0)) { - rmSync(directory, { recursive: true, force: true }) - } -}) - -function artifact() { - const directory = mkdtempSync(join(tmpdir(), 'darwin-terminal-node-cache-')) - directories.push(directory) - const inputs = darwinTerminalNodeInputs('arm64') - const bytes = Buffer.from('built Node artifact') - const executable = join(directory, 'node') - writeFileSync(executable, bytes) - writeFileSync( - join(directory, 'receipt.json'), - JSON.stringify({ - inputs, - executableSha256: createHash('sha256').update(bytes).digest('hex') - }) - ) - return { directory, inputs, executable } -} - -it('requires review when the Node pin moves instead of silently dropping the fix', () => { - expect(() => darwinTerminalNodeInputs('arm64', '99.0.0')).toThrow('Review the Darwin') - expect(() => darwinTerminalNodeInputs('riscv64')).toThrow('Unsupported') -}) - -it('separates architecture and includes the exact patch and source identity', () => { - const arm = darwinTerminalNodeInputs('arm64') - const x64 = darwinTerminalNodeInputs('x64') - expect(arm.arch).not.toBe(x64.arch) - expect(arm.sourceSha256).toMatch(/^[0-9a-f]{64}$/) - expect(arm.workerSourceSha256).toMatch(/^[0-9a-f]{64}$/) - expect(arm.patchSha256).toBe( - createHash('sha256') - .update( - readFileSync(join(import.meta.dirname, '../node-runtime-patches/darwin-worker-wait.patch')) - ) - .digest('hex') - ) -}) - -it('reuses a built artifact only when its bytes and every build input match', async () => { - const { directory, inputs, executable } = artifact() - expect(await readBuiltDarwinTerminalNode(directory, inputs)).toBe(executable) - expect(await readBuiltDarwinTerminalNode(directory, { ...inputs, arch: 'x64' })).toBeNull() - expect( - await readBuiltDarwinTerminalNode(directory, { ...inputs, patchSha256: 'other' }) - ).toBeNull() -}) - -it('rejects a tampered cached executable', async () => { - const { directory, inputs, executable } = artifact() - writeFileSync(executable, 'different bytes') - expect(await readBuiltDarwinTerminalNode(directory, inputs)).toBeNull() -}) - -it('rejects unchanged executable bytes built with an older build script', async () => { - const { directory, inputs, executable } = artifact() - const receiptPath = join(directory, 'receipt.json') - const receipt = JSON.parse(readFileSync(receiptPath, 'utf8')) - expect(inputs.buildScriptsSha256['build-darwin-terminal-node.mjs']).toMatch(/^[0-9a-f]{64}$/) - expect(await readBuiltDarwinTerminalNode(directory, inputs)).toBe(executable) - receipt.inputs.buildScriptsSha256['build-darwin-terminal-node.mjs'] = 'older build recipe' - writeFileSync(receiptPath, JSON.stringify(receipt)) - expect(await readBuiltDarwinTerminalNode(directory, inputs)).toBeNull() - delete receipt.inputs.buildScriptsSha256 - writeFileSync(receiptPath, JSON.stringify(receipt)) - expect(await readBuiltDarwinTerminalNode(directory, inputs)).toBeNull() -}) - -it('rejects a partial build, invalid receipt, or non-file executable', async () => { - const { directory, inputs, executable } = artifact() - rmSync(executable) - expect(await readBuiltDarwinTerminalNode(directory, inputs)).toBeNull() - mkdirSync(executable) - expect(await readBuiltDarwinTerminalNode(directory, inputs)).toBeNull() - writeFileSync(join(directory, 'receipt.json'), '{') - expect(await readBuiltDarwinTerminalNode(directory, inputs)).toBeNull() -}) diff --git a/config/scripts/darwin-terminal-node-probe.cjs b/config/scripts/darwin-terminal-node-probe.cjs deleted file mode 100644 index ec3ad408b94..00000000000 --- a/config/scripts/darwin-terminal-node-probe.cjs +++ /dev/null @@ -1,80 +0,0 @@ -const assert = require('node:assert/strict') -const { setTimeout: delay } = require('node:timers/promises') -const fs = require('node:fs/promises') -const crypto = require('node:crypto') -const dns = require('node:dns/promises') -const pty = require(process.argv[3]) -const canary = require(process.argv[2]) - -const deadline = setTimeout(() => process.exit(2), 45_000) -async function checkIo() { - await Promise.all([ - fs.stat(process.execPath), - dns.lookup('localhost'), - new Promise((resolve, reject) => { - crypto.randomBytes(32, (error) => (error ? reject(error) : resolve())) - }) - ]) -} - -async function checkPty() { - const terminal = pty.spawn('/bin/sh', ['-c', 'read line; printf "ECHO:%s\\n" "$line"'], { - cwd: process.argv[4], - env: process.env - }) - let output = '' - terminal.onData((chunk) => { - output += chunk - }) - const exited = new Promise((resolve, reject) => { - terminal.onExit(({ exitCode }) => - exitCode === 0 ? resolve() : reject(new Error('PTY failed')) - ) - }) - terminal.write('ORCA_WORKER_WAKEUP_TEST\r') - await exited - assert.ok(output.includes('ECHO:ORCA_WORKER_WAKEUP_TEST')) -} - -async function main() { - assert.equal(process.version, process.argv[5]) - await checkIo() - await checkPty() - canary.enroll() - let before - for (let i = 0; i < 100; i++) { - await delay(10) - before = JSON.parse(canary.snapshot()) - if (before.callbacks === 4 && before.waits.every((wait) => wait.parked)) { - break - } - } - assert.equal(before.callbacks, 4) - assert.ok(before.waits.every((wait) => wait.parked)) - await delay(31_000) - const after = JSON.parse(canary.snapshot()) - assert.ok(after.waits.every((wait) => wait.parked)) - for (let i = 0; i < 4; i++) { - const previous = BigInt(`0x${before.waits[i].generation}`) - const current = BigInt(`0x${after.waits[i].generation}`) - const advances = BigInt.asUintN(32, current - previous) - assert.ok(advances > 0n) - assert.ok(advances <= 16n * 256n) - } - await checkIo() - clearTimeout(deadline) - console.log( - JSON.stringify({ - node: process.version, - uv: process.versions.uv, - arch: process.arch, - fsDnsCrypto: 'passed', - ptyInput: 'passed', - idleWaitRefresh: 'passed' - }) - ) -} -main().catch((error) => { - console.error(error) - process.exit(1) -}) diff --git a/config/scripts/darwin-terminal-node-workflow.test.mjs b/config/scripts/darwin-terminal-node-workflow.test.mjs deleted file mode 100644 index 0d51d3c6eb1..00000000000 --- a/config/scripts/darwin-terminal-node-workflow.test.mjs +++ /dev/null @@ -1,147 +0,0 @@ -import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { dirname, join } from 'node:path' -import { runProcess } from '@orca/process-host' -import { expect, it } from 'vitest' -import { parse } from 'yaml' -import { readDarwinVerificationArch } from './verify-darwin-terminal-node.mjs' -import { classifyPrJobs } from './pr-code-change-scope.mjs' - -function workflow(name) { - return parse( - readFileSync(join(import.meta.dirname, '../../.github/workflows', `${name}.yml`), 'utf8') - ) -} - -it.each([ - 'config/node-runtime-patches/darwin-worker-wait.patch', - 'config/scripts/build-darwin-terminal-node.mjs', - 'config/scripts/pr-darwin-terminal-node-route.mjs', - 'config/scripts/verify-darwin-terminal-node.mjs', - 'config/scripts/darwin-terminal-node-probe.cjs', - 'config/scripts/__fixtures__/darwin-threadpool-wait.h', - 'config/scripts/macos-terminal-host-bundle.cjs', - 'config/scripts/pinned-node-downloads.mjs', - 'config/scripts/update-node-runtime-pin.mjs', - 'config/scripts/orcad-prebuild-node-gyp.mjs', - 'config/scripts/orcad-prebuild-slot-contents.mjs', - 'src/shared/node-runtime-pin.ts' -])('qualifies the actual macOS runtime when %s changes', (file) => { - expect(classifyPrJobs([file]).darwin_terminal_node).toBe(true) -}) - -it('does not source-build Node for unrelated renderer changes', () => { - expect(classifyPrJobs(['src/renderer/src/App.tsx']).darwin_terminal_node).toBe(false) -}) - -it.each(['adhoc', 'daily', 'hourly', 'release'])( - 'builds both %s runtime architectures after installing build dependencies', - (channel) => { - const jobs = Object.values(workflow(`${channel}-mac-build`).jobs) - const steps = jobs.find((job) => - job.steps?.some((step) => step.name === 'Build and verify the patched terminal runtimes') - ).steps - const build = steps.findIndex( - (step) => step.name === 'Build and verify the patched terminal runtimes' - ) - expect(build).toBeGreaterThan(steps.findIndex((step) => step.name === 'Install dependencies')) - expect(steps[build].run).toContain('pnpm run build:packages') - expect(steps[build].run).toContain('verify-darwin-terminal-node.mjs --arch=arm64') - expect(steps[build].run).toContain('verify-darwin-terminal-node.mjs --arch=x64') - const packaging = steps.findIndex((step) => step.with?.command?.includes('--runtime=electron')) - expect(build).toBeLessThan(packaging) - const cache = steps.find((step) => step.with?.key?.startsWith('orca-node-runtime-mac-')) - for (const dependency of [ - 'pinned-node-downloads.mjs', - 'update-node-runtime-pin.mjs', - 'macos-terminal-host-bundle.cjs' - ]) { - expect(cache.with.key).toContain(dependency) - } - if (channel === 'daily') { - expect(steps[build].if).toBe("steps.freshness.outputs.should_build == 'true'") - } - if (channel === 'adhoc' || channel === 'release') { - // Older refs and tags have no source-build recipe and must keep the pinned download. - expect(steps[build].if).toBe( - "hashFiles('config/scripts/build-darwin-terminal-node.mjs') != ''" - ) - expect(steps[packaging].with.command).toMatch( - /^if \[ ! -f config\/scripts\/build-darwin-terminal-node\.mjs \]; then node .*pinned-node-downloads\.mjs darwin-arm64 darwin-x64; fi && / - ) - } - } -) - -it.skipIf(process.platform === 'win32').each([ - ['success', 'true', 0], - ['failure', 'true', 1], - ['cancelled', 'true', 1], - ['skipped', 'true', 1], - ['', 'true', 1], - ['skipped', 'false', 0] -])('aggregate handles runtime result %s (required=%s)', async (result, required, code) => { - const gate = workflow('pr').jobs.verify.steps.find( - (step) => step.name === 'Require successful checks' - ) - const env = Object.fromEntries( - Object.keys(gate.env).map((key) => [key, key.endsWith('_SHOULD_RUN') ? 'true' : 'success']) - ) - const checked = await runProcess({ - program: 'bash', - args: ['-c', gate.run], - env: { - ...process.env, - ...env, - SHOULD_RUN: 'true', - DARWIN_TERMINAL_NODE: result, - DARWIN_TERMINAL_NODE_SHOULD_RUN: required - }, - timeoutMs: 10_000 - }) - expect(checked.code).toBe(code) -}) - -it('classifies runtime changes from the dependency-free CI sparse checkout', async () => { - const root = join(import.meta.dirname, '../..') - const directory = mkdtempSync(join(tmpdir(), 'orca-runtime-sparse-checkout-')) - try { - const checkout = workflow('pr').jobs.code_paths.steps.find( - (step) => step.with?.['sparse-checkout'] - ) - for (const entry of checkout.with['sparse-checkout'].trim().split('\n')) { - const path = entry.trim().replace(/^\//, '') - const source = join(root, path) - if (statSync(source).isFile()) { - const destination = join(directory, path) - mkdirSync(dirname(destination), { recursive: true }) - copyFileSync(source, destination) - } - } - const result = await runProcess({ - program: process.execPath, - args: ['config/scripts/pr-code-change-scope.mjs'], - cwd: directory, - input: 'config/node-runtime-patches/darwin-worker-wait.patch\n', - timeoutMs: 10_000 - }) - expect(result.code, result.stderr).toBe(0) - expect(result.stdout).toContain('darwin_terminal_node=true') - expect(result.stdout).toContain('package_windows=false') - } finally { - rmSync(directory, { recursive: true, force: true }) - } -}) - -it('selects the requested runtime architecture and retains the native default', () => { - expect(readDarwinVerificationArch([], 'arm64')).toBe('arm64') - expect(readDarwinVerificationArch(['--arch=x64'], 'arm64')).toBe('x64') - expect(readDarwinVerificationArch(['--arch=arm64'], 'x64')).toBe('arm64') -}) - -it.each([['--arch=riscv64'], ['--arch'], ['--unknown'], ['--arch=x64', '--arch=arm64']])( - 'rejects invalid verifier arguments %j before building a runtime', - (...args) => { - expect(() => readDarwinVerificationArch(args, 'arm64')).toThrow('Usage:') - } -) diff --git a/config/scripts/darwin-terminal-worker-stress.cjs b/config/scripts/darwin-terminal-worker-stress.cjs deleted file mode 100644 index e7a335d5c57..00000000000 --- a/config/scripts/darwin-terminal-worker-stress.cjs +++ /dev/null @@ -1,82 +0,0 @@ -const { setTimeout: delay } = require('node:timers/promises') -const fs = require('node:fs/promises') -const pty = require(process.argv[3]) -const canary = require(process.argv[2]) - -const target = 30_000_000 -const began = Date.now() -const terminal = pty.spawn( - '/bin/sh', - ['-c', 'while IFS= read -r line; do printf "ECHO:%s\\n" "$line"; done'], - { cwd: process.argv[4], env: process.env } -) -let output = '' -terminal.onData((chunk) => { - output = (output + chunk).slice(-4096) -}) -const deadline = setTimeout(() => { - console.error('Stress run exceeded one hour') - terminal.kill() - process.kill(process.pid, 'SIGKILL') -}, 60 * 60_000) - -async function echo(marker, timeoutMs = 10_000) { - terminal.write(`${marker}\r`) - const echoDeadline = Date.now() + timeoutMs - while (Date.now() < echoDeadline) { - if (output.includes(`ECHO:${marker}`)) { - return - } - await delay(10) - } - throw new Error(`No PTY echo for ${marker}`) -} - -async function main() { - await echo('ORCA_BEFORE') - canary.startStress() - let previous = 0 - let lastProgress = Date.now() - let lastPrint = 0 - while (canary.stressProgress() < target) { - await delay(100) - const completed = canary.stressProgress() - if (completed !== previous) { - previous = completed - lastProgress = Date.now() - } - if (Date.now() - lastPrint > 10_000) { - console.log(JSON.stringify({ elapsedMs: Date.now() - began, completed })) - lastPrint = Date.now() - } - if (Date.now() - lastProgress > 3000) { - const marker = `ORCA_AFTER_STALL_${completed}` - await Promise.race([ - Promise.all([ - ...Array.from({ length: 4 }, () => fs.stat(process.execPath)), - echo(marker, 40_000), - (async () => { - while (canary.stressProgress() <= completed) { - await delay(100) - } - })() - ]), - delay(40_000, undefined, { ref: false }).then(() => { - throw new Error(`Workers stranded after ${completed} jobs: I/O recovery timed out`) - }) - ]) - lastProgress = Date.now() - } - } - await fs.stat(process.execPath) - await echo('ORCA_AFTER') - console.log(JSON.stringify({ result: 'passed', arch: process.arch, completed: target })) - clearTimeout(deadline) - terminal.kill() -} -main().catch((error) => { - console.error(error) - terminal.kill() - // Stalled libuv cleanup can block process.exit(). - process.kill(process.pid, 'SIGKILL') -}) diff --git a/config/scripts/macos-terminal-host-bundle.cjs b/config/scripts/macos-terminal-host-bundle.cjs index 64a1902b34e..7339c7e3e52 100644 --- a/config/scripts/macos-terminal-host-bundle.cjs +++ b/config/scripts/macos-terminal-host-bundle.cjs @@ -294,10 +294,10 @@ async function loadNodeRuntimePin() { } async function pinnedDarwinNode(arch) { - const { ensureDarwinTerminalNode } = await import( - pathToFileURL(resolve(__dirname, 'build-darwin-terminal-node.mjs')).href + const { ensurePinnedNodeExecutable } = await import( + pathToFileURL(resolve(__dirname, 'pinned-node-downloads.mjs')).href ) - return ensureDarwinTerminalNode({ arch }) + return ensurePinnedNodeExecutable({ target: `darwin-${arch}` }) } /** diff --git a/config/scripts/mobile-release-check-scope.test.mjs b/config/scripts/mobile-release-check-scope.test.mjs index 6940f17e91e..7195be14d45 100644 --- a/config/scripts/mobile-release-check-scope.test.mjs +++ b/config/scripts/mobile-release-check-scope.test.mjs @@ -145,7 +145,6 @@ function fixture(detectorStep = detector) { for (const file of [ 'package.json', 'config/scripts/pr-code-change-scope.mjs', - 'config/scripts/pr-darwin-terminal-node-route.mjs', 'config/scripts/pr-package-test-paths.mjs', 'config/scripts/pr-layout-oracle-route.mjs', 'config/scripts/mobile-release-check-scope.mjs', diff --git a/config/scripts/package-electron-runtime-contract.test.mjs b/config/scripts/package-electron-runtime-contract.test.mjs index 728d14d9a21..5921f7f6727 100644 --- a/config/scripts/package-electron-runtime-contract.test.mjs +++ b/config/scripts/package-electron-runtime-contract.test.mjs @@ -141,7 +141,7 @@ describe('Electron runtime package contract', () => { expect(releaseWorkflow.jobs['publish-release'].needs).toContain('build-mac') expect(releaseWorkflow.jobs['publish-release'].needs).toContain('verify-mac-build') const macVerification = releaseWorkflow.jobs['verify-mac-build'] - expect(macVerification.needs).toEqual(['cut', 'build-mac']) + expect(macVerification.needs).toEqual(['cut', 'build-mac', 'build']) expect(macVerification.if).toContain('!cancelled()') expect(macVerification.if).not.toContain('needs.build.result') expect(macVerification.permissions).toEqual({ actions: 'read', contents: 'read' }) diff --git a/config/scripts/pr-auxiliary-preflight-admission.test.mjs b/config/scripts/pr-auxiliary-preflight-admission.test.mjs index cd573eacaf8..a0abbcee847 100644 --- a/config/scripts/pr-auxiliary-preflight-admission.test.mjs +++ b/config/scripts/pr-auxiliary-preflight-admission.test.mjs @@ -383,7 +383,6 @@ function createMobileCiScopeRepository( for (const file of [ 'package.json', 'config/scripts/pr-code-change-scope.mjs', - 'config/scripts/pr-darwin-terminal-node-route.mjs', 'config/scripts/pr-package-test-paths.mjs', 'config/scripts/pr-layout-oracle-route.mjs', 'config/scripts/git-pull-request-diff-base.mjs', diff --git a/config/scripts/pr-code-change-scope.mjs b/config/scripts/pr-code-change-scope.mjs index 59f37ef6241..8550dcc6bcc 100644 --- a/config/scripts/pr-code-change-scope.mjs +++ b/config/scripts/pr-code-change-scope.mjs @@ -1,7 +1,6 @@ import { readFileSync } from 'node:fs' import { join } from 'node:path' import { pathToFileURL } from 'node:url' -import { isDarwinTerminalNodeSource } from './pr-darwin-terminal-node-route.mjs' import { isLayoutOracleSource } from './pr-layout-oracle-route.mjs' import { isCloudTestRunnerInput, @@ -38,7 +37,6 @@ export const PR_CHECK_JOBS = [ 'managed_hook_node18', 'package', 'package_windows', - 'darwin_terminal_node', 'layout_oracle', 'dependency_isolation' ] @@ -559,26 +557,10 @@ export function classifyPrJobs( } } -const FILE_ROUTED_JOBS = new Map([ - ['git_compatibility', (file) => matchesPrefix(file, GIT_COMPAT_PREFIXES)], - ['xterm_patch_sync', (file) => matchesPrefix(file, XTERM_PREFIXES)], - ['shell_contracts', (file) => matchesPrefix(file, SHELL_PREFIXES)], - ['orcad_browser', (file) => matchesPrefix(file, ORCAD_BROWSER_PREFIXES)], - ['cross-version-wire', (file) => matchesPrefix(file, CROSS_VERSION_WIRE_PREFIXES)], - ['managed_hook_node18', (file) => matchesPrefix(file, MANAGED_HOOK_PREFIXES)], - ['package', isLinuxPackagePath], - ['package_windows', isWindowsPackagePath], - ['layout_oracle', isLayoutOracleSource], - ['dependency_isolation', (file) => matchesPrefix(file, DEPENDENCY_ISOLATION_PREFIXES)], - ['darwin_terminal_node', isDarwinTerminalNodeSource] -]) - function jobDetector(job) { - const routesFile = FILE_ROUTED_JOBS.get(job) - if (routesFile) { - return (files) => files.some(routesFile) - } switch (job) { + case 'git_compatibility': + return (files) => files.some((file) => matchesPrefix(file, GIT_COMPAT_PREFIXES)) case 'codex_index_heal_contract': return (files) => files.some( @@ -586,9 +568,27 @@ function jobDetector(job) { !/^src\/main\/codex\/codex-app-server-(client|session)\.test\.ts(?!.)/s.test(file) && matchesPrefix(file, CODEX_INDEX_HEAL_CONTRACT_PREFIXES) ) + case 'xterm_patch_sync': + return (files) => files.some((file) => matchesPrefix(file, XTERM_PREFIXES)) + case 'shell_contracts': + return (files) => files.some((file) => matchesPrefix(file, SHELL_PREFIXES)) + case 'orcad_browser': + return (files) => files.some((file) => matchesPrefix(file, ORCAD_BROWSER_PREFIXES)) // Handle inside map too: default detector otherwise runs this job on every desktop-relevant PR. case 'mobile_web_app': return changesMobileWebApp + case 'cross-version-wire': + return (files) => files.some((file) => matchesPrefix(file, CROSS_VERSION_WIRE_PREFIXES)) + case 'managed_hook_node18': + return (files) => files.some((file) => matchesPrefix(file, MANAGED_HOOK_PREFIXES)) + case 'package': + return (files) => files.some(isLinuxPackagePath) + case 'package_windows': + return (files) => files.some(isWindowsPackagePath) + case 'layout_oracle': + return (files) => files.some(isLayoutOracleSource) + case 'dependency_isolation': + return (files) => files.some((file) => matchesPrefix(file, DEPENDENCY_ISOLATION_PREFIXES)) default: return () => true } diff --git a/config/scripts/pr-darwin-terminal-node-route.mjs b/config/scripts/pr-darwin-terminal-node-route.mjs deleted file mode 100644 index c290648e9b1..00000000000 --- a/config/scripts/pr-darwin-terminal-node-route.mjs +++ /dev/null @@ -1,20 +0,0 @@ -const DARWIN_TERMINAL_NODE_PREFIXES = [ - 'config/node-runtime-patches/', - 'config/scripts/build-darwin-terminal-node', - 'config/scripts/verify-darwin-terminal-node', - 'config/scripts/darwin-terminal-node-', - 'config/scripts/darwin-terminal-worker-stress', - 'config/scripts/pr-darwin-terminal-node-route', - 'config/scripts/__fixtures__/darwin-terminal-worker-wait', - 'config/scripts/__fixtures__/darwin-threadpool-wait', - 'config/scripts/macos-terminal-host-bundle', - 'config/scripts/pinned-node-downloads', - 'config/scripts/update-node-runtime-pin', - 'config/scripts/orcad-prebuild-node-gyp', - 'config/scripts/orcad-prebuild-slot-contents', - 'src/shared/node-runtime-pin.ts' -] - -export function isDarwinTerminalNodeSource(file) { - return DARWIN_TERMINAL_NODE_PREFIXES.some((prefix) => file.startsWith(prefix)) -} diff --git a/config/scripts/release-mac-build-workflow-dispatch.test.mjs b/config/scripts/release-mac-build-workflow-dispatch.test.mjs index e4d907e266a..f6410b846be 100644 --- a/config/scripts/release-mac-build-workflow-dispatch.test.mjs +++ b/config/scripts/release-mac-build-workflow-dispatch.test.mjs @@ -164,7 +164,7 @@ describe('release mac build workflow dispatch', () => { expect(options.releaseRunId).toBe('987') expect(options.workflow).toBe('release-mac-build.yml') expect(options.pollSeconds).toBe(30) - expect(options.timeoutMinutes).toBe(390) + expect(options.timeoutMinutes).toBe(90) }) }) @@ -195,84 +195,6 @@ const splitRun = { } describe('release mac build split dispatch and verification', () => { - it('hands a pending exact run to verification after observing 180 minutes without renewing its deadline', async () => { - const startedAt = Date.parse('2026-06-30T12:00:00Z') - let currentTime = startedAt - const live = { ...splitRun, status: 'in_progress', conclusion: null } - const { fetch } = createGitHubFetch([ - jsonResponse(200, { workflow_run_id: 123 }), - jsonResponse(200, live), - jsonResponse(200, live), - jsonResponse(200, live) - ]) - const appendOutput = vi.fn() - const result = await executeReleaseMacBuildMode( - 'dispatch', - { ...splitEnvironment, RELEASE_MAC_BUILD_OBSERVE_MINUTES: '180' }, - { - fetch, - appendOutput, - now: () => currentTime, - sleep: async () => { - currentTime += 180 * 60_000 - } - } - ) - expect(result).toMatchObject({ id: 123, status: 'in_progress', conclusion: null }) - expect(appendOutput).toHaveBeenCalledOnce() - expect(appendOutput.mock.calls[0][1]).toContain( - `wait_deadline_ms=${startedAt + 390 * 60_000}\n` - ) - const { fetch: finalFetch } = createGitHubFetch([ - jsonResponse(200, { ...splitRun, updated_at: '2026-06-30T18:00:00Z' }) - ]) - await expect( - executeReleaseMacBuildMode( - 'wait', - { - ...splitEnvironment, - RELEASE_MAC_BUILD_WAIT_DEADLINE_MS: String(startedAt + 390 * 60_000) - }, - { fetch: finalFetch, now: () => startedAt + 360 * 60_000 } - ) - ).resolves.toMatchObject({ - status: 'completed', - conclusion: 'success' - }) - const { fetch: expiredFetch } = createGitHubFetch([jsonResponse(200, live)]) - await expect( - executeReleaseMacBuildMode( - 'wait', - { - ...splitEnvironment, - RELEASE_MAC_BUILD_WAIT_DEADLINE_MS: String(startedAt + 390 * 60_000) - }, - { fetch: expiredFetch, now: () => startedAt + 391 * 60_000 } - ) - ).rejects.toThrow(/Timed out/) - }) - - it.each([ - { ...splitRun, conclusion: 'failure' }, - { ...splitRun, head_sha: 'b'.repeat(40) } - ])('does not hand off failure or changed identity as pending: %j', async (observed) => { - const { fetch } = createGitHubFetch([ - jsonResponse(200, { workflow_run_id: 123 }), - jsonResponse(200, splitRun), - jsonResponse(200, observed) - ]) - await expect( - executeReleaseMacBuildMode( - 'dispatch', - { - ...splitEnvironment, - RELEASE_MAC_BUILD_OBSERVE_MINUTES: '180' - }, - { fetch, appendOutput: vi.fn(), now: () => Date.parse('2026-06-30T12:00:00Z') } - ) - ).rejects.toThrow() - }) - it('waits for the exact dispatched run metadata to become consistent', async () => { const { fetch, requests } = createGitHubFetch([ jsonResponse(200, { workflow_run_id: 123 }), @@ -327,7 +249,7 @@ describe('release mac build split dispatch and verification', () => { expect(requests[0].headers['X-GitHub-Api-Version']).toBe('2026-03-10') expect(appendOutput).toHaveBeenCalledWith( splitEnvironment.GITHUB_OUTPUT, - `run_id=123\nworkflow_sha=${splitRun.head_sha}\nrun_attempt=1\nwait_deadline_ms=${Date.parse('2026-06-30T18:30:00Z')}\n` + `run_id=123\nworkflow_sha=${splitRun.head_sha}\nrun_attempt=1\nwait_deadline_ms=${splitEnvironment.RELEASE_MAC_BUILD_WAIT_DEADLINE_MS}\n` ) }) diff --git a/config/scripts/release-windows-signing-gate.test.mjs b/config/scripts/release-windows-signing-gate.test.mjs index 20340b16742..5a6d022d44a 100644 --- a/config/scripts/release-windows-signing-gate.test.mjs +++ b/config/scripts/release-windows-signing-gate.test.mjs @@ -5,7 +5,6 @@ import { runInNewContext } from 'node:vm' import { runProcess } from '@orca/process-host' import { describe, expect, it } from 'vitest' import { parse } from 'yaml' -import { readReleaseMacBuildWorkflowOptions } from './run-release-mac-build-workflow.mjs' const { jobs } = parse(readFileSync('.github/workflows/release-cut.yml', 'utf8')) const { steps } = jobs.build @@ -36,39 +35,12 @@ function signingAllowed(submission, outcome, attempt = 1, platform = 'win', uplo } describe('Windows release signing waits for macOS success', () => { - it('covers the cold runtime and packaging budget in two hosted observation jobs', () => { - const child = parse(readFileSync('.github/workflows/release-mac-build.yml', 'utf8')).jobs[ - 'build-mac' - ] - const dispatch = jobs['build-mac'].steps.find((step) => step.id === 'dispatch') - const options = readReleaseMacBuildWorkflowOptions({ - GITHUB_REPOSITORY: 'stablyai/orca', - GITHUB_RUN_ID: '1', - GITHUB_TOKEN: 'test', - RELEASE_MAC_BUILD_REF: 'main', - RELEASE_MAC_BUILD_TAG: 'v1.2.3', - ...dispatch.env - }) - const observationMinutes = Number(dispatch.env.RELEASE_MAC_BUILD_OBSERVE_MINUTES) - expect(child['timeout-minutes']).toBe(360) - expect(options.timeoutMinutes).toBeGreaterThanOrEqual(child['timeout-minutes'] + 30) - expect(observationMinutes).toBe(180) - expect(jobs['build-mac']['timeout-minutes']).toBeGreaterThan(observationMinutes) - expect(jobs['verify-mac-build']['timeout-minutes']).toBeGreaterThan( - options.timeoutMinutes - observationMinutes - ) - for (const job of [jobs['build-mac'], jobs['verify-mac-build'], jobs.build]) { - expect(job['timeout-minutes']).toBeLessThanOrEqual(360) - } - }) - - it('waits outside the Windows runner, then verifies the exact macOS run before signing', () => { + it('packages concurrently, then verifies the exact dispatched macOS run before signing', () => { expect(jobs.build.needs).toContain('build-mac') expect(jobs.build.if).toContain("needs.build-mac.result == 'success'") - expect(jobs.build.needs).toContain('verify-mac-build') - expect(jobs.build.if).toContain("needs.verify-mac-build.result == 'success'") + expect(jobs.build.needs).not.toContain('verify-mac-build') expect(jobs['build-mac'].needs).not.toContain('build') - expect(jobs['verify-mac-build'].needs).not.toContain('build') + expect(jobs['verify-mac-build'].needs).toContain('build') const windows = jobs.build.strategy.matrix.include.find((entry) => entry.platform === 'win') expect(windows.release_command).toMatch( /^node config\/scripts\/ensure-native-runtime\.mjs --runtime=electron; if \(\$LASTEXITCODE -ne 0\) \{ exit \$LASTEXITCODE \}; pnpm exec electron-builder / diff --git a/config/scripts/run-release-mac-build-workflow.mjs b/config/scripts/run-release-mac-build-workflow.mjs index 92cea65c06d..5123b81c327 100644 --- a/config/scripts/run-release-mac-build-workflow.mjs +++ b/config/scripts/run-release-mac-build-workflow.mjs @@ -2,7 +2,7 @@ import { appendFileSync } from 'node:fs' import { pathToFileURL } from 'node:url' const DEFAULT_API_VERSION = '2026-03-10' -const DEFAULT_TIMEOUT_MINUTES = 390 +const DEFAULT_TIMEOUT_MINUTES = 90 const DEFAULT_POLL_SECONDS = 30 const RUN_DISCOVERY_TIMEOUT_SECONDS = 120 @@ -100,11 +100,6 @@ export async function finishReleaseMacBuildWorkflow(options, workflowRunId, deps const api = createGitHubApiClient(options, deps) const completedRun = await waitForReleaseMacBuildRun(api, workflowRunId, options, deps) - if (completedRun.status !== 'completed' && deps.observationDeadlineMs != null) { - console.log(`Mac release build workflow remains pending: ${completedRun.html_url}`) - return completedRun - } - if (completedRun.conclusion !== 'success') { throw new Error( `Mac release build workflow ${completedRun.html_url ?? completedRun.id} concluded ${ @@ -195,13 +190,9 @@ export async function waitForReleaseMacBuildRun(api, workflowRunId, options, dep } return run } - const observedAtMs = now() - if (observedAtMs > deadlineMs) { + if (now() > deadlineMs) { break } - if (deps.observationDeadlineMs != null && observedAtMs >= deps.observationDeadlineMs) { - return run - } console.log( `Mac release build workflow is ${run.status}; polling again in ${options.pollSeconds}s` @@ -319,15 +310,6 @@ export async function executeReleaseMacBuildMode(mode, env = process.env, deps = output, `run_id=${run.id}\nworkflow_sha=${run.head_sha}\nrun_attempt=${run.run_attempt}\nwait_deadline_ms=${deadlineMs}\n` ) - if (env.RELEASE_MAC_BUILD_OBSERVE_MINUTES) { - const observationMinutes = readPositiveInteger(env.RELEASE_MAC_BUILD_OBSERVE_MINUTES, 180) - return await finishReleaseMacBuildWorkflow(options, run.id, { - ...deps, - deadlineMs, - observationDeadlineMs: now() + observationMinutes * 60 * 1000, - expectedIdentity: { id: run.id, headSha: run.head_sha, attempt: run.run_attempt } - }) - } return run } if (mode === 'wait') { diff --git a/config/scripts/verify-darwin-terminal-node.mjs b/config/scripts/verify-darwin-terminal-node.mjs deleted file mode 100644 index ed865305eeb..00000000000 --- a/config/scripts/verify-darwin-terminal-node.mjs +++ /dev/null @@ -1,118 +0,0 @@ -import { cpSync, mkdtempSync, rmSync } from 'node:fs' -import { createRequire } from 'node:module' -import { tmpdir } from 'node:os' -import { dirname, join } from 'node:path' -import { runProcess, runProcessSync } from '@orca/process-host' -import { ensureDarwinTerminalNode } from './build-darwin-terminal-node.mjs' -import { preparePinnedNodeDir } from './pinned-node-downloads.mjs' -import { nodeGypRebuild, stageNodeAddonApi } from './orcad-prebuild-node-gyp.mjs' -import { isDirectInvocation } from './script-entry-detection.mjs' -import { NODE_RUNTIME_PIN } from '../../src/shared/node-runtime-pin.ts' - -export function readDarwinVerificationArch(args, defaultArch = process.arch) { - if (args.length > 1 || (args.length === 1 && !/^--arch=(arm64|x64)$/.test(args[0]))) { - throw new Error('Usage: verify-darwin-terminal-node.mjs [--arch=arm64|x64]') - } - const arch = args.length ? args[0].slice('--arch='.length) : defaultArch - if (arch !== 'arm64' && arch !== 'x64') { - throw new Error(`Unsupported macOS terminal Node architecture: ${arch}`) - } - return arch -} - -async function verifyDarwinTerminalNode(arch) { - if (process.platform !== 'darwin') { - throw new Error('Verify the Darwin terminal runtime on a macOS host') - } - const executable = await ensureDarwinTerminalNode({ arch }) - const scratch = mkdtempSync(join(tmpdir(), 'orca-darwin-terminal-node-proof-')) - const require = createRequire(import.meta.url) - try { - const headers = await preparePinnedNodeDir({ - target: `darwin-${arch}`, - workDir: join(scratch, 'headers') - }) - const sourcePty = dirname(require.resolve('node-pty/package.json')) - const stagedPty = join(scratch, 'node-pty') - for (const entry of ['package.json', 'binding.gyp', 'src', 'lib']) { - cpSync(join(sourcePty, entry), join(stagedPty, entry), { recursive: true }) - } - stageNodeAddonApi(sourcePty, stagedPty) - await nodeGypRebuild({ - stagedDir: stagedPty, - workDir: scratch, - nodeDir: headers, - runProcess: (options) => - runProcessSync({ - ...options, - program: executable, - timeoutMs: 120_000, - env: { ...options.env, ORCA_BACKGROUND_LAUNCH: '1', npm_config_arch: arch } - }) - }) - const addon = join(scratch, 'worker-wait.node') - const compiled = await runProcess({ - program: 'clang', - args: [ - '-arch', - arch === 'x64' ? 'x86_64' : 'arm64', - '-O2', - '-shared', - '-fPIC', - '-undefined', - 'dynamic_lookup', - '-I', - join(headers, 'include/node'), - join(import.meta.dirname, '__fixtures__/darwin-terminal-worker-wait.c'), - '-o', - addon - ], - timeoutMs: 60_000 - }) - if (compiled.code !== 0) { - throw new Error(compiled.stderr) - } - const result = await runProcess({ - program: executable, - args: [ - join(import.meta.dirname, 'darwin-terminal-node-probe.cjs'), - addon, - stagedPty, - scratch, - `v${NODE_RUNTIME_PIN.version}` - ], - env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1', UV_THREADPOOL_SIZE: '4' }, - timeoutMs: 60_000 - }) - if (result.code !== 0) { - throw new Error(`Runtime proof failed (${result.code}): ${result.stderr}`) - } - if (JSON.parse(result.stdout.trim()).arch !== arch) { - throw new Error(`Runtime proof used the wrong architecture; expected ${arch}`) - } - console.log(result.stdout.trim()) - if (process.env.ORCA_DARWIN_WORKER_STRESS === '1') { - const stress = await runProcess({ - program: executable, - args: [ - join(import.meta.dirname, 'darwin-terminal-worker-stress.cjs'), - addon, - stagedPty, - scratch - ], - env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1', UV_THREADPOOL_SIZE: '4' }, - timeoutMs: 61 * 60_000 - }) - console.log(stress.stdout.trim()) - if (stress.code !== 0) { - throw new Error(`Sequential worker stress failed: ${stress.stderr}`) - } - } - } finally { - rmSync(scratch, { recursive: true, force: true }) - } -} - -if (isDirectInvocation(import.meta.url, process.argv[1])) { - await verifyDarwinTerminalNode(readDarwinVerificationArch(process.argv.slice(2))) -} diff --git a/config/scripts/win32-test-lane-registration.test.mjs b/config/scripts/win32-test-lane-registration.test.mjs index 99ac812cd20..a191e2a9726 100644 --- a/config/scripts/win32-test-lane-registration.test.mjs +++ b/config/scripts/win32-test-lane-registration.test.mjs @@ -149,35 +149,22 @@ export function requiresEnvOptIn(source) { }) } -// Treat any runs-on label/array/group/matrix that may be Windows as a lane; simple matrix axes resolve -// from their declared values and an unresolvable expression fails closed for review. -export function couldRunOnWindows(runsOn, matrix = {}) { +// Treat any runs-on label/array/group/matrix that may be Windows as a lane; unresolvable expression fails closed for review. +export function couldRunOnWindows(runsOn) { const labels = typeof runsOn === 'string' ? [runsOn] : Array.isArray(runsOn) ? runsOn : [...(runsOn?.labels ?? []), runsOn?.group ?? ''].flat() - return labels.some((label) => { - const axis = /^\$\{\{\s*matrix\.(\w+)\s*\}\}$/.exec(String(label))?.[1] - const values = axis && matrix[axis] - if (Array.isArray(values) && values.length > 0) { - const included = Array.isArray(matrix.include) - ? matrix.include.flatMap((entry) => (entry[axis] === undefined ? [] : [entry[axis]])) - : [] - return [...values, ...included].some((value) => couldRunOnWindows(value)) - } - return /windows/i.test(String(label)) || String(label).includes('${{') - }) + return labels.some((label) => /windows/i.test(String(label)) || String(label).includes('${{')) } /** The Windows job's curated files: the boundary list plus any argv the lane steps spell out. */ function readWindowsWorkflow() { const workflow = parse(readFileSync(join(projectDir, '.github/workflows/pr.yml'), 'utf8')) const jobs = Object.entries(workflow.jobs ?? {}) - const windowsJobs = jobs.filter(([, job]) => - couldRunOnWindows(job?.['runs-on'], job?.strategy?.matrix) - ) + const windowsJobs = jobs.filter(([, job]) => couldRunOnWindows(job?.['runs-on'])) const steps = workflow.jobs?.[WINDOWS_LANE_JOB]?.steps ?? [] const runs = WINDOWS_LANE_STEPS.map((name) => { const step = steps.find((candidate) => candidate?.name === name) @@ -410,21 +397,6 @@ describe('manual opt-in classification', () => { }) describe('Windows runner detection', () => { - it('resolves explicit matrix axes while keeping unknown expressions conservative', () => { - expect(couldRunOnWindows('${{ matrix.os }}', { os: ['macos-15', 'macos-15-intel'] })).toBe( - false - ) - expect(couldRunOnWindows('${{ matrix.os }}', { os: ['macos-15', 'windows-2022'] })).toBe(true) - expect(couldRunOnWindows('${{ matrix.os }}', { os: [] })).toBe(true) - expect( - couldRunOnWindows('${{ matrix.os }}', { - os: ['macos-15'], - include: [{ os: 'windows-2022' }] - }) - ).toBe(true) - expect(couldRunOnWindows('${{ inputs.runner }}', { os: ['macos-15'] })).toBe(true) - }) - it('reads every runs-on spelling that could land on Windows', () => { expect(couldRunOnWindows('windows-2022')).toBe(true) // The spelling that would have slipped past an equality test. diff --git a/docs/bug-reproductions/darwin-worker-wakeup.md b/docs/bug-reproductions/darwin-worker-wakeup.md deleted file mode 100644 index b88bf3722eb..00000000000 --- a/docs/bug-reproductions/darwin-worker-wakeup.md +++ /dev/null @@ -1,181 +0,0 @@ -# macOS terminal daemon worker wakeup stall - -On 2026-10-15, a long-running packaged macOS daemon accepted RPC traffic and PTY -output but stopped delivering terminal input or creating terminals. Its Node -24.21.0/libuv 1.52.1 event loop remained responsive. All four worker threads were -idle in `__psynch_cvwait`, with 38 queued jobs and no completed work waiting for -the loop. Submitting one asynchronous stat grew the queue to 39 without a callback. -This distinguishes the incident from a busy or saturated worker pool. - -A read-only native snapshot found these pthread condition generations: - -```text -L = U = 0x953f6000 -S = 0x953f5c02 -worker wait L: 0x153f6000, 0x953f6000, 0x953f5f00, 0x953f5e00 -``` - -The oldest waiter was exactly `0x80000000` behind the current generation. This -matches the psynch half-range failure discussed in -[libuv #5280](https://github.com/libuv/libuv/issues/5280). Kernel waiter ordering -can turn wake signals into preposts; once the signal generation U catches L, -subsequent signals can return in userspace while workers remain asleep. - -## Recurrence on 2026-10-23 - -A preserved v50 daemon running the unpatched Node 24.21.0/libuv 1.52.1 -runtime stopped accepting input with 155 sessions. Authenticated control requests -still answered in 2–3 ms and terminal output continued; new terminal creation -timed out after 30 seconds. Asynchronous stat, directory and crypto callbacks -stalled, while all four workers slept in `__psynch_cvwait`, the global work queue -was nonempty and the completion queue was empty. The existing detector logged -`async-io-stalled`. A working Claude terminal belonged to a different, older -daemon, so the apparent Codex/Claude distinction was a daemon ownership difference. - -A one-off native diagnostic interrupted all four verified workers while holding -their pool mutex. Queued work drained, fresh stats completed in about 1 ms, and -input resumed with the same PTY incarnation and agent processes. The user -confirmed recovery. Inspector access was then closed. This validates recovery -without restarting agents, but private-memory access and signal injection are -not production recovery mechanisms; bounded worker waits are the runtime fix. - -## Reproduction and limits - -The [upstream candidate](https://github.com/libuv/libuv/pull/5351) includes a -manual witness using production libuv workers and their real queue. It parks one -worker, advances the condition through 8,388,605 timed waits on the **same mutex**, -enrolls three fresh workers, and submits four jobs. It changes no private pthread -counters and does not occupy the worker pool with blocking filesystem requests. - -| Build | Jobs | Callbacks | Queue | Idle workers | -| ------------------------------ | ---- | --------- | -------- | ------------ | -| Unpatched, half-range boundary | 0 | 0 | Nonempty | 4 | -| Patched, same aging witness | 4 | 4 | Empty | 4 | -| Short control, either build | 4 | 4 | Empty | 4 | - -The unpatched witness produced L = U = 2147483904, S = 2147482882, and oldest -wait L = 256: exactly the same half-range relationship as the live daemon. -The patched witness refreshed the old wait during aging and completed normally. -A separate idle-generation regression fails on unpatched libuv and passes with -the patch; nineteen focused pool, cancellation, fork and condition tests pass. - -The controlled witness proves the native mechanism; the later sequential-job -reproduction below establishes an ordinary scheduling path to it. The precise -Orca job-producer sequence is still unproven. Stress runs with up to 60 million jobs, stop/continue cycles, -signals and PTY churn did not independently reproduce that enrollment sequence. -The earlier PTY fault injection established symptoms only. - -## Ordinary-workload reproduction - -A later run on the exact unpatched packaged Node also reproduced the failure -without condition aging or fault injection: queue one empty native job, and queue -its successor from the completion callback. Four workers service one outstanding -job. An initial run stalled after 11,032,951 completed jobs and hung in worker -cleanup on normal exit. A second instrumented run started with a successful real -PTY echo and stalled after 11,886,884 completed jobs: - -```text -L = 3043043584; oldest worker wait L = 895559936 -L - oldest wait L = 2147483648 = 0x80000000 -four idle workers; one queued native job -``` - -Four fresh async stats and a PTY write produced zero stat callbacks and no PTY -echo after 15 seconds; six jobs remained queued and U caught L. One worker had -stayed parked while millions of sequential jobs ran through the others. This -provides an ordinary scheduling path to the exact live-daemon signature. An -otherwise similar 30-million-job run completed, so the scheduling is intermittent. -The patched runtime completed all 30 million jobs, a fresh filesystem request -and real PTY echo. This still does not identify the exact sequence of Orca's own -job producers. - -The repository now includes a portable-to-both-macOS-architectures N-API workload -and PTY oracle with no private pthread fields. Run it after the normal check: - -```sh -ORCA_BACKGROUND_LAUNCH=1 ORCA_DARWIN_WORKER_STRESS=1 node config/scripts/verify-darwin-terminal-node.mjs -``` - -The workload runs 30 million sequential native jobs and requires PTY echo before -and after. After three seconds without progress, it allows forty seconds for -the thirty-second worker timeout and scheduling margin, requiring all four fresh -stat callbacks, a new PTY echo and resumed native work. It kills only its own disposable test -process if libuv cleanup is stranded. This probabilistic stress is opt-in; the -idle-refresh regression remains the regular CI oracle. - -## Runtime mitigation - -Only the private Darwin worker wait changes: it uses a 30-second relative timed -condition wait, retaining the existing queue predicate and Darwin error workaround. -Workers still wake immediately when signaled. An idle timeout rechecks queued -work and refreshes the wait generation, allowing recovery from lost wakeups. -The default four-worker pool adds at most eight timeout wakes per minute; the -regression also caps generation advancement to reject an accidental wake loop. -Public condition variables and other platforms are unchanged. - -`config/scripts/build-darwin-terminal-node.mjs` builds the pinned official Node -source with this patch. The recipe now follows Node 24.22.1: its source archive -matches the official SHA-256 manifest, the patch applies with zero fuzz, and the -reviewed patched worker source is byte-identical to the Node 24.21.0 recipe. The recipe pins the source archive hash, reviewed patched -source hash, architecture and minimum OS. Cache receipts verify the resulting -executable bytes and fingerprint the build scripts, so changes to build logic -invalidate cached binaries. A Node pin change fails until the recipe is reviewed. -This preserves the existing signed -helper layout and stable host copy rather than changing its launch or permissions. - -Run the shipping-runtime check on each macOS architecture: - -```sh -ORCA_BACKGROUND_LAUNCH=1 node config/scripts/verify-darwin-terminal-node.mjs -``` - -It builds the runtime, checks real PTY input and filesystem/DNS/crypto operations, -observes all four idle worker generations refresh, bounds their advancement, and -checks fresh asynchronous work. It inspects only its own test process. The Node 24.21.0 recipe passed on Apple silicon (x64 under Rosetta) and on -physical ARM and Intel CI ([run](https://github.com/stablyai/orca/actions/runs/38795005777)). -On 2026-10-23, the repository-built Node 24.22.1 arm64 runtime also passed: - -```json -{ - "node": "v24.22.1", - "uv": "1.52.1", - "arch": "arm64", - "fsDnsCrypto": "passed", - "ptyInput": "passed", - "idleWaitRefresh": "passed" -} -``` - -The official unpatched 24.22.1 runtime passed the same initial PTY and I/O checks, -then failed the idle-generation `advances > 0n` assertion. This supplies a current-pin -red/green regression. ARM CI qualification passed on PR commit `6f006d865193` in -[run 39922624279](https://github.com/stablyai/orca/actions/runs/39922624279); -Intel qualification was still running at review time. The cold -build also exposed multi-minute synchronous source cleanup; the cache lease is -now released after atomic publication and before cleanup, so a blocked heartbeat -cannot let a waiting builder steal the completed producer's lock. Cold source compilation is -substantial and is cached separately from packaging retries. - -## Coverage and remaining risks - -The packaged macOS terminal daemon receives this runtime, including local folder -workspaces and worktrees, and clients attached to that daemon. Existing daemons -keep their executable until deliberately restarted. The detection/restart prompt -is tracked separately in [Orca #29087](https://github.com/stablyai/orca/pull/29087). -No automatic restart or queued-input replay is part of this patch. - -Linux/Windows and WSL are unaffected by the Darwin-only change. Standalone/SSH -server runtimes and the development Electron fallback do not receive this patch. -Mobile/relay clients benefit only when their execution host uses the patched -helper. An isolated Developer ID signed copy of the installed helper, with the -repository-built runtime substituted, served a PTY under hardened runtime. Its -protocol 45 required a matching smoke-test handshake; the original patch checkout used 46. A -repeat on October 23 with Node 24.22.1 passed strict signature verification, -real PTY input, filesystem/DNS/crypto and all four idle worker refreshes under -hardened runtime. This disposable copy used the installed helper's native -dependencies and an untimestamped Developer ID signature; it does not establish -notarization or qualification of a complete current package. A complete package -build, launchd/TCC attribution, older macOS releases, battery -impact and extended soak remain verification gaps. Do not infer remote process -death from missing callbacks or loss of contact. The experimental reliability gate records these -limits; an upstream merge and a Node release can eventually replace this backport. diff --git a/docs/reference/macos-daemon-tcc-attribution.md b/docs/reference/macos-daemon-tcc-attribution.md index f4a559ea4e1..fe3d9b61ad7 100644 --- a/docs/reference/macos-daemon-tcc-attribution.md +++ b/docs/reference/macos-daemon-tcc-attribution.md @@ -109,9 +109,8 @@ The helper declares no links, documents or types, has a lower `CFBundleVersion` launchd leaves no record, but another process running the copy's executable directly can make macOS register it, so retirement runs `lsregister -u` on the bundle before deleting it. -The original helper used the official Node UUID, shared by that release and architecture. -The worker-wakeup backport builds Node from source and has a different UUID; its launchd/TCC -attribution still needs verification. Local Network access passed in the original signed spike: +The helper's main-executable UUID is Node's, shared by every official Node 24.21.0 binary of that +architecture, so it is not unique to Orca. Local Network access still passed in the signed spike: macOS matched the client by its code signature, and the job's `AssociatedBundleIdentifiers` ties it to Orca. Do not register copies explicitly (`lsregister -f` or `LSRegisterURL`). diff --git a/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts b/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts index 13ade9d2952..7e9c560ccc0 100644 --- a/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts +++ b/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts @@ -3,7 +3,7 @@ import { PREVIOUS_DAEMON_PROTOCOL_VERSIONS, PROTOCOL_VERSION } from './types' describe('foreground-confirmation daemon protocol', () => { it('rejects daemons from before the fresh-confirmation RPC', () => { - expect(PROTOCOL_VERSION).toBe(52) + expect(PROTOCOL_VERSION).toBe(51) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(19) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(22) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(23) @@ -23,6 +23,6 @@ describe('foreground-confirmation daemon protocol', () => { expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(37) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(38) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(39) - expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(51) + expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(50) }) }) diff --git a/src/main/daemon/daemon-protocol-version.ts b/src/main/daemon/daemon-protocol-version.ts index da94db289e3..0416fcc0f03 100644 --- a/src/main/daemon/daemon-protocol-version.ts +++ b/src/main/daemon/daemon-protocol-version.ts @@ -1,5 +1,4 @@ // Why: daemons survive app updates, so wire behavior must be version-gated. -// v52 starts new terminals on a fresh daemon whose patched Node recovers lost macOS worker wake-ups. // v51 checks terminal identity, ownership and attachments before reserving teardown. // v50 makes the `codex` function keep Orca's terminal identity under a filtering shell_environment_policy. // v49 makes the `claude` function follow the selection over the user's own CLAUDE_CONFIG_DIR. @@ -10,7 +9,7 @@ // v43 dropped it with the revert); v41 stages long startup commands as sourced scripts; v40 rolled // the #25130/#24636 shell-wrapper changes and the wider agent list into a fresh daemon; older owners // stay attachable. -export const PROTOCOL_VERSION = 52 +export const PROTOCOL_VERSION = 51 export const FENCED_TERMINATION_PROTOCOL_VERSION = 51 export const CLAUDE_ACCOUNT_FUNCTION_DAEMON_PROTOCOL_VERSION = 42 export const CLAUDE_ACCOUNT_FUNCTION_REVERTED_DAEMON_PROTOCOL_VERSION = 43 @@ -45,7 +44,7 @@ export const CLEAN_DISCONNECT_PROTOCOL_VERSION = 24 export const MODE_2031_UNSUBSCRIBE_FACT_PROTOCOL_VERSION = 29 export const PREVIOUS_DAEMON_PROTOCOL_VERSIONS = [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, - 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 + 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50 ] as const export function supportsColorQueryReplyColors(protocolVersion: number): boolean { diff --git a/src/shared/local-build-compatibility-contract.json b/src/shared/local-build-compatibility-contract.json index 6732acba726..031ab7ce8dc 100644 --- a/src/shared/local-build-compatibility-contract.json +++ b/src/shared/local-build-compatibility-contract.json @@ -3,10 +3,10 @@ "appId": "com.stablyai.orca", "stateSchemaVersion": 1, "readableStateSchemaVersions": [1], - "daemonProtocolVersion": 52, + "daemonProtocolVersion": 51, "attachableDaemonProtocolVersions": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, - 51, 52 + 51 ] } diff --git a/src/shared/local-build-compatibility-contract.ts b/src/shared/local-build-compatibility-contract.ts index ce1b8bd573a..f719d6759b8 100644 --- a/src/shared/local-build-compatibility-contract.ts +++ b/src/shared/local-build-compatibility-contract.ts @@ -3,10 +3,10 @@ export const LOCAL_BUILD_COMPATIBILITY_CONTRACT = { appId: 'com.stablyai.orca', stateSchemaVersion: 1, readableStateSchemaVersions: [1], - daemonProtocolVersion: 52, + daemonProtocolVersion: 51, attachableDaemonProtocolVersions: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, - 51, 52 + 51 ] } as const diff --git a/src/shared/telemetry-daemon-event-schemas.ts b/src/shared/telemetry-daemon-event-schemas.ts index 8de1a9cf86b..4517ec8cb1c 100644 --- a/src/shared/telemetry-daemon-event-schemas.ts +++ b/src/shared/telemetry-daemon-event-schemas.ts @@ -72,7 +72,7 @@ export const daemonAdoptedSchema = z export const DAEMON_ASYNC_IO_STALL_DURATION_BUCKETS = ['<1m', '1-5m', '5-30m', '30m+'] as const -// Why: measures whether #29133's runtime patch makes daemon worker-pool stalls recover. +// Why: measures how often daemon worker-pool stalls start and how long they last before clearing. const daemonAsyncIoHealthBase = { protocol_generation: z.number().int().positive().max(1_000), app_version_match: z.enum(DAEMON_ADOPTED_APP_VERSION_MATCH)