From 81e2d3cbf200ea99527785da7624bbc9183259b5 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Sun, 6 Sep 2026 17:18:55 -0700 Subject: [PATCH] test: validate permanent packaged browser workflow and participation --- .github/workflows/e2e.yml | 472 ++---------------- .github/workflows/packaged-browser-e2e.yml | 70 +++ config/reliability-gates.jsonc | 101 ++++ .../packaged-browser-lane-contract.test.mjs | 40 ++ config/scripts/pr-e2e-source-routing.mjs | 2 +- .../verify-packaged-browser-participation.mjs | 20 + ...fy-packaged-browser-participation.test.mjs | 57 +++ .../verify-playwright-participation.mjs | 42 ++ .../scripts/verify-wsl-e2e-participation.mjs | 40 +- config/scripts/wsl-e2e-lane-contract.test.mjs | 1 + 10 files changed, 384 insertions(+), 461 deletions(-) create mode 100644 .github/workflows/packaged-browser-e2e.yml create mode 100644 config/scripts/packaged-browser-lane-contract.test.mjs create mode 100644 config/scripts/verify-packaged-browser-participation.mjs create mode 100644 config/scripts/verify-packaged-browser-participation.test.mjs create mode 100644 config/scripts/verify-playwright-participation.mjs diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f75d7ba00bb..a57d8f71a24 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,442 +1,70 @@ -name: E2E - -run-name: E2E ${{ inputs.ref || github.ref }} - -# Why: checkout + artifact upload only; callers can only further restrict. -permissions: - contents: read - +name: Packaged browser compatibility on: + workflow_dispatch: + schedule: + - cron: '20 8 * * 1' workflow_call: inputs: ref: - description: Ref to check out (defaults to the calling workflow's ref) - required: false type: string - test_files: - description: JSON array of changed specs; empty runs the full suite required: false - type: string - ssh_source_changed: - description: '"true" when the PR touches SSH execution source; gates the Docker-SSH lane' - required: false - type: string - workflow_dispatch: - inputs: - ref: - description: Ref to check out (defaults to the workflow ref) - required: false - type: string - test_files: - description: JSON array of specs to run; empty runs the full suite - required: false - type: string - schedule: - # Why: GitHub cron uses UTC; these slots map to 10am and 3pm - # America/Phoenix for the default-branch E2E run. - - cron: '0 17,22 * * *' - +permissions: + contents: read jobs: - build: - name: build e2e app + compatibility: runs-on: ubuntu-latest - timeout-minutes: 10 - + timeout-minutes: 25 steps: - - name: Checkout - uses: actions/checkout@v6 + - uses: actions/checkout@v6 with: - ref: ${{ inputs.ref || github.ref }} - - # Why: the build's plain-Node daemon smoke load resolves node-pty. + ref: ${{ inputs.ref || github.sha }} + persist-credentials: false + - name: Install headless tools + run: sudo apt-get update && sudo apt-get install -y build-essential openssh-client python3 ripgrep xvfb zsh openbox x11-utils - uses: ./.github/actions/install-node-dependencies with: - native-runtime: node - - # Why: building here avoids parallel builds inside Playwright globalSetup; - # paired-browser specs also need the standalone web bundle. - - name: Build E2E outputs + native-runtime: electron + - name: Download pinned old release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release download v1.4.188 --repo stablyai/orca --pattern orca-ide_1.4.188_amd64.deb --dir "$RUNNER_TEMP/old-orca" + python3 - <<'PYVERIFY' + import base64,hashlib,os,pathlib,subprocess + root=pathlib.Path(os.environ['RUNNER_TEMP'])/'old-orca' + package=root/'orca-ide_1.4.188_amd64.deb' + expected='uGONFUDfinYggxcT9ac72wnnlofLQaqasDDeP0HWOSqarBwTi1Ax3khmzKUY3vUnvuYOpSCEmsH4InzLZ2vg6g==' + assert base64.b64encode(hashlib.sha512(package.read_bytes()).digest()).decode()==expected + extracted=root/'extracted' + subprocess.run(['dpkg-deb','-x',str(package),str(extracted)],check=True) + candidates=[extracted/'opt'/'Orca'/'orca-ide'] + assert candidates[0].is_file() and os.access(candidates[0],os.X_OK) + assert len(candidates)==1,candidates + with open(os.environ['GITHUB_ENV'],'a') as env: env.write('ORCA_CROSS_VERSION_PACKAGED_EXECUTABLE='+str(candidates[0])+'\n') + print('Verified old package:',candidates[0]) + PYVERIFY + - name: Build current Electron app env: VITE_EXPOSE_STORE: 'true' run: | - status=0 - pnpm run build:relay & - relay_pid=$! - npx electron-vite build --mode e2e || status=1 - pnpm run build:web-from-renderer || status=1 - wait "$relay_pid" || status=1 - exit "$status" - - - name: Upload E2E build output - uses: actions/upload-artifact@v7 - with: - name: e2e-build-out - path: out/ - # Why: build-relay.mjs writes each relay's marker as `out/relay//.version`, - # and upload-artifact drops dotfiles by default — consumers then fail SSH specs with - # "local relay build is missing its version marker". - include-hidden-files: true - retention-days: 1 - if-no-files-found: error - - # Build Electron-native dependencies once per workflow. Consumer shards restore - # this immutable cache instead of compiling the same ABI concurrently. - prepare-native-cache: - name: prepare Electron native cache - runs-on: ubuntu-latest - timeout-minutes: 15 - - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - ref: ${{ inputs.ref || github.ref }} - - - name: Install native build tools - run: sudo apt-get update && sudo apt-get install -y build-essential python3 - - - uses: ./.github/actions/install-node-dependencies - with: - native-runtime: electron - - e2e: - name: e2e ${{ matrix.shard_name }} - needs: [build, prepare-native-cache] - if: inputs.test_files == '' - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - include: - # Fourteen scheduled runs averaged 24.6 minutes per shard; shards 4 - # and 9 repeatedly hit the 30-minute cap. A 12-way trial still left - # one 30-minute shard, so 14 gives the suite enough failure headroom. - - shard: '1/14' - shard_name: 1-of-14 - - shard: '2/14' - shard_name: 2-of-14 - - shard: '3/14' - shard_name: 3-of-14 - - shard: '4/14' - shard_name: 4-of-14 - - shard: '5/14' - shard_name: 5-of-14 - - shard: '6/14' - shard_name: 6-of-14 - - shard: '7/14' - shard_name: 7-of-14 - - shard: '8/14' - shard_name: 8-of-14 - - shard: '9/14' - shard_name: 9-of-14 - - shard: '10/14' - shard_name: 10-of-14 - - shard: '11/14' - shard_name: 11-of-14 - - shard: '12/14' - shard_name: 12-of-14 - - shard: '13/14' - shard_name: 13-of-14 - - shard: '14/14' - shard_name: 14-of-14 - - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - ref: ${{ inputs.ref || github.ref }} - - # Native cache misses need the compiler, Electron needs Xvfb, and paired - # Quick Open needs ripgrep. Install them in one apt transaction per shard. - - name: Install native build and headless UI tools - run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk python3 ripgrep xvfb zsh openbox x11-utils - - - uses: ./.github/actions/install-node-dependencies - with: - native-runtime: electron - - - name: Download E2E build output - uses: actions/download-artifact@v8 - with: - name: e2e-build-out - path: out/ - - # Why: the Electron suite is wall-clock constrained on OSS runners, but - # multiple Electron apps on one Xvfb VM contend on git/Chromium resources. - # Sharding keeps each VM at one Playwright worker while splitting the - # headless suite across separate runners. - # SKIP_BUILD makes Playwright globalSetup reuse the single build job's - # artifact instead of starting five concurrent electron-vite builds. - # ORCA_E2E_FORWARD_APP_LOGS keeps startup failures visible when Electron - # launches but never creates a BrowserWindow. - - name: Run E2E tests (${{ matrix.shard_name }}) - run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1 ORCA_RELAY_PATH="$GITHUB_WORKSPACE/out/relay" pnpm run test:e2e --shard=${{ matrix.shard }} - - # Why: Playwright retains traces/screenshots only on failure. Uploading - # them as an artifact makes post-mortem debugging on CI possible without - # re-running locally. - - name: Upload Playwright traces - if: failure() - uses: actions/upload-artifact@v7 - with: - name: playwright-traces-${{ matrix.shard_name }} - path: test-results/ - retention-days: 7 - if-no-files-found: ignore - - changed-e2e: - name: changed e2e specs - needs: [build, prepare-native-cache] - if: inputs.test_files != '' - runs-on: ubuntu-latest - # Why 45: pr.yml now maps SSH source edits onto Docker-backed specs, so this lane can - # pay a container image build plus ~22 serial SSH tests on top of the changed specs. - timeout-minutes: 45 - - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - ref: ${{ inputs.ref || github.ref }} - - - name: Install native build and headless UI tools - # Why ripgrep: Quick Open's bounded host-side search requires rg instead of an - # unbounded inventory fallback; the paired fixture exercises that real boundary. - # Why openssh-client: the Docker-SSH fixture shells out to ssh/ssh-keygen, and this - # lane now receives those specs from pr.yml's SSH source mapping. - run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk openssh-client python3 ripgrep xvfb zsh openbox x11-utils - - - uses: ./.github/actions/install-node-dependencies - with: - native-runtime: electron - - - name: Download E2E build output - uses: actions/download-artifact@v8 - with: - name: e2e-build-out - path: out/ - - - name: Run changed E2E specs + pnpm run build:relay + pnpm exec electron-vite build --mode e2e + pnpm run build:web-from-renderer + - name: Run both mixed-version directions env: - TEST_FILES_JSON: ${{ inputs.test_files }} - run: | - # Why the native IME spec is dropped: it test.skip()s itself without - # ORCA_E2E_NATIVE_IBUS_HANGUL, which this lane cannot set because it has no ibus - # session. Running it here reported a green skip as coverage. - mapfile -t TEST_FILES < <(jq -r '.[] | select( - . != "tests/e2e/ssh-startup-exec-readiness.spec.ts" and - . != "tests/e2e/paired-startup-exec-readiness.spec.ts" and - . != "tests/e2e/local-ssh-browser-routing.spec.ts" and - . != "tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts" and - . != "tests/e2e/ssh-localhost.spec.ts" and - . != "tests/e2e/ssh-client-hosted-browser-drop-reconnect.spec.ts" and - . != "tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts" and - . != "tests/e2e/terminal-ibus-hangul-native.spec.ts" - )' <<<"$TEST_FILES_JSON") - if [ "${#TEST_FILES[@]}" -eq 0 ]; then - echo "Changed specs are all owned by dedicated lanes." - exit 0 - fi - E2E_ENV=(SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1 ORCA_RELAY_PATH="$GITHUB_WORKSPACE/out/relay") - # Second clause: a spec that reads ORCA_E2E_SSH_DOCKER test.skip()s itself without it, so - # naming only one trigger silently skipped every other Docker-SSH spec in this lane. - # The first clause stays because that spec needs Docker without referencing the variable. - if printf '%s\n' "${TEST_FILES[@]}" | grep -qx 'tests/e2e/ephemeral-vm-provisioned-root.spec.ts' \ - || grep -l 'ORCA_E2E_SSH_DOCKER' "${TEST_FILES[@]}" >/dev/null 2>&1; then - E2E_ENV+=(ORCA_E2E_SSH_DOCKER=1) - fi - E2E_PROJECT_ARGS=() - if grep -l '@headful' "${TEST_FILES[@]}" >/dev/null; then - E2E_PROJECT_ARGS+=(--project=electron-headful) - fi - xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env "${E2E_ENV[@]}" \ - pnpm run test:e2e "${TEST_FILES[@]}" --workers=1 "${E2E_PROJECT_ARGS[@]}" - - - name: Upload Playwright traces - if: failure() - uses: actions/upload-artifact@v7 - with: - name: playwright-traces-changed - path: test-results/ - retention-days: 7 - if-no-files-found: ignore - - ssh-docker-watcher-isolation: - name: ssh docker watcher isolation - needs: [build, prepare-native-cache] - # effect of one route listing a startup-readiness spec — pruning that spec would have - # silently retired the whole lane. The signal is now derived from the SSH routes directly. - # The explicit spec clauses stay for their honest purpose: changed-e2e hands these specs to this - # lane, so editing one must still run it here. - if: >- - inputs.test_files == '' || - inputs.ssh_source_changed == 'true' || - contains(inputs.test_files, 'tests/e2e/local-ssh-browser-routing.spec.ts') || - contains(inputs.test_files, 'tests/e2e/ssh-client-hosted-browser-drop-reconnect.spec.ts') || - contains(inputs.test_files, 'tests/e2e/ssh-startup-exec-readiness.spec.ts') || - contains(inputs.test_files, 'tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts') || - contains(inputs.test_files, 'tests/e2e/paired-startup-exec-readiness.spec.ts') - runs-on: ubuntu-latest - # Why 60: this lane now also runs the remaining Docker-SSH specs serially. They average - # ~18s but several budget 4-10 minutes per test, so a slow run lands far above the old 35 - # — and the sharded lanes already show that a lane which times out is a lane nobody trusts. - timeout-minutes: 60 - - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - ref: ${{ inputs.ref || github.ref }} - - - name: Install native build and headless UI tools - run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk openssh-client python3 ripgrep xvfb zsh openbox x11-utils - - - uses: ./.github/actions/install-node-dependencies - with: - native-runtime: electron - - - name: Download E2E build output - uses: actions/download-artifact@v8 - with: - name: e2e-build-out - path: out/ - - # Why: this is the release-path proof that the deployed Linux relay keeps - # its PTY and explorer live across a real watcher SIGSEGV. - - name: Run Docker SSH watcher isolation E2E - run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker-watcher-isolation - - # Why: Playwright empties test-results/ when it starts, so each step here used to - # destroy the previous step's traces. Only the last lane's failure was ever - # diagnosable from the artifact; set each lane aside before the next one runs. - - name: Keep watcher-isolation traces + PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/packaged-browser-results.json + run: >- + xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh + env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 + pnpm exec playwright test --config tests/playwright.config.ts + tests/e2e/packaged-mixed-version-browser-placement.spec.ts + --project=electron-headless --workers=1 --retries=0 --repeat-each=3 --reporter=list,json + - name: Require all six compatibility executions if: always() - run: | - if [ -d test-results ]; then - mkdir -p e2e-traces - mv test-results "e2e-traces/watcher-isolation" - fi - - # Why always(): this lane gates SSH parking/retention plus startup-exec - # readiness across live SSH, headed paired, and headless serve topologies. - - name: Run Docker SSH terminal parking + startup readiness E2E - if: always() - run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker-terminal-parking - - - name: Keep terminal-parking traces - if: always() - run: | - if [ -d test-results ]; then - mkdir -p e2e-traces - mv test-results "e2e-traces/terminal-parking" - fi - - # Why here rather than the sharded lanes: the shards set no ORCA_E2E_SSH_DOCKER, so every - # spec below skipped itself while the shard still reported green. Running them on this one - # VM pays the fixture image build once instead of ten times, and keeps an SSH regression - # legible as an SSH-named failure. - - name: Run remaining Docker SSH E2E - if: always() - run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker - - - name: Keep remaining-ssh-docker traces - if: always() - run: | - if [ -d test-results ]; then - mkdir -p e2e-traces - mv test-results "e2e-traces/remaining-ssh-docker" - fi - - - name: Upload watcher isolation traces - if: failure() - uses: actions/upload-artifact@v7 - with: - name: playwright-traces-ssh-docker-watcher-isolation - path: e2e-traces/ - retention-days: 7 - if-no-files-found: ignore - - ssh-browser-network-route: - name: ssh browser network route - if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts') - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ inputs.ref || github.ref }} - - uses: ./.github/actions/install-node-dependencies - with: - native-runtime: node - - name: Install SSH client - run: sudo apt-get update && sudo apt-get install -y openssh-client - - name: Run Docker SSH browser network route journeys - env: - ORCA_BACKGROUND_LAUNCH: '1' - ORCA_RUN_DOCKER_SSH_BROWSER_E2E: '1' - run: node_modules/.bin/vitest run --config config/vitest.config.ts tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts - - ssh-localhost: - name: localhost SSH terminal and hooks - needs: [build, prepare-native-cache] - if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/ssh-localhost.spec.ts') - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ inputs.ref || github.ref }} - - name: Install SSH server and headless tools - run: sudo apt-get update && sudo apt-get install -y build-essential openssh-client openssh-server python3 ripgrep xvfb zsh openbox x11-utils - - uses: ./.github/actions/install-node-dependencies - with: - native-runtime: electron - - uses: actions/download-artifact@v8 - with: - name: e2e-build-out - path: out/ - - name: Start isolated localhost SSH server - shell: bash - run: | - # Bare shells install Pi extensions only for an existing agent home. - mkdir -p "$HOME/.pi/agent" - fixture="$RUNNER_TEMP/orca-localhost-sshd" - mkdir -p "$fixture" - ssh-keygen -q -t ed25519 -N '' -f "$fixture/host_key" - ssh-keygen -q -t ed25519 -N '' -f "$fixture/client_key" - cat > "$fixture/sshd_config" <> "$GITHUB_ENV" - - name: Run localhost SSH terminal and hook journey - env: - SKIP_BUILD: '1' - ORCA_E2E_SSH_LOCALHOST: '1' - ORCA_FEATURE_REMOTE_AGENT_HOOKS: '1' - ORCA_E2E_FORWARD_APP_LOGS: '1' - run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-localhost.spec.ts --project=electron-headless --workers=1 + run: node config/scripts/verify-packaged-browser-participation.mjs test-results/packaged-browser-results.json - uses: actions/upload-artifact@v7 - if: failure() + if: always() with: - name: localhost-ssh-traces + name: packaged-mixed-version-audit path: test-results/ - retention-days: 7 - if-no-files-found: ignore + retention-days: 3 diff --git a/.github/workflows/packaged-browser-e2e.yml b/.github/workflows/packaged-browser-e2e.yml new file mode 100644 index 00000000000..a57d8f71a24 --- /dev/null +++ b/.github/workflows/packaged-browser-e2e.yml @@ -0,0 +1,70 @@ +name: Packaged browser compatibility +on: + workflow_dispatch: + schedule: + - cron: '20 8 * * 1' + workflow_call: + inputs: + ref: + type: string + required: false +permissions: + contents: read +jobs: + compatibility: + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref || github.sha }} + persist-credentials: false + - name: Install headless tools + run: sudo apt-get update && sudo apt-get install -y build-essential openssh-client python3 ripgrep xvfb zsh openbox x11-utils + - uses: ./.github/actions/install-node-dependencies + with: + native-runtime: electron + - name: Download pinned old release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release download v1.4.188 --repo stablyai/orca --pattern orca-ide_1.4.188_amd64.deb --dir "$RUNNER_TEMP/old-orca" + python3 - <<'PYVERIFY' + import base64,hashlib,os,pathlib,subprocess + root=pathlib.Path(os.environ['RUNNER_TEMP'])/'old-orca' + package=root/'orca-ide_1.4.188_amd64.deb' + expected='uGONFUDfinYggxcT9ac72wnnlofLQaqasDDeP0HWOSqarBwTi1Ax3khmzKUY3vUnvuYOpSCEmsH4InzLZ2vg6g==' + assert base64.b64encode(hashlib.sha512(package.read_bytes()).digest()).decode()==expected + extracted=root/'extracted' + subprocess.run(['dpkg-deb','-x',str(package),str(extracted)],check=True) + candidates=[extracted/'opt'/'Orca'/'orca-ide'] + assert candidates[0].is_file() and os.access(candidates[0],os.X_OK) + assert len(candidates)==1,candidates + with open(os.environ['GITHUB_ENV'],'a') as env: env.write('ORCA_CROSS_VERSION_PACKAGED_EXECUTABLE='+str(candidates[0])+'\n') + print('Verified old package:',candidates[0]) + PYVERIFY + - name: Build current Electron app + env: + VITE_EXPOSE_STORE: 'true' + run: | + pnpm run build:relay + pnpm exec electron-vite build --mode e2e + pnpm run build:web-from-renderer + - name: Run both mixed-version directions + env: + PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/packaged-browser-results.json + run: >- + xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh + env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 + pnpm exec playwright test --config tests/playwright.config.ts + tests/e2e/packaged-mixed-version-browser-placement.spec.ts + --project=electron-headless --workers=1 --retries=0 --repeat-each=3 --reporter=list,json + - name: Require all six compatibility executions + if: always() + run: node config/scripts/verify-packaged-browser-participation.mjs test-results/packaged-browser-results.json + - uses: actions/upload-artifact@v7 + if: always() + with: + name: packaged-mixed-version-audit + path: test-results/ + retention-days: 3 diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 73ea28a08e0..686b74fa7ae 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -18472,6 +18472,107 @@ "The new PR lane is outside verify until reliability is established." ], "demotionRule": "Keep experimental if provisioning or an execution flakes; never promote by skipping a case, raising timeouts, or retrying until green." + }, + { + "id": "browser.packaged-mixed-version-placement", + "title": "Packaged browser placement across versions", + "maturity": "experimental", + "protection": "partial", + "owner": "browser-runtime", + "layer": "electron-packaged", + "surfaces": [ + "paired browser placement" + ], + "platforms": [ + "linux", + "macos", + "windows" + ], + "providers": [ + "paired-runtime" + ], + "coveredPlatforms": [ + "linux" + ], + "coveredProviders": [ + "paired-runtime" + ], + "coverageNotes": "Published Linux 1.4.188 desktop against current source in both directions; scheduled weekly and manually runnable. No required PR check.", + "motivatingLinks": [ + "https://github.com/stablyai/orca/actions/runs/34069063016" + ], + "invariant": "A paired client and host without client-hosted browser capabilities retain server-hosted browser placement across supported version skew.", + "oracle": "Require both existing named browser placement scenarios to pass three times with one attempt, zero skips, zero failures, and no report errors.", + "commands": [ + "gh workflow run packaged-browser-e2e.yml", + "pnpm exec playwright test tests/e2e/packaged-mixed-version-browser-placement.spec.ts --config tests/playwright.config.ts --project=electron-headless --workers=1 --repeat-each=3 --retries=0", + "node_modules/.bin/vitest run --config config/vitest.config.ts config/scripts/packaged-browser-lane-contract.test.mjs config/scripts/verify-packaged-browser-participation.test.mjs", + "gh run view 34069063016 --log" + ], + "testFiles": [ + "tests/e2e/packaged-mixed-version-browser-placement.spec.ts", + "config/scripts/packaged-browser-lane-contract.test.mjs", + "config/scripts/verify-packaged-browser-participation.test.mjs" + ], + "assertionRefs": [ + { + "file": "tests/e2e/packaged-mixed-version-browser-placement.spec.ts", + "assertions": [ + "old client and old host lack client-host and browser-tunnel capabilities", + "browser contents remain owned by the server and the expected snapshot marker is readable" + ] + }, + { + "file": "config/scripts/verify-packaged-browser-participation.test.mjs", + "assertions": [ + "reject missing, substituted, skipped and retried scenarios" + ] + }, + { + "file": "config/scripts/packaged-browser-lane-contract.test.mjs", + "assertions": [ + "verify pinned package checksum before extraction", + "require both directions three times and run report verification even on failure" + ] + } + ], + "evidenceRuns": [ + { + "date": "2026-09-07", + "runner": "ci", + "platform": "linux", + "result": "passed", + "command": "gh run view 34069063016 --log", + "durationSeconds": 120, + "summary": "Both unmodified compatibility cases passed three times at 5a99f935 with published1.4.188 and main f7d52160162; retries0. Final permanent workflow verification remains pending." + } + ], + "runtimeBudget": { + "p95Seconds": 1500, + "scope": "CI job timeout; not a measured p95" + }, + "flakeHistory": { + "status": "soaking", + "evidence": "Initial executable discovery matched CLI and desktop and was corrected before any tests ran. Corrected baseline2/2 and repeat6/6 pass." + }, + "redGreenEvidence": { + "status": "partial", + "evidence": "Participation unit tests reject missing and retried scenarios; no application mutation proof." + }, + "performanceBudget": { + "required": false, + "evidence": "Compatibility assertions, not a performance benchmark." + }, + "promotionCriteria": [ + "Final workflow JSON report proves all six executions.", + "Collect repeated scheduled history before making this required." + ], + "knownGaps": [ + "Linux1.4.188 only; no macOS or Windows packaged coverage.", + "No folder workspace, SSH execution host or live-service coverage.", + "Other released version pairs remain untested; not a required PR check." + ], + "demotionRule": "Keep experimental if any direction skips or fails; do not extend timeouts or retry to green." } ] } diff --git a/config/scripts/packaged-browser-lane-contract.test.mjs b/config/scripts/packaged-browser-lane-contract.test.mjs new file mode 100644 index 00000000000..0c1111294a1 --- /dev/null +++ b/config/scripts/packaged-browser-lane-contract.test.mjs @@ -0,0 +1,40 @@ +import { readFileSync } from 'node:fs' +import { describe, expect, it } from 'vitest' +import { parse } from 'yaml' + +const workflow = parse( + readFileSync(new URL('../../.github/workflows/packaged-browser-e2e.yml', import.meta.url), 'utf8') +) +const steps = workflow.jobs.compatibility.steps + +describe('packaged browser compatibility lane', () => { + it('runs weekly and supports immutable manual or reusable revisions', () => { + expect(workflow.on.schedule).toHaveLength(1) + expect(workflow.on).toHaveProperty('workflow_dispatch') + expect(steps[0].with.ref).toBe('${{ inputs.ref || github.sha }}') + expect(workflow.permissions).toEqual({ contents: 'read' }) + }) + + it('verifies the pinned package before selecting the desktop executable', () => { + const download = steps.find((step) => step.name === 'Download pinned old release').run + expect(download).toContain('gh release download v1.4.188') + expect(download).toContain('hashlib.sha512(package.read_bytes())') + expect(download).toContain("extracted/'opt'/'Orca'/'orca-ide'") + expect(download.indexOf('assert base64.')).toBeLessThan(download.indexOf("['dpkg-deb'")) + }) + + it('requires both directions three times and rejects silent skips', () => { + const run = steps.find((step) => step.name === 'Run both mixed-version directions') + expect(run.run).toContain('tests/e2e/packaged-mixed-version-browser-placement.spec.ts') + expect(run.run).toContain('--repeat-each=3') + expect(run.run).toContain('--retries=0') + expect(run.run).toContain('--reporter=list,json') + const verify = steps.find((step) => step.name === 'Require all six compatibility executions') + expect(verify.if).toBe('always()') + expect(verify.run).toBe( + `node config/scripts/verify-packaged-browser-participation.mjs ${run.env.PLAYWRIGHT_JSON_OUTPUT_FILE}` + ) + expect(steps.at(-1).if).toBe('always()') + expect(steps.at(-1).with.path).toBe('test-results/') + }) +}) diff --git a/config/scripts/pr-e2e-source-routing.mjs b/config/scripts/pr-e2e-source-routing.mjs index bda022159d4..308f1dfdaa3 100644 --- a/config/scripts/pr-e2e-source-routing.mjs +++ b/config/scripts/pr-e2e-source-routing.mjs @@ -41,7 +41,7 @@ export const PR_E2E_SOURCE_ROUTES = [ ], matches: (file) => isProductSource(file) && - /^(?:config\/scripts\/verify-wsl-e2e-participation\.mjs$|src\/main\/(?:wsl[/-]|pty\/.*wsl|providers\/wsl)|src\/shared\/(?:wsl-|windows-terminal-shell)|src\/renderer\/src\/.*(?:terminal-paste|pty-paste)|tests\/e2e\/(?:golden-tab-bar-agent-launch\.spec|terminal-windows-shell-paste-ownership\.spec|helpers\/(?:wsl-golden-stub-agent|golden-stub-agent))|\.github\/(?:actions\/setup-wsl-test-runtime\/|workflows\/windows-wsl-e2e\.yml))/.test( + /^(?:config\/scripts\/(?:verify-wsl-e2e-participation|verify-playwright-participation)\.mjs$|src\/main\/(?:wsl[/-]|pty\/.*wsl|providers\/wsl)|src\/shared\/(?:wsl-|windows-terminal-shell)|src\/renderer\/src\/.*(?:terminal-paste|pty-paste)|tests\/e2e\/(?:golden-tab-bar-agent-launch\.spec|terminal-windows-shell-paste-ownership\.spec|helpers\/(?:wsl-golden-stub-agent|golden-stub-agent))|\.github\/(?:actions\/setup-wsl-test-runtime\/|workflows\/windows-wsl-e2e\.yml))/.test( file ) }, diff --git a/config/scripts/verify-packaged-browser-participation.mjs b/config/scripts/verify-packaged-browser-participation.mjs new file mode 100644 index 00000000000..c165ab46f19 --- /dev/null +++ b/config/scripts/verify-packaged-browser-participation.mjs @@ -0,0 +1,20 @@ +import { readFileSync } from 'node:fs' +import { pathToFileURL } from 'node:url' +import { verifyPlaywrightParticipation } from './verify-playwright-participation.mjs' + +export const PACKAGED_BROWSER_TEST_TITLES = [ + 'keeps an old packaged client on the current server-hosted path', + 'keeps a current client on an old packaged server-hosted path' +] + +export function verifyPackagedBrowserParticipation(report) { + verifyPlaywrightParticipation(report, { + titles: PACKAGED_BROWSER_TEST_TITLES, + label: 'Packaged browser' + }) +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + verifyPackagedBrowserParticipation(JSON.parse(readFileSync(process.argv[2], 'utf8'))) + console.log('Both packaged browser directions passed three times without skips or retries.') +} diff --git a/config/scripts/verify-packaged-browser-participation.test.mjs b/config/scripts/verify-packaged-browser-participation.test.mjs new file mode 100644 index 00000000000..6508777e19a --- /dev/null +++ b/config/scripts/verify-packaged-browser-participation.test.mjs @@ -0,0 +1,57 @@ +import { describe, expect, it } from 'vitest' +import { + verifyPackagedBrowserParticipation, + PACKAGED_BROWSER_TEST_TITLES +} from './verify-packaged-browser-participation.mjs' + +function report() { + return { + stats: { expected: 6, skipped: 0, unexpected: 0, flaky: 0 }, + suites: [ + { + suites: [ + { + specs: PACKAGED_BROWSER_TEST_TITLES.map((title) => ({ + title, + tests: Array.from({ length: 3 }, () => ({ + expectedStatus: 'passed', + results: [{ status: 'passed' }] + })) + })) + } + ] + } + ] + } +} + +describe('Packaged browser participation', () => { + it('accepts both named scenarios executed three times', () => { + expect(() => verifyPackagedBrowserParticipation(report())).not.toThrow() + }) + it.each(['skipped', 'unexpected', 'flaky'])('rejects a nonzero %s result', (key) => { + const value = report() + value.stats[key] = 1 + expect(() => verifyPackagedBrowserParticipation(value)).toThrow('participation failed') + }) + it('rejects missing scenarios even when aggregate counts claim six passes', () => { + const value = report() + value.suites[0].suites[0].specs.pop() + expect(() => verifyPackagedBrowserParticipation(value)).toThrow('requires three executions') + }) + it('rejects an unrelated scenario substituted for an expected scenario', () => { + const value = report() + value.suites[0].suites[0].specs[0].title = 'native shell passes' + expect(() => verifyPackagedBrowserParticipation(value)).toThrow( + 'Unexpected Packaged browser scenario' + ) + }) + it('rejects a pass obtained after a failed attempt', () => { + const value = report() + value.suites[0].suites[0].specs[0].tests[0].results.unshift({ status: 'failed' }) + expect(() => verifyPackagedBrowserParticipation(value)).toThrow('without retries') + }) + it('rejects missing report content', () => { + expect(() => verifyPackagedBrowserParticipation({})).toThrow('participation failed') + }) +}) diff --git a/config/scripts/verify-playwright-participation.mjs b/config/scripts/verify-playwright-participation.mjs new file mode 100644 index 00000000000..d78f2757f1e --- /dev/null +++ b/config/scripts/verify-playwright-participation.mjs @@ -0,0 +1,42 @@ +export function verifyPlaywrightParticipation(report, { titles, label, repetitions = 3 }) { + const stats = report?.stats + if ( + !stats || + stats.expected !== titles.length * repetitions || + stats.skipped !== 0 || + stats.unexpected !== 0 || + stats.flaky !== 0 || + report.errors?.length + ) { + throw new Error(`${label} participation failed: ${JSON.stringify(stats)}`) + } + const counts = new Map(titles.map((title) => [title, 0])) + const visit = (suites) => { + for (const suite of suites ?? []) { + for (const spec of suite.specs ?? []) { + if (!counts.has(spec.title)) { + throw new Error(`Unexpected ${label} scenario: ${spec.title}`) + } + for (const test of spec.tests ?? []) { + if ( + test.expectedStatus !== 'passed' || + test.results?.length !== 1 || + test.results[0].status !== 'passed' + ) { + throw new Error(`${label} scenario did not pass without retries: ${spec.title}`) + } + counts.set(spec.title, counts.get(spec.title) + 1) + } + } + visit(suite.suites) + } + } + visit(report.suites) + for (const [title, count] of counts) { + if (count !== repetitions) { + throw new Error( + `${label} scenario requires ${repetitions === 3 ? 'three' : repetitions} executions: ${title} (${count})` + ) + } + } +} diff --git a/config/scripts/verify-wsl-e2e-participation.mjs b/config/scripts/verify-wsl-e2e-participation.mjs index 21570ef7689..9e8c9252b7f 100644 --- a/config/scripts/verify-wsl-e2e-participation.mjs +++ b/config/scripts/verify-wsl-e2e-participation.mjs @@ -1,3 +1,4 @@ +import { verifyPlaywrightParticipation } from './verify-playwright-participation.mjs' import { readFileSync } from 'node:fs' import { pathToFileURL } from 'node:url' @@ -8,44 +9,7 @@ export const WSL_TEST_TITLES = [ ] export function verifyWslParticipation(report) { - const stats = report?.stats - if ( - !stats || - stats.expected !== 9 || - stats.skipped !== 0 || - stats.unexpected !== 0 || - stats.flaky !== 0 || - report.errors?.length - ) { - throw new Error(`WSL participation failed: ${JSON.stringify(stats)}`) - } - const counts = new Map(WSL_TEST_TITLES.map((title) => [title, 0])) - const visit = (suites) => { - for (const suite of suites ?? []) { - for (const spec of suite.specs ?? []) { - if (!counts.has(spec.title)) { - throw new Error(`Unexpected WSL scenario: ${spec.title}`) - } - for (const test of spec.tests ?? []) { - if ( - test.expectedStatus !== 'passed' || - test.results?.length !== 1 || - test.results[0].status !== 'passed' - ) { - throw new Error(`WSL scenario did not pass without retries: ${spec.title}`) - } - counts.set(spec.title, counts.get(spec.title) + 1) - } - } - visit(suite.suites) - } - } - visit(report.suites) - for (const [title, count] of counts) { - if (count !== 3) { - throw new Error(`WSL scenario requires three executions: ${title} (${count})`) - } - } + verifyPlaywrightParticipation(report, { titles: WSL_TEST_TITLES, label: 'WSL' }) } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { diff --git a/config/scripts/wsl-e2e-lane-contract.test.mjs b/config/scripts/wsl-e2e-lane-contract.test.mjs index 0369eb7c0c4..6790e19e5fe 100644 --- a/config/scripts/wsl-e2e-lane-contract.test.mjs +++ b/config/scripts/wsl-e2e-lane-contract.test.mjs @@ -8,6 +8,7 @@ const read = (path) => readFileSync(new URL(`../../${path}`, import.meta.url), ' describe('real WSL terminal lane', () => { it.each([ 'config/scripts/verify-wsl-e2e-participation.mjs', + 'config/scripts/verify-playwright-participation.mjs', 'src/main/wsl-availability.ts', 'src/main/wsl/wsl-runner.ts', 'src/main/pty/wsl-orca-env.ts',