mirror of
https://github.com/stablyai/orca.git
synced 2026-09-30 00:03:15 +00:00
test: validate permanent packaged browser workflow and participation
This commit is contained in:
Neil
+50
-422
@@ -1,442 +1,70 @@
|
||||
name: E2E
|
||||
|
||||
run-name: E2E ${{ inputs.ref || github.ref }}
|
||||
|
||||
# Why: checkout + artifact upload only; callers can only further restrict.
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
name: Packaged browser compatibility
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '20 8 * * 1'
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
description: Ref to check out (defaults to the calling workflow's ref)
|
||||
required: false
|
||||
type: string
|
||||
test_files:
|
||||
description: JSON array of changed specs; empty runs the full suite
|
||||
required: false
|
||||
type: string
|
||||
ssh_source_changed:
|
||||
description: '"true" when the PR touches SSH execution source; gates the Docker-SSH lane'
|
||||
required: false
|
||||
type: string
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
ref:
|
||||
description: Ref to check out (defaults to the workflow ref)
|
||||
required: false
|
||||
type: string
|
||||
test_files:
|
||||
description: JSON array of specs to run; empty runs the full suite
|
||||
required: false
|
||||
type: string
|
||||
schedule:
|
||||
# Why: GitHub cron uses UTC; these slots map to 10am and 3pm
|
||||
# America/Phoenix for the default-branch E2E run.
|
||||
- cron: '0 17,22 * * *'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
build:
|
||||
name: build e2e app
|
||||
compatibility:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
# Why: the build's plain-Node daemon smoke load resolves node-pty.
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
persist-credentials: false
|
||||
- name: Install headless tools
|
||||
run: sudo apt-get update && sudo apt-get install -y build-essential openssh-client python3 ripgrep xvfb zsh openbox x11-utils
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
with:
|
||||
native-runtime: node
|
||||
|
||||
# Why: building here avoids parallel builds inside Playwright globalSetup;
|
||||
# paired-browser specs also need the standalone web bundle.
|
||||
- name: Build E2E outputs
|
||||
native-runtime: electron
|
||||
- name: Download pinned old release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
gh release download v1.4.188 --repo stablyai/orca --pattern orca-ide_1.4.188_amd64.deb --dir "$RUNNER_TEMP/old-orca"
|
||||
python3 - <<'PYVERIFY'
|
||||
import base64,hashlib,os,pathlib,subprocess
|
||||
root=pathlib.Path(os.environ['RUNNER_TEMP'])/'old-orca'
|
||||
package=root/'orca-ide_1.4.188_amd64.deb'
|
||||
expected='uGONFUDfinYggxcT9ac72wnnlofLQaqasDDeP0HWOSqarBwTi1Ax3khmzKUY3vUnvuYOpSCEmsH4InzLZ2vg6g=='
|
||||
assert base64.b64encode(hashlib.sha512(package.read_bytes()).digest()).decode()==expected
|
||||
extracted=root/'extracted'
|
||||
subprocess.run(['dpkg-deb','-x',str(package),str(extracted)],check=True)
|
||||
candidates=[extracted/'opt'/'Orca'/'orca-ide']
|
||||
assert candidates[0].is_file() and os.access(candidates[0],os.X_OK)
|
||||
assert len(candidates)==1,candidates
|
||||
with open(os.environ['GITHUB_ENV'],'a') as env: env.write('ORCA_CROSS_VERSION_PACKAGED_EXECUTABLE='+str(candidates[0])+'\n')
|
||||
print('Verified old package:',candidates[0])
|
||||
PYVERIFY
|
||||
- name: Build current Electron app
|
||||
env:
|
||||
VITE_EXPOSE_STORE: 'true'
|
||||
run: |
|
||||
status=0
|
||||
pnpm run build:relay &
|
||||
relay_pid=$!
|
||||
npx electron-vite build --mode e2e || status=1
|
||||
pnpm run build:web-from-renderer || status=1
|
||||
wait "$relay_pid" || status=1
|
||||
exit "$status"
|
||||
|
||||
- name: Upload E2E build output
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: e2e-build-out
|
||||
path: out/
|
||||
# Why: build-relay.mjs writes each relay's marker as `out/relay/<platform>/.version`,
|
||||
# and upload-artifact drops dotfiles by default — consumers then fail SSH specs with
|
||||
# "local relay build is missing its version marker".
|
||||
include-hidden-files: true
|
||||
retention-days: 1
|
||||
if-no-files-found: error
|
||||
|
||||
# Build Electron-native dependencies once per workflow. Consumer shards restore
|
||||
# this immutable cache instead of compiling the same ABI concurrently.
|
||||
prepare-native-cache:
|
||||
name: prepare Electron native cache
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Install native build tools
|
||||
run: sudo apt-get update && sudo apt-get install -y build-essential python3
|
||||
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
with:
|
||||
native-runtime: electron
|
||||
|
||||
e2e:
|
||||
name: e2e ${{ matrix.shard_name }}
|
||||
needs: [build, prepare-native-cache]
|
||||
if: inputs.test_files == ''
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
# Fourteen scheduled runs averaged 24.6 minutes per shard; shards 4
|
||||
# and 9 repeatedly hit the 30-minute cap. A 12-way trial still left
|
||||
# one 30-minute shard, so 14 gives the suite enough failure headroom.
|
||||
- shard: '1/14'
|
||||
shard_name: 1-of-14
|
||||
- shard: '2/14'
|
||||
shard_name: 2-of-14
|
||||
- shard: '3/14'
|
||||
shard_name: 3-of-14
|
||||
- shard: '4/14'
|
||||
shard_name: 4-of-14
|
||||
- shard: '5/14'
|
||||
shard_name: 5-of-14
|
||||
- shard: '6/14'
|
||||
shard_name: 6-of-14
|
||||
- shard: '7/14'
|
||||
shard_name: 7-of-14
|
||||
- shard: '8/14'
|
||||
shard_name: 8-of-14
|
||||
- shard: '9/14'
|
||||
shard_name: 9-of-14
|
||||
- shard: '10/14'
|
||||
shard_name: 10-of-14
|
||||
- shard: '11/14'
|
||||
shard_name: 11-of-14
|
||||
- shard: '12/14'
|
||||
shard_name: 12-of-14
|
||||
- shard: '13/14'
|
||||
shard_name: 13-of-14
|
||||
- shard: '14/14'
|
||||
shard_name: 14-of-14
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
# Native cache misses need the compiler, Electron needs Xvfb, and paired
|
||||
# Quick Open needs ripgrep. Install them in one apt transaction per shard.
|
||||
- name: Install native build and headless UI tools
|
||||
run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk python3 ripgrep xvfb zsh openbox x11-utils
|
||||
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
with:
|
||||
native-runtime: electron
|
||||
|
||||
- name: Download E2E build output
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: e2e-build-out
|
||||
path: out/
|
||||
|
||||
# Why: the Electron suite is wall-clock constrained on OSS runners, but
|
||||
# multiple Electron apps on one Xvfb VM contend on git/Chromium resources.
|
||||
# Sharding keeps each VM at one Playwright worker while splitting the
|
||||
# headless suite across separate runners.
|
||||
# SKIP_BUILD makes Playwright globalSetup reuse the single build job's
|
||||
# artifact instead of starting five concurrent electron-vite builds.
|
||||
# ORCA_E2E_FORWARD_APP_LOGS keeps startup failures visible when Electron
|
||||
# launches but never creates a BrowserWindow.
|
||||
- name: Run E2E tests (${{ matrix.shard_name }})
|
||||
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1 ORCA_RELAY_PATH="$GITHUB_WORKSPACE/out/relay" pnpm run test:e2e --shard=${{ matrix.shard }}
|
||||
|
||||
# Why: Playwright retains traces/screenshots only on failure. Uploading
|
||||
# them as an artifact makes post-mortem debugging on CI possible without
|
||||
# re-running locally.
|
||||
- name: Upload Playwright traces
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: playwright-traces-${{ matrix.shard_name }}
|
||||
path: test-results/
|
||||
retention-days: 7
|
||||
if-no-files-found: ignore
|
||||
|
||||
changed-e2e:
|
||||
name: changed e2e specs
|
||||
needs: [build, prepare-native-cache]
|
||||
if: inputs.test_files != ''
|
||||
runs-on: ubuntu-latest
|
||||
# Why 45: pr.yml now maps SSH source edits onto Docker-backed specs, so this lane can
|
||||
# pay a container image build plus ~22 serial SSH tests on top of the changed specs.
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Install native build and headless UI tools
|
||||
# Why ripgrep: Quick Open's bounded host-side search requires rg instead of an
|
||||
# unbounded inventory fallback; the paired fixture exercises that real boundary.
|
||||
# Why openssh-client: the Docker-SSH fixture shells out to ssh/ssh-keygen, and this
|
||||
# lane now receives those specs from pr.yml's SSH source mapping.
|
||||
run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk openssh-client python3 ripgrep xvfb zsh openbox x11-utils
|
||||
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
with:
|
||||
native-runtime: electron
|
||||
|
||||
- name: Download E2E build output
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: e2e-build-out
|
||||
path: out/
|
||||
|
||||
- name: Run changed E2E specs
|
||||
pnpm run build:relay
|
||||
pnpm exec electron-vite build --mode e2e
|
||||
pnpm run build:web-from-renderer
|
||||
- name: Run both mixed-version directions
|
||||
env:
|
||||
TEST_FILES_JSON: ${{ inputs.test_files }}
|
||||
run: |
|
||||
# Why the native IME spec is dropped: it test.skip()s itself without
|
||||
# ORCA_E2E_NATIVE_IBUS_HANGUL, which this lane cannot set because it has no ibus
|
||||
# session. Running it here reported a green skip as coverage.
|
||||
mapfile -t TEST_FILES < <(jq -r '.[] | select(
|
||||
. != "tests/e2e/ssh-startup-exec-readiness.spec.ts" and
|
||||
. != "tests/e2e/paired-startup-exec-readiness.spec.ts" and
|
||||
. != "tests/e2e/local-ssh-browser-routing.spec.ts" and
|
||||
. != "tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts" and
|
||||
. != "tests/e2e/ssh-localhost.spec.ts" and
|
||||
. != "tests/e2e/ssh-client-hosted-browser-drop-reconnect.spec.ts" and
|
||||
. != "tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts" and
|
||||
. != "tests/e2e/terminal-ibus-hangul-native.spec.ts"
|
||||
)' <<<"$TEST_FILES_JSON")
|
||||
if [ "${#TEST_FILES[@]}" -eq 0 ]; then
|
||||
echo "Changed specs are all owned by dedicated lanes."
|
||||
exit 0
|
||||
fi
|
||||
E2E_ENV=(SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1 ORCA_RELAY_PATH="$GITHUB_WORKSPACE/out/relay")
|
||||
# Second clause: a spec that reads ORCA_E2E_SSH_DOCKER test.skip()s itself without it, so
|
||||
# naming only one trigger silently skipped every other Docker-SSH spec in this lane.
|
||||
# The first clause stays because that spec needs Docker without referencing the variable.
|
||||
if printf '%s\n' "${TEST_FILES[@]}" | grep -qx 'tests/e2e/ephemeral-vm-provisioned-root.spec.ts' \
|
||||
|| grep -l 'ORCA_E2E_SSH_DOCKER' "${TEST_FILES[@]}" >/dev/null 2>&1; then
|
||||
E2E_ENV+=(ORCA_E2E_SSH_DOCKER=1)
|
||||
fi
|
||||
E2E_PROJECT_ARGS=()
|
||||
if grep -l '@headful' "${TEST_FILES[@]}" >/dev/null; then
|
||||
E2E_PROJECT_ARGS+=(--project=electron-headful)
|
||||
fi
|
||||
xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env "${E2E_ENV[@]}" \
|
||||
pnpm run test:e2e "${TEST_FILES[@]}" --workers=1 "${E2E_PROJECT_ARGS[@]}"
|
||||
|
||||
- name: Upload Playwright traces
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: playwright-traces-changed
|
||||
path: test-results/
|
||||
retention-days: 7
|
||||
if-no-files-found: ignore
|
||||
|
||||
ssh-docker-watcher-isolation:
|
||||
name: ssh docker watcher isolation
|
||||
needs: [build, prepare-native-cache]
|
||||
# effect of one route listing a startup-readiness spec — pruning that spec would have
|
||||
# silently retired the whole lane. The signal is now derived from the SSH routes directly.
|
||||
# The explicit spec clauses stay for their honest purpose: changed-e2e hands these specs to this
|
||||
# lane, so editing one must still run it here.
|
||||
if: >-
|
||||
inputs.test_files == '' ||
|
||||
inputs.ssh_source_changed == 'true' ||
|
||||
contains(inputs.test_files, 'tests/e2e/local-ssh-browser-routing.spec.ts') ||
|
||||
contains(inputs.test_files, 'tests/e2e/ssh-client-hosted-browser-drop-reconnect.spec.ts') ||
|
||||
contains(inputs.test_files, 'tests/e2e/ssh-startup-exec-readiness.spec.ts') ||
|
||||
contains(inputs.test_files, 'tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts') ||
|
||||
contains(inputs.test_files, 'tests/e2e/paired-startup-exec-readiness.spec.ts')
|
||||
runs-on: ubuntu-latest
|
||||
# Why 60: this lane now also runs the remaining Docker-SSH specs serially. They average
|
||||
# ~18s but several budget 4-10 minutes per test, so a slow run lands far above the old 35
|
||||
# — and the sharded lanes already show that a lane which times out is a lane nobody trusts.
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Install native build and headless UI tools
|
||||
run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk openssh-client python3 ripgrep xvfb zsh openbox x11-utils
|
||||
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
with:
|
||||
native-runtime: electron
|
||||
|
||||
- name: Download E2E build output
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: e2e-build-out
|
||||
path: out/
|
||||
|
||||
# Why: this is the release-path proof that the deployed Linux relay keeps
|
||||
# its PTY and explorer live across a real watcher SIGSEGV.
|
||||
- name: Run Docker SSH watcher isolation E2E
|
||||
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker-watcher-isolation
|
||||
|
||||
# Why: Playwright empties test-results/ when it starts, so each step here used to
|
||||
# destroy the previous step's traces. Only the last lane's failure was ever
|
||||
# diagnosable from the artifact; set each lane aside before the next one runs.
|
||||
- name: Keep watcher-isolation traces
|
||||
PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/packaged-browser-results.json
|
||||
run: >-
|
||||
xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh
|
||||
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1
|
||||
pnpm exec playwright test --config tests/playwright.config.ts
|
||||
tests/e2e/packaged-mixed-version-browser-placement.spec.ts
|
||||
--project=electron-headless --workers=1 --retries=0 --repeat-each=3 --reporter=list,json
|
||||
- name: Require all six compatibility executions
|
||||
if: always()
|
||||
run: |
|
||||
if [ -d test-results ]; then
|
||||
mkdir -p e2e-traces
|
||||
mv test-results "e2e-traces/watcher-isolation"
|
||||
fi
|
||||
|
||||
# Why always(): this lane gates SSH parking/retention plus startup-exec
|
||||
# readiness across live SSH, headed paired, and headless serve topologies.
|
||||
- name: Run Docker SSH terminal parking + startup readiness E2E
|
||||
if: always()
|
||||
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker-terminal-parking
|
||||
|
||||
- name: Keep terminal-parking traces
|
||||
if: always()
|
||||
run: |
|
||||
if [ -d test-results ]; then
|
||||
mkdir -p e2e-traces
|
||||
mv test-results "e2e-traces/terminal-parking"
|
||||
fi
|
||||
|
||||
# Why here rather than the sharded lanes: the shards set no ORCA_E2E_SSH_DOCKER, so every
|
||||
# spec below skipped itself while the shard still reported green. Running them on this one
|
||||
# VM pays the fixture image build once instead of ten times, and keeps an SSH regression
|
||||
# legible as an SSH-named failure.
|
||||
- name: Run remaining Docker SSH E2E
|
||||
if: always()
|
||||
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker
|
||||
|
||||
- name: Keep remaining-ssh-docker traces
|
||||
if: always()
|
||||
run: |
|
||||
if [ -d test-results ]; then
|
||||
mkdir -p e2e-traces
|
||||
mv test-results "e2e-traces/remaining-ssh-docker"
|
||||
fi
|
||||
|
||||
- name: Upload watcher isolation traces
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: playwright-traces-ssh-docker-watcher-isolation
|
||||
path: e2e-traces/
|
||||
retention-days: 7
|
||||
if-no-files-found: ignore
|
||||
|
||||
ssh-browser-network-route:
|
||||
name: ssh browser network route
|
||||
if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
with:
|
||||
native-runtime: node
|
||||
- name: Install SSH client
|
||||
run: sudo apt-get update && sudo apt-get install -y openssh-client
|
||||
- name: Run Docker SSH browser network route journeys
|
||||
env:
|
||||
ORCA_BACKGROUND_LAUNCH: '1'
|
||||
ORCA_RUN_DOCKER_SSH_BROWSER_E2E: '1'
|
||||
run: node_modules/.bin/vitest run --config config/vitest.config.ts tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts
|
||||
|
||||
ssh-localhost:
|
||||
name: localhost SSH terminal and hooks
|
||||
needs: [build, prepare-native-cache]
|
||||
if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/ssh-localhost.spec.ts')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
- name: Install SSH server and headless tools
|
||||
run: sudo apt-get update && sudo apt-get install -y build-essential openssh-client openssh-server python3 ripgrep xvfb zsh openbox x11-utils
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
with:
|
||||
native-runtime: electron
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: e2e-build-out
|
||||
path: out/
|
||||
- name: Start isolated localhost SSH server
|
||||
shell: bash
|
||||
run: |
|
||||
# Bare shells install Pi extensions only for an existing agent home.
|
||||
mkdir -p "$HOME/.pi/agent"
|
||||
fixture="$RUNNER_TEMP/orca-localhost-sshd"
|
||||
mkdir -p "$fixture"
|
||||
ssh-keygen -q -t ed25519 -N '' -f "$fixture/host_key"
|
||||
ssh-keygen -q -t ed25519 -N '' -f "$fixture/client_key"
|
||||
cat > "$fixture/sshd_config" <<EOF
|
||||
Port 22222
|
||||
ListenAddress 127.0.0.1
|
||||
HostKey $fixture/host_key
|
||||
PidFile $fixture/sshd.pid
|
||||
AuthorizedKeysFile $fixture/client_key.pub
|
||||
StrictModes no
|
||||
PasswordAuthentication no
|
||||
KbdInteractiveAuthentication no
|
||||
UsePAM yes
|
||||
AllowUsers $(id -un)
|
||||
Subsystem sftp internal-sftp
|
||||
EOF
|
||||
sudo mkdir -p /run/sshd
|
||||
sudo /usr/sbin/sshd -f "$fixture/sshd_config" -E "$fixture/sshd.log"
|
||||
ssh -i "$fixture/client_key" -p 22222 -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null 127.0.0.1 true || { sudo cat "$fixture/sshd.log"; exit 1; }
|
||||
{
|
||||
echo "ORCA_E2E_SSH_PORT=22222"
|
||||
echo "ORCA_E2E_SSH_USER=$(id -un)"
|
||||
echo "ORCA_E2E_SSH_IDENTITY_FILE=$fixture/client_key"
|
||||
} >> "$GITHUB_ENV"
|
||||
- name: Run localhost SSH terminal and hook journey
|
||||
env:
|
||||
SKIP_BUILD: '1'
|
||||
ORCA_E2E_SSH_LOCALHOST: '1'
|
||||
ORCA_FEATURE_REMOTE_AGENT_HOOKS: '1'
|
||||
ORCA_E2E_FORWARD_APP_LOGS: '1'
|
||||
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-localhost.spec.ts --project=electron-headless --workers=1
|
||||
run: node config/scripts/verify-packaged-browser-participation.mjs test-results/packaged-browser-results.json
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: failure()
|
||||
if: always()
|
||||
with:
|
||||
name: localhost-ssh-traces
|
||||
name: packaged-mixed-version-audit
|
||||
path: test-results/
|
||||
retention-days: 7
|
||||
if-no-files-found: ignore
|
||||
retention-days: 3
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
name: Packaged browser compatibility
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '20 8 * * 1'
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
type: string
|
||||
required: false
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
compatibility:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
persist-credentials: false
|
||||
- name: Install headless tools
|
||||
run: sudo apt-get update && sudo apt-get install -y build-essential openssh-client python3 ripgrep xvfb zsh openbox x11-utils
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
with:
|
||||
native-runtime: electron
|
||||
- name: Download pinned old release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
gh release download v1.4.188 --repo stablyai/orca --pattern orca-ide_1.4.188_amd64.deb --dir "$RUNNER_TEMP/old-orca"
|
||||
python3 - <<'PYVERIFY'
|
||||
import base64,hashlib,os,pathlib,subprocess
|
||||
root=pathlib.Path(os.environ['RUNNER_TEMP'])/'old-orca'
|
||||
package=root/'orca-ide_1.4.188_amd64.deb'
|
||||
expected='uGONFUDfinYggxcT9ac72wnnlofLQaqasDDeP0HWOSqarBwTi1Ax3khmzKUY3vUnvuYOpSCEmsH4InzLZ2vg6g=='
|
||||
assert base64.b64encode(hashlib.sha512(package.read_bytes()).digest()).decode()==expected
|
||||
extracted=root/'extracted'
|
||||
subprocess.run(['dpkg-deb','-x',str(package),str(extracted)],check=True)
|
||||
candidates=[extracted/'opt'/'Orca'/'orca-ide']
|
||||
assert candidates[0].is_file() and os.access(candidates[0],os.X_OK)
|
||||
assert len(candidates)==1,candidates
|
||||
with open(os.environ['GITHUB_ENV'],'a') as env: env.write('ORCA_CROSS_VERSION_PACKAGED_EXECUTABLE='+str(candidates[0])+'\n')
|
||||
print('Verified old package:',candidates[0])
|
||||
PYVERIFY
|
||||
- name: Build current Electron app
|
||||
env:
|
||||
VITE_EXPOSE_STORE: 'true'
|
||||
run: |
|
||||
pnpm run build:relay
|
||||
pnpm exec electron-vite build --mode e2e
|
||||
pnpm run build:web-from-renderer
|
||||
- name: Run both mixed-version directions
|
||||
env:
|
||||
PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/packaged-browser-results.json
|
||||
run: >-
|
||||
xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh
|
||||
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1
|
||||
pnpm exec playwright test --config tests/playwright.config.ts
|
||||
tests/e2e/packaged-mixed-version-browser-placement.spec.ts
|
||||
--project=electron-headless --workers=1 --retries=0 --repeat-each=3 --reporter=list,json
|
||||
- name: Require all six compatibility executions
|
||||
if: always()
|
||||
run: node config/scripts/verify-packaged-browser-participation.mjs test-results/packaged-browser-results.json
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: packaged-mixed-version-audit
|
||||
path: test-results/
|
||||
retention-days: 3
|
||||
@@ -18472,6 +18472,107 @@
|
||||
"The new PR lane is outside verify until reliability is established."
|
||||
],
|
||||
"demotionRule": "Keep experimental if provisioning or an execution flakes; never promote by skipping a case, raising timeouts, or retrying until green."
|
||||
},
|
||||
{
|
||||
"id": "browser.packaged-mixed-version-placement",
|
||||
"title": "Packaged browser placement across versions",
|
||||
"maturity": "experimental",
|
||||
"protection": "partial",
|
||||
"owner": "browser-runtime",
|
||||
"layer": "electron-packaged",
|
||||
"surfaces": [
|
||||
"paired browser placement"
|
||||
],
|
||||
"platforms": [
|
||||
"linux",
|
||||
"macos",
|
||||
"windows"
|
||||
],
|
||||
"providers": [
|
||||
"paired-runtime"
|
||||
],
|
||||
"coveredPlatforms": [
|
||||
"linux"
|
||||
],
|
||||
"coveredProviders": [
|
||||
"paired-runtime"
|
||||
],
|
||||
"coverageNotes": "Published Linux 1.4.188 desktop against current source in both directions; scheduled weekly and manually runnable. No required PR check.",
|
||||
"motivatingLinks": [
|
||||
"https://github.com/stablyai/orca/actions/runs/34069063016"
|
||||
],
|
||||
"invariant": "A paired client and host without client-hosted browser capabilities retain server-hosted browser placement across supported version skew.",
|
||||
"oracle": "Require both existing named browser placement scenarios to pass three times with one attempt, zero skips, zero failures, and no report errors.",
|
||||
"commands": [
|
||||
"gh workflow run packaged-browser-e2e.yml",
|
||||
"pnpm exec playwright test tests/e2e/packaged-mixed-version-browser-placement.spec.ts --config tests/playwright.config.ts --project=electron-headless --workers=1 --repeat-each=3 --retries=0",
|
||||
"node_modules/.bin/vitest run --config config/vitest.config.ts config/scripts/packaged-browser-lane-contract.test.mjs config/scripts/verify-packaged-browser-participation.test.mjs",
|
||||
"gh run view 34069063016 --log"
|
||||
],
|
||||
"testFiles": [
|
||||
"tests/e2e/packaged-mixed-version-browser-placement.spec.ts",
|
||||
"config/scripts/packaged-browser-lane-contract.test.mjs",
|
||||
"config/scripts/verify-packaged-browser-participation.test.mjs"
|
||||
],
|
||||
"assertionRefs": [
|
||||
{
|
||||
"file": "tests/e2e/packaged-mixed-version-browser-placement.spec.ts",
|
||||
"assertions": [
|
||||
"old client and old host lack client-host and browser-tunnel capabilities",
|
||||
"browser contents remain owned by the server and the expected snapshot marker is readable"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "config/scripts/verify-packaged-browser-participation.test.mjs",
|
||||
"assertions": [
|
||||
"reject missing, substituted, skipped and retried scenarios"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "config/scripts/packaged-browser-lane-contract.test.mjs",
|
||||
"assertions": [
|
||||
"verify pinned package checksum before extraction",
|
||||
"require both directions three times and run report verification even on failure"
|
||||
]
|
||||
}
|
||||
],
|
||||
"evidenceRuns": [
|
||||
{
|
||||
"date": "2026-09-07",
|
||||
"runner": "ci",
|
||||
"platform": "linux",
|
||||
"result": "passed",
|
||||
"command": "gh run view 34069063016 --log",
|
||||
"durationSeconds": 120,
|
||||
"summary": "Both unmodified compatibility cases passed three times at 5a99f935 with published1.4.188 and main f7d52160162; retries0. Final permanent workflow verification remains pending."
|
||||
}
|
||||
],
|
||||
"runtimeBudget": {
|
||||
"p95Seconds": 1500,
|
||||
"scope": "CI job timeout; not a measured p95"
|
||||
},
|
||||
"flakeHistory": {
|
||||
"status": "soaking",
|
||||
"evidence": "Initial executable discovery matched CLI and desktop and was corrected before any tests ran. Corrected baseline2/2 and repeat6/6 pass."
|
||||
},
|
||||
"redGreenEvidence": {
|
||||
"status": "partial",
|
||||
"evidence": "Participation unit tests reject missing and retried scenarios; no application mutation proof."
|
||||
},
|
||||
"performanceBudget": {
|
||||
"required": false,
|
||||
"evidence": "Compatibility assertions, not a performance benchmark."
|
||||
},
|
||||
"promotionCriteria": [
|
||||
"Final workflow JSON report proves all six executions.",
|
||||
"Collect repeated scheduled history before making this required."
|
||||
],
|
||||
"knownGaps": [
|
||||
"Linux1.4.188 only; no macOS or Windows packaged coverage.",
|
||||
"No folder workspace, SSH execution host or live-service coverage.",
|
||||
"Other released version pairs remain untested; not a required PR check."
|
||||
],
|
||||
"demotionRule": "Keep experimental if any direction skips or fails; do not extend timeouts or retry to green."
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { parse } from 'yaml'
|
||||
|
||||
const workflow = parse(
|
||||
readFileSync(new URL('../../.github/workflows/packaged-browser-e2e.yml', import.meta.url), 'utf8')
|
||||
)
|
||||
const steps = workflow.jobs.compatibility.steps
|
||||
|
||||
describe('packaged browser compatibility lane', () => {
|
||||
it('runs weekly and supports immutable manual or reusable revisions', () => {
|
||||
expect(workflow.on.schedule).toHaveLength(1)
|
||||
expect(workflow.on).toHaveProperty('workflow_dispatch')
|
||||
expect(steps[0].with.ref).toBe('${{ inputs.ref || github.sha }}')
|
||||
expect(workflow.permissions).toEqual({ contents: 'read' })
|
||||
})
|
||||
|
||||
it('verifies the pinned package before selecting the desktop executable', () => {
|
||||
const download = steps.find((step) => step.name === 'Download pinned old release').run
|
||||
expect(download).toContain('gh release download v1.4.188')
|
||||
expect(download).toContain('hashlib.sha512(package.read_bytes())')
|
||||
expect(download).toContain("extracted/'opt'/'Orca'/'orca-ide'")
|
||||
expect(download.indexOf('assert base64.')).toBeLessThan(download.indexOf("['dpkg-deb'"))
|
||||
})
|
||||
|
||||
it('requires both directions three times and rejects silent skips', () => {
|
||||
const run = steps.find((step) => step.name === 'Run both mixed-version directions')
|
||||
expect(run.run).toContain('tests/e2e/packaged-mixed-version-browser-placement.spec.ts')
|
||||
expect(run.run).toContain('--repeat-each=3')
|
||||
expect(run.run).toContain('--retries=0')
|
||||
expect(run.run).toContain('--reporter=list,json')
|
||||
const verify = steps.find((step) => step.name === 'Require all six compatibility executions')
|
||||
expect(verify.if).toBe('always()')
|
||||
expect(verify.run).toBe(
|
||||
`node config/scripts/verify-packaged-browser-participation.mjs ${run.env.PLAYWRIGHT_JSON_OUTPUT_FILE}`
|
||||
)
|
||||
expect(steps.at(-1).if).toBe('always()')
|
||||
expect(steps.at(-1).with.path).toBe('test-results/')
|
||||
})
|
||||
})
|
||||
@@ -41,7 +41,7 @@ export const PR_E2E_SOURCE_ROUTES = [
|
||||
],
|
||||
matches: (file) =>
|
||||
isProductSource(file) &&
|
||||
/^(?:config\/scripts\/verify-wsl-e2e-participation\.mjs$|src\/main\/(?:wsl[/-]|pty\/.*wsl|providers\/wsl)|src\/shared\/(?:wsl-|windows-terminal-shell)|src\/renderer\/src\/.*(?:terminal-paste|pty-paste)|tests\/e2e\/(?:golden-tab-bar-agent-launch\.spec|terminal-windows-shell-paste-ownership\.spec|helpers\/(?:wsl-golden-stub-agent|golden-stub-agent))|\.github\/(?:actions\/setup-wsl-test-runtime\/|workflows\/windows-wsl-e2e\.yml))/.test(
|
||||
/^(?:config\/scripts\/(?:verify-wsl-e2e-participation|verify-playwright-participation)\.mjs$|src\/main\/(?:wsl[/-]|pty\/.*wsl|providers\/wsl)|src\/shared\/(?:wsl-|windows-terminal-shell)|src\/renderer\/src\/.*(?:terminal-paste|pty-paste)|tests\/e2e\/(?:golden-tab-bar-agent-launch\.spec|terminal-windows-shell-paste-ownership\.spec|helpers\/(?:wsl-golden-stub-agent|golden-stub-agent))|\.github\/(?:actions\/setup-wsl-test-runtime\/|workflows\/windows-wsl-e2e\.yml))/.test(
|
||||
file
|
||||
)
|
||||
},
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { verifyPlaywrightParticipation } from './verify-playwright-participation.mjs'
|
||||
|
||||
export const PACKAGED_BROWSER_TEST_TITLES = [
|
||||
'keeps an old packaged client on the current server-hosted path',
|
||||
'keeps a current client on an old packaged server-hosted path'
|
||||
]
|
||||
|
||||
export function verifyPackagedBrowserParticipation(report) {
|
||||
verifyPlaywrightParticipation(report, {
|
||||
titles: PACKAGED_BROWSER_TEST_TITLES,
|
||||
label: 'Packaged browser'
|
||||
})
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
verifyPackagedBrowserParticipation(JSON.parse(readFileSync(process.argv[2], 'utf8')))
|
||||
console.log('Both packaged browser directions passed three times without skips or retries.')
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
verifyPackagedBrowserParticipation,
|
||||
PACKAGED_BROWSER_TEST_TITLES
|
||||
} from './verify-packaged-browser-participation.mjs'
|
||||
|
||||
function report() {
|
||||
return {
|
||||
stats: { expected: 6, skipped: 0, unexpected: 0, flaky: 0 },
|
||||
suites: [
|
||||
{
|
||||
suites: [
|
||||
{
|
||||
specs: PACKAGED_BROWSER_TEST_TITLES.map((title) => ({
|
||||
title,
|
||||
tests: Array.from({ length: 3 }, () => ({
|
||||
expectedStatus: 'passed',
|
||||
results: [{ status: 'passed' }]
|
||||
}))
|
||||
}))
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
describe('Packaged browser participation', () => {
|
||||
it('accepts both named scenarios executed three times', () => {
|
||||
expect(() => verifyPackagedBrowserParticipation(report())).not.toThrow()
|
||||
})
|
||||
it.each(['skipped', 'unexpected', 'flaky'])('rejects a nonzero %s result', (key) => {
|
||||
const value = report()
|
||||
value.stats[key] = 1
|
||||
expect(() => verifyPackagedBrowserParticipation(value)).toThrow('participation failed')
|
||||
})
|
||||
it('rejects missing scenarios even when aggregate counts claim six passes', () => {
|
||||
const value = report()
|
||||
value.suites[0].suites[0].specs.pop()
|
||||
expect(() => verifyPackagedBrowserParticipation(value)).toThrow('requires three executions')
|
||||
})
|
||||
it('rejects an unrelated scenario substituted for an expected scenario', () => {
|
||||
const value = report()
|
||||
value.suites[0].suites[0].specs[0].title = 'native shell passes'
|
||||
expect(() => verifyPackagedBrowserParticipation(value)).toThrow(
|
||||
'Unexpected Packaged browser scenario'
|
||||
)
|
||||
})
|
||||
it('rejects a pass obtained after a failed attempt', () => {
|
||||
const value = report()
|
||||
value.suites[0].suites[0].specs[0].tests[0].results.unshift({ status: 'failed' })
|
||||
expect(() => verifyPackagedBrowserParticipation(value)).toThrow('without retries')
|
||||
})
|
||||
it('rejects missing report content', () => {
|
||||
expect(() => verifyPackagedBrowserParticipation({})).toThrow('participation failed')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,42 @@
|
||||
export function verifyPlaywrightParticipation(report, { titles, label, repetitions = 3 }) {
|
||||
const stats = report?.stats
|
||||
if (
|
||||
!stats ||
|
||||
stats.expected !== titles.length * repetitions ||
|
||||
stats.skipped !== 0 ||
|
||||
stats.unexpected !== 0 ||
|
||||
stats.flaky !== 0 ||
|
||||
report.errors?.length
|
||||
) {
|
||||
throw new Error(`${label} participation failed: ${JSON.stringify(stats)}`)
|
||||
}
|
||||
const counts = new Map(titles.map((title) => [title, 0]))
|
||||
const visit = (suites) => {
|
||||
for (const suite of suites ?? []) {
|
||||
for (const spec of suite.specs ?? []) {
|
||||
if (!counts.has(spec.title)) {
|
||||
throw new Error(`Unexpected ${label} scenario: ${spec.title}`)
|
||||
}
|
||||
for (const test of spec.tests ?? []) {
|
||||
if (
|
||||
test.expectedStatus !== 'passed' ||
|
||||
test.results?.length !== 1 ||
|
||||
test.results[0].status !== 'passed'
|
||||
) {
|
||||
throw new Error(`${label} scenario did not pass without retries: ${spec.title}`)
|
||||
}
|
||||
counts.set(spec.title, counts.get(spec.title) + 1)
|
||||
}
|
||||
}
|
||||
visit(suite.suites)
|
||||
}
|
||||
}
|
||||
visit(report.suites)
|
||||
for (const [title, count] of counts) {
|
||||
if (count !== repetitions) {
|
||||
throw new Error(
|
||||
`${label} scenario requires ${repetitions === 3 ? 'three' : repetitions} executions: ${title} (${count})`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { verifyPlaywrightParticipation } from './verify-playwright-participation.mjs'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
@@ -8,44 +9,7 @@ export const WSL_TEST_TITLES = [
|
||||
]
|
||||
|
||||
export function verifyWslParticipation(report) {
|
||||
const stats = report?.stats
|
||||
if (
|
||||
!stats ||
|
||||
stats.expected !== 9 ||
|
||||
stats.skipped !== 0 ||
|
||||
stats.unexpected !== 0 ||
|
||||
stats.flaky !== 0 ||
|
||||
report.errors?.length
|
||||
) {
|
||||
throw new Error(`WSL participation failed: ${JSON.stringify(stats)}`)
|
||||
}
|
||||
const counts = new Map(WSL_TEST_TITLES.map((title) => [title, 0]))
|
||||
const visit = (suites) => {
|
||||
for (const suite of suites ?? []) {
|
||||
for (const spec of suite.specs ?? []) {
|
||||
if (!counts.has(spec.title)) {
|
||||
throw new Error(`Unexpected WSL scenario: ${spec.title}`)
|
||||
}
|
||||
for (const test of spec.tests ?? []) {
|
||||
if (
|
||||
test.expectedStatus !== 'passed' ||
|
||||
test.results?.length !== 1 ||
|
||||
test.results[0].status !== 'passed'
|
||||
) {
|
||||
throw new Error(`WSL scenario did not pass without retries: ${spec.title}`)
|
||||
}
|
||||
counts.set(spec.title, counts.get(spec.title) + 1)
|
||||
}
|
||||
}
|
||||
visit(suite.suites)
|
||||
}
|
||||
}
|
||||
visit(report.suites)
|
||||
for (const [title, count] of counts) {
|
||||
if (count !== 3) {
|
||||
throw new Error(`WSL scenario requires three executions: ${title} (${count})`)
|
||||
}
|
||||
}
|
||||
verifyPlaywrightParticipation(report, { titles: WSL_TEST_TITLES, label: 'WSL' })
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
|
||||
@@ -8,6 +8,7 @@ const read = (path) => readFileSync(new URL(`../../${path}`, import.meta.url), '
|
||||
describe('real WSL terminal lane', () => {
|
||||
it.each([
|
||||
'config/scripts/verify-wsl-e2e-participation.mjs',
|
||||
'config/scripts/verify-playwright-participation.mjs',
|
||||
'src/main/wsl-availability.ts',
|
||||
'src/main/wsl/wsl-runner.ts',
|
||||
'src/main/pty/wsl-orca-env.ts',
|
||||
|
||||
Reference in New Issue
Block a user