orca/.github/workflows/computer-task-benchmark.yml

name: Computer-use task benchmark

on:
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
    paths:
      - 'config/benchmarks/computer-use/**'
      - 'native/computer-use-linux/**'
      - 'native/computer-use-windows/**'
      - '.github/workflows/computer-task-benchmark.yml'
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: computer-task-benchmark-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  ORCA_BACKGROUND_LAUNCH: '1'
  CUA_TAG: cua-driver-rs-v0.38.0
  CUA_DRIVER_RS_TELEMETRY_ENABLED: '0'
  DO_NOT_TRACK: '1'
  CUA_DRIVER_RS_UPDATE_CHECK: '0'
  ELECTRON_VERSION: '43.8.1'
  BASELINE_SHA: 030c88deecf3adc212d22f57b085e4bd5c6e48e6

jobs:
  contract:
    if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
    steps:
      - uses: actions/checkout@v6
        with:
          persist-credentials: false
      - uses: actions/setup-python@v6
        with:
          python-version: '3.12'
      - run: python -m unittest discover -s config/benchmarks/computer-use -p 'test_*.py'

  linux:
    if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
    runs-on: ubuntu-latest
    timeout-minutes: 40
    strategy:
      fail-fast: false
      matrix:
        target: [fixture, electron]
    env:
      BENCH_TARGET: ${{ matrix.target }}
      ORCA_COMPUTER_BENCH_ISOLATED: '1'
      ACCESSIBILITY_ENABLED: '1'
    steps:
      - uses: actions/checkout@v6
        with:
          persist-credentials: false
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
      - name: Fetch immutable baseline
        run: git fetch --no-tags --depth=1 origin "$BASELINE_SHA"
      - name: Install isolated desktop and accessibility libraries
        run: |
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends xvfb openbox dbus-x11 at-spi2-core python3-gi \
            gir1.2-atspi-2.0 gir1.2-gtk-3.0 python3-tk xdotool xclip ripgrep libnss3 libgbm1 libasound2t64
      - name: Download and verify pinned cua-driver
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          mkdir -p "$RUNNER_TEMP/cua-dl" "$RUNNER_TEMP/cua"
          cd "$RUNNER_TEMP/cua-dl"
          asset='cua-driver-rs-0.38.0-linux-x86_64-binary.tar.gz'
          gh release download "$CUA_TAG" --repo trycua/cua --pattern checksums.txt --pattern "$asset"
          rg " $asset\$" checksums.txt | sha256sum -c -
          tar -xzf "$asset" -C "$RUNNER_TEMP/cua"
      - name: Install Electron
        if: matrix.target == 'electron'
        run: |
          npm install --no-save --no-package-lock --prefix "$RUNNER_TEMP/electron" "electron@$ELECTRON_VERSION"
          node "$RUNNER_TEMP/electron/node_modules/electron/install.js"
      - name: Run task comparison on Xvfb
        run: |
          export CANDIDATE_SHA="$(git rev-parse HEAD)"
          xvfb-run -a --server-args='-screen 0 1280x1024x24' dbus-run-session -- bash -euc '
            openbox >"$RUNNER_TEMP/openbox.log" 2>&1 &
            /usr/bin/python3 config/benchmarks/computer-use/bench.py --target "$BENCH_TARGET" \
              --baseline "$BASELINE_SHA" --candidate "$CANDIDATE_SHA" \
              --cua "$RUNNER_TEMP/cua/cua-driver" \
              --electron "$RUNNER_TEMP/electron/node_modules/electron/dist/electron" \
              --out "out/task-linux-$BENCH_TARGET" --n 3 --budget 1800
            if [ "$BENCH_TARGET" = electron ]; then
              /usr/bin/python3 config/benchmarks/computer-use/linux_focus_probe.py \
                --electron "$RUNNER_TEMP/electron/node_modules/electron/dist/electron" \
                --baseline "$BASELINE_SHA" --candidate "$CANDIDATE_SHA" \
                --out "out/task-linux-$BENCH_TARGET/focus-probe"
            fi
          '
      - if: always()
        uses: actions/upload-artifact@v4
        with:
          name: computer-task-linux-${{ matrix.target }}
          path: out/task-linux-${{ matrix.target }}
          if-no-files-found: warn

  windows:
    if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
    runs-on: windows-latest
    timeout-minutes: 40
    strategy:
      fail-fast: false
      matrix:
        target: [fixture, electron]
    env:
      BENCH_TARGET: ${{ matrix.target }}
      ORCA_COMPUTER_BENCH_ISOLATED: '1'
    steps:
      - uses: actions/checkout@v6
        with:
          persist-credentials: false
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
      - uses: actions/setup-python@v6
        with:
          python-version: '3.12'
      - name: Fetch immutable baseline
        run: git fetch --no-tags --depth=1 origin $env:BASELINE_SHA
      - name: Download and verify pinned cua-driver
        env:
          GH_TOKEN: ${{ github.token }}
        shell: pwsh
        run: |
          $download = Join-Path $env:RUNNER_TEMP 'cua-dl'
          $destination = Join-Path $env:RUNNER_TEMP 'cua'
          New-Item -ItemType Directory -Force $download, $destination | Out-Null
          $asset = 'cua-driver-rs-0.38.0-windows-x86_64-binary.zip'
          gh release download $env:CUA_TAG --repo trycua/cua --pattern checksums.txt --pattern $asset --dir $download
          $matches = @(Select-String -Path (Join-Path $download 'checksums.txt') -Pattern " $([regex]::Escape($asset))$")
          if ($matches.Count -ne 1) { throw 'expected exactly one checksum' }
          $expected = $matches[0].Line.Split(' ')[0]
          $actual = (Get-FileHash (Join-Path $download $asset) -Algorithm SHA256).Hash.ToLower()
          if ($expected -ne $actual) { throw 'cua-driver checksum mismatch' }
          Expand-Archive (Join-Path $download $asset) -DestinationPath $destination
      - name: Install Electron
        if: matrix.target == 'electron'
        run: |
          npm install --no-save --no-package-lock --prefix "$env:RUNNER_TEMP\electron" "electron@$env:ELECTRON_VERSION"
          node "$env:RUNNER_TEMP\electron\node_modules\electron\install.js"
      - name: Run task comparison on disposable Windows desktop
        run: |
          $candidate = git rev-parse HEAD
          python config/benchmarks/computer-use/bench.py --target $env:BENCH_TARGET `
            --baseline $env:BASELINE_SHA --candidate $candidate `
            --cua "$env:RUNNER_TEMP\cua\cua-driver.exe" `
            --electron "$env:RUNNER_TEMP\electron\node_modules\electron\dist\electron.exe" `
            --out "out/task-windows-$env:BENCH_TARGET" --n 3 --budget 1800
      - if: always()
        uses: actions/upload-artifact@v4
        with:
          name: computer-task-windows-${{ matrix.target }}
          path: out/task-windows-${{ matrix.target }}
          if-no-files-found: warn