Files
moli/.github/workflows/ci.yml
T
ldm0 83e7091c5a fix(cdp): enforce DOM node publication across commands
Separate backend identity, session frontend bindings, and publication. Publish missing Inspector paths before returning usable node IDs, validate CDP frontend inputs, and retain explicit internal automation bindings.

Preserve expanded and associated subtrees during refresh and mutation resync, including shallow template fragments. Cover cross-command behavior and message ordering with a raw CDP client mirror, Chromium comparisons, and real chromedp queries.
2026-10-29 00:01:33 +08:00

1047 lines
40 KiB
YAML

name: CI
on:
pull_request:
types: [opened, synchronize, reopened]
push:
branches:
- main
workflow_dispatch:
inputs:
base_ref:
description: Base commit or ref (defaults to the previous commit)
required: false
default: ""
head_ref:
description: Head commit or ref (defaults to the dispatched commit)
required: false
default: ""
# Superseded main pushes share a group; PR and manual runs stay independent.
concurrency:
group: ci-${{ github.event_name == 'push' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'push' }}
permissions:
contents: read
env:
CARGO_TERM_COLOR: always
CARGO_INCREMENTAL: "0"
GIT_CONFIG_COUNT: "1"
GIT_CONFIG_KEY_0: safe.directory
GIT_CONFIG_VALUE_0: ${{ github.workspace }}
jobs:
resolve-refs:
name: Resolve exact revisions
runs-on: ubuntu-latest
outputs:
base_sha: ${{ steps.refs.outputs.base_sha }}
head_sha: ${{ steps.refs.outputs.head_sha }}
execution_order: ${{ steps.refs.outputs.execution_order }}
steps:
- name: Checkout exact HEAD
uses: actions/checkout@v4
with:
fetch-depth: 0
persist-credentials: false
ref: ${{ github.event.pull_request.head.sha || inputs.head_ref || github.sha }}
- name: Resolve HEAD and common ancestor
id: refs
shell: bash
env:
TARGET_BASE_REF: ${{ github.event.pull_request.base.sha || inputs.base_ref }}
HEAD_REF: ${{ github.event.pull_request.head.sha || inputs.head_ref || github.sha }}
EVENT_BEFORE: ${{ github.event.before }}
run: |
set -euo pipefail
head_sha=$(git rev-parse "${HEAD_REF}^{commit}")
target_base_ref=$TARGET_BASE_REF
if [[ -z "$target_base_ref" ]]; then
if [[ "$GITHUB_EVENT_NAME" == push \
&& -n "$EVENT_BEFORE" \
&& "$EVENT_BEFORE" != 0000000000000000000000000000000000000000 ]]; then
target_base_ref=$EVENT_BEFORE
else
target_base_ref="${head_sha}^"
fi
fi
target_base_sha=$(git rev-parse "${target_base_ref}^{commit}")
base_sha=$(git merge-base "$target_base_sha" "$head_sha")
if (( GITHUB_RUN_NUMBER % 2 == 0 )); then
execution_order=head-first
else
execution_order=base-first
fi
echo "base_sha=$base_sha" >> "$GITHUB_OUTPUT"
echo "head_sha=$head_sha" >> "$GITHUB_OUTPUT"
echo "execution_order=$execution_order" >> "$GITHUB_OUTPUT"
format:
name: Format
runs-on: ubuntu-latest
container: rust:1.96.1-bookworm
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install rustfmt
run: rustup component add rustfmt
- name: Check formatting
run: cargo fmt --all --check
clippy:
name: Clippy
runs-on: ubuntu-latest
container: rust:1.96.1-bookworm
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install AWS-LC build dependencies
run: |
set -eu
apt-get update
apt-get install --yes --no-install-recommends cmake libclang-dev
- name: Install clippy
run: rustup component add clippy
- name: Check clippy
run: cargo clippy --workspace --all-targets --all-features -- -D warnings
nextest:
name: Nextest
runs-on: [self-hosted, moli-ci]
container:
image: rust:1.96.1-bookworm
# Docker seeds new named volumes from the pinned image. Keep toolchains
# and Cargo downloads local to each self-hosted Docker daemon. Bump both
# volume names when the image or Rust toolchain changes.
volumes:
- moli-ci-cargo-home-1-96-1-v1:/usr/local/cargo
- moli-ci-rustup-home-1-96-1-v1:/usr/local/rustup
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install AWS-LC build dependencies
run: |
set -eu
sed -i -E \
's#https?://(deb|security)\.debian\.org/#https://mirrors.tuna.tsinghua.edu.cn/#g' \
/etc/apt/sources.list.d/debian.sources
apt-get update
apt-get install --yes --no-install-recommends cmake libclang-dev
- name: Install cargo-nextest
uses: taiki-e/install-action@13608cbb45b01feb47ef444ab1a42dc41ad56f1a # v2.79.11
with:
tool: nextest@0.9.132
- name: Run full workspace tests
run: cargo nextest run --workspace --profile ci
windows-persistent-storage:
name: Windows persistent storage
runs-on: windows-2025
timeout-minutes: 25
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install pinned Rust toolchain
shell: pwsh
run: |
$ErrorActionPreference = "Stop"
$PSNativeCommandUseErrorActionPreference = $true
$toolchain = (Get-Content rust-toolchain -Raw).Trim()
rustup toolchain install $toolchain --profile minimal --no-self-update
- name: Test Windows profile, OPFS, and storage paths
run: >-
cargo test --locked
-p moli-browser-profile
-p moli-opfs
-p moli-storage-service
python-harness-tests:
name: Python harness tests
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install uv
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
version: 0.11.14
python-version: "3.12"
enable-cache: true
cache-dependency-glob: |
moli-benchmark/uv.lock
moli-frontend-smoke/uv.lock
moli-cdp-smoke/uv.lock
moli-webdriver-smoke/uv.lock
- name: Install harness dependencies
run: |
uv sync --project moli-benchmark --locked
uv sync --project moli-frontend-smoke --locked
uv sync --project moli-cdp-smoke --locked
uv sync --project moli-webdriver-smoke --locked
- name: Test benchmark harness
shell: bash
run: |
set -euo pipefail
for test_file in moli-benchmark/tests/test_*.py; do
case "$test_file" in
*/test_wpt_cross*.py) continue ;;
esac
uv run --project moli-benchmark --no-sync python "$test_file"
done
- name: Test frontend differential harness
run: uv run --project moli-frontend-smoke --no-sync pytest moli-frontend-smoke/tests
- name: Test CDP smoke diagnostics
run: >-
uv run --project moli-cdp-smoke --no-sync python -m unittest discover
--start-directory moli-cdp-smoke/tests
- name: Test WebDriver smoke diagnostics
run: >-
uv run --project moli-webdriver-smoke --no-sync python -m unittest discover
--start-directory moli-webdriver-smoke/tests
node-harness-tests:
name: Node harness and fixture tests
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: 24.15.0
cache: npm
cache-dependency-path: |
moli-frontend-smoke/package-lock.json
moli-benchmark/browser-spider-local/package-lock.json
- name: Install dependencies
run: |
npm ci --prefix moli-frontend-smoke
npm ci --prefix moli-benchmark/browser-spider-local
- name: Validate frontend fixtures
working-directory: moli-frontend-smoke
run: |
npm run check:catalog
npm run test:node
npm run typecheck
npm run build
- name: Test Spider harness
working-directory: moli-benchmark/browser-spider-local
run: npm test
- name: Test trusted report helpers
run: node --test .github/scripts/*.test.cjs
build-release-head:
name: Build release · HEAD
needs: resolve-refs
runs-on: [self-hosted, moli-ci]
timeout-minutes: 180
container:
image: rust:1.96.1-bookworm
volumes:
- moli-ci-cargo-home-1-96-1-v1:/usr/local/cargo
- moli-ci-rustup-home-1-96-1-v1:/usr/local/rustup
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ needs.resolve-refs.outputs.head_sha }}
- name: Install AWS-LC build dependencies
run: |
set -eu
sed -i -E \
's#https?://(deb|security)\.debian\.org/#https://mirrors.tuna.tsinghua.edu.cn/#g' \
/etc/apt/sources.list.d/debian.sources
apt-get update
apt-get install --yes --no-install-recommends cmake libclang-dev
- name: Build release binary
run: cargo build --release --locked -p moli
- name: Package release artifact
shell: bash
run: |
set -euo pipefail
package_dir=target/ci-package/moli-release-head
mkdir -p "$package_dir"
install -m 0755 target/release/moli "$package_dir/moli"
printf '%s\n' '${{ needs.resolve-refs.outputs.head_sha }}' > "$package_dir/revision.txt"
(cd "$package_dir" && sha256sum moli > SHA256SUMS)
tar -C target/ci-package -czf target/moli-release-head.tar.gz moli-release-head
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: moli-release-head
path: target/moli-release-head.tar.gz
compression-level: 0
if-no-files-found: error
retention-days: 3
build-release-base:
name: Build release · common ancestor
needs: resolve-refs
runs-on: [self-hosted, moli-ci]
timeout-minutes: 180
container:
image: rust:1.96.1-bookworm
volumes:
- moli-ci-cargo-home-1-96-1-v1:/usr/local/cargo
- moli-ci-rustup-home-1-96-1-v1:/usr/local/rustup
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ needs.resolve-refs.outputs.base_sha }}
- name: Install AWS-LC build dependencies
run: |
set -eu
sed -i -E \
's#https?://(deb|security)\.debian\.org/#https://mirrors.tuna.tsinghua.edu.cn/#g' \
/etc/apt/sources.list.d/debian.sources
apt-get update
apt-get install --yes --no-install-recommends cmake libclang-dev
- name: Build release binary
run: cargo build --release --locked -p moli
- name: Package release artifact
shell: bash
run: |
set -euo pipefail
package_dir=target/ci-package/moli-release-base
mkdir -p "$package_dir"
install -m 0755 target/release/moli "$package_dir/moli"
printf '%s\n' '${{ needs.resolve-refs.outputs.base_sha }}' > "$package_dir/revision.txt"
(cd "$package_dir" && sha256sum moli > SHA256SUMS)
tar -C target/ci-package -czf target/moli-release-base.tar.gz moli-release-base
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: moli-release-base
path: target/moli-release-base.tar.gz
compression-level: 0
if-no-files-found: error
retention-days: 3
webmainbench:
name: WebMainBench · 545 pages
needs: [resolve-refs, build-release-head]
# Run on the hosted VM: unshare needs a network namespace, which is not
# available in the default Docker job container.
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ needs.resolve-refs.outputs.head_sha }}
- name: Use HEAD release
uses: ./.github/actions/use-ci-release
with:
side: head
revision: ${{ needs.resolve-refs.outputs.head_sha }}
- name: Install replay tools
run: |
set -eu
# Ubuntu runners may use legacy .list or DEB822 .sources files.
for apt_source in /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do
[ -f "$apt_source" ] || continue
sudo sed -i -E \
's#https?://(([[:alnum:]-]+\.)?archive|security|ports)\.ubuntu\.com/#https://mirrors.tuna.tsinghua.edu.cn/#g' \
"$apt_source"
done
sudo apt-get update
sudo apt-get install --yes --no-install-recommends python3 iproute2 openssl util-linux
- name: Download pinned WebMainBench corpus
env:
PYTHONPATH: moli-benchmark
run: >-
python3 -m moli_benchmark.webmainbench download
--output target/webmainbench-data/WebMainBench_545.jsonl
- name: Replay all pages with external networking disabled
timeout-minutes: 25
env:
HEAD_SHA: ${{ needs.resolve-refs.outputs.head_sha }}
run: >-
sudo unshare --net --
env PYTHONPATH="$PWD/moli-benchmark"
python3 -u -m moli_benchmark.webmainbench run
--moli-bin "$PWD/target/ci-bin/moli-release-head/moli"
--dataset "$PWD/target/webmainbench-data/WebMainBench_545.jsonl"
--output "$PWD/target/webmainbench-results"
--revision "$HEAD_SHA"
- name: Render job summary
if: ${{ always() }}
run: |
if [[ -f target/webmainbench-results/summary.md ]]; then
cat target/webmainbench-results/summary.md >> "$GITHUB_STEP_SUMMARY"
fi
- name: Upload every page's output and diagnostics
if: ${{ always() }}
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: webmainbench-results
path: target/webmainbench-results
if-no-files-found: warn
retention-days: 7
cdp-smoke:
name: CDP smoke
needs: [resolve-refs, build-release-head, cdp-smoke-chromium]
if: ${{ !cancelled() && needs.resolve-refs.result == 'success' && needs.build-release-head.result == 'success' }}
runs-on: ubuntu-latest
timeout-minutes: 60
container: rust:1.96.1-bookworm
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ needs.resolve-refs.outputs.head_sha }}
- name: Use HEAD release
uses: ./.github/actions/use-ci-release
with:
side: head
revision: ${{ needs.resolve-refs.outputs.head_sha }}
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
version: 0.11.14
python-version: "3.12"
enable-cache: true
cache-dependency-glob: moli-cdp-smoke/uv.lock
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: 24.15.0
cache: npm
cache-dependency-path: moli-cdp-smoke/package-lock.json
- name: Install dependencies
run: |
uv sync --project moli-cdp-smoke --locked
npm ci --prefix moli-cdp-smoke
- name: Run full CDP smoke
run: |
export MOLI_BIN="$PWD/target/ci-bin/moli-release-head/moli"
uv run --project moli-cdp-smoke --no-sync moli-cdp-smoke \
--browser moli --strict-full-suite --repeat 2 --jobs 4 --output-dir target/smoke/cdp-ci
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
with:
go-version-file: moli-cdp-smoke/chromedp/go.mod
cache-dependency-path: moli-cdp-smoke/chromedp/go.sum
- name: Run chromedp DOM search regression
working-directory: moli-cdp-smoke/chromedp
env:
MOLI_BIN: ${{ github.workspace }}/target/ci-bin/moli-release-head/moli
run: go test -mod=readonly -count=1 ./...
- name: Require full Chromium smoke
if: ${{ always() }}
env:
CHROMIUM_SMOKE_RESULT: ${{ needs.cdp-smoke-chromium.result }}
run: |
if [ "$CHROMIUM_SMOKE_RESULT" != success ]; then
echo "::error::Chromium reference smoke ended with '$CHROMIUM_SMOKE_RESULT'; inspect the 'CDP smoke (Chromium)' job. The preceding step reports the Moli suite result."
exit 1
fi
- name: Upload diagnostics
if: ${{ always() }}
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: cdp-smoke-diagnostics
path: target/smoke/cdp-ci
if-no-files-found: error
retention-days: 7
cdp-smoke-chromium:
name: CDP smoke (Chromium)
needs: [resolve-refs]
runs-on: [self-hosted, moli-ci]
timeout-minutes: 30
container: mcr.microsoft.com/playwright/python:v1.58.0-noble
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ needs.resolve-refs.outputs.head_sha }}
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
version: 0.11.14
python-version: "3.12"
enable-cache: true
cache-dependency-glob: moli-cdp-smoke/uv.lock
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: 24.15.0
cache: npm
cache-dependency-path: moli-cdp-smoke/package-lock.json
- name: Install smoke dependencies
run: |
uv sync --project moli-cdp-smoke --locked
npm ci --prefix moli-cdp-smoke
- name: Run full CDP smoke on Chromium
run: |
uv run --project moli-cdp-smoke --no-sync moli-cdp-smoke \
--browser chromium --strict-full-suite --repeat 2 --retries 3 --jobs 4 --output-dir target/smoke/cdp-chromium-ci
- name: Upload Chromium diagnostics
if: ${{ always() }}
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: cdp-smoke-chromium-diagnostics
path: target/smoke/cdp-chromium-ci
if-no-files-found: error
retention-days: 7
native-pdf:
name: Native PDF rendering and links
needs: [resolve-refs, build-release-head]
runs-on: ubuntu-latest
timeout-minutes: 15
container: rust:1.96.1-bookworm
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ needs.resolve-refs.outputs.head_sha }}
- name: Use HEAD release
uses: ./.github/actions/use-ci-release
with:
side: head
revision: ${{ needs.resolve-refs.outputs.head_sha }}
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
version: 0.11.14
python-version: "3.12"
enable-cache: true
cache-dependency-glob: scripts/check-native-pdf.py.lock
- name: Install Poppler for translucent color glyph regressions
run: |
apt-get update
apt-get install --no-install-recommends -y poppler-utils
- name: Check rendered pixels, text and link targets and regions
run: >-
uv run --locked --script scripts/check-native-pdf.py
--moli target/ci-bin/moli-release-head/moli
--output-dir target/native-pdf-ci
- name: Upload PDFs and reader evidence
if: ${{ always() }}
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: native-pdf-results
path: target/native-pdf-ci
if-no-files-found: warn
retention-days: 7
webdriver-smoke:
name: WebDriver smoke
needs: [resolve-refs, build-release-head]
runs-on: ubuntu-latest
timeout-minutes: 60
container: rust:1.96.1-bookworm
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ needs.resolve-refs.outputs.head_sha }}
- name: Use HEAD release
uses: ./.github/actions/use-ci-release
with:
side: head
revision: ${{ needs.resolve-refs.outputs.head_sha }}
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
version: 0.11.14
python-version: "3.12"
enable-cache: true
cache-dependency-glob: moli-webdriver-smoke/uv.lock
- name: Install dependencies
run: uv sync --project moli-webdriver-smoke --locked
- name: Run full WebDriver smoke
shell: bash
run: |
export MOLI_BIN="$PWD/target/ci-bin/moli-release-head/moli"
mkdir -p target/smoke/webdriver-ci
set -o pipefail
uv run --project moli-webdriver-smoke --no-sync \
moli-webdriver-smoke --continue-on-failure \
2>&1 | tee target/smoke/webdriver-ci/run.log
- name: Upload diagnostics
if: ${{ always() }}
continue-on-error: true
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: webdriver-smoke-diagnostics
path: target/smoke/webdriver-ci
if-no-files-found: warn
retention-days: 7
frontend-differential:
name: Frontend differential · 1020 cases
needs: [resolve-refs, build-release-head]
runs-on: [self-hosted, moli-ci]
timeout-minutes: 45
container: mcr.microsoft.com/playwright/python:v1.58.0-noble
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ needs.resolve-refs.outputs.head_sha }}
- name: Use HEAD release
uses: ./.github/actions/use-ci-release
with:
side: head
revision: ${{ needs.resolve-refs.outputs.head_sha }}
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
version: 0.11.14
python-version: "3.12"
enable-cache: true
cache-dependency-glob: |
moli-frontend-smoke/uv.lock
moli-cdp-smoke/uv.lock
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: 24.15.0
- name: Install dependencies and build fixtures
run: |
uv sync --project moli-frontend-smoke --locked
uv sync --project moli-cdp-smoke --locked
npm ci --prefix moli-frontend-smoke
npm run build --prefix moli-frontend-smoke
- name: Resolve pinned Chromium
id: chromium
shell: bash
run: |
set -euo pipefail
chromium_bin=$(uv run --project moli-cdp-smoke --no-sync python -c \
'from playwright.sync_api import sync_playwright; p = sync_playwright().start(); print(p.chromium.executable_path); p.stop()')
test -x "$chromium_bin"
wrapper="$PWD/target/ci-bin/chromium-ci"
printf '%s\n' '#!/bin/sh' "exec \"$chromium_bin\" --no-sandbox \"\$@\"" > "$wrapper"
chmod 0755 "$wrapper"
echo "binary=$wrapper" >> "$GITHUB_OUTPUT"
- name: Run full differential
id: frontend
continue-on-error: true
# Leave headroom above the network/storage fixtures' 60-second event
# deadline when running the full catalog on shared hosts.
run: >-
uv run --project moli-frontend-smoke --no-sync moli-frontend-smoke
--chromium-bin "${{ steps.chromium.outputs.binary }}"
--moli-bin "$PWD/target/ci-bin/moli-release-head/moli"
--timeout-ms 90000
--reference-infrastructure-retries 1
--output target/frontend-smoke
- name: Validate frontend artifacts
id: analysis
if: ${{ always() && steps.frontend.outcome != 'skipped' }}
continue-on-error: true
run: >-
uv run --project moli-frontend-smoke --no-sync
moli-frontend-smoke-analyze results target/frontend-smoke
- name: Upload frontend evidence
if: ${{ always() && steps.frontend.outcome != 'skipped' }}
continue-on-error: true
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: frontend-differential-results
path: target/frontend-smoke
if-no-files-found: warn
retention-days: 7
- name: Require differential and artifact validation
if: ${{ always() }}
env:
FRONTEND_OUTCOME: ${{ steps.frontend.outcome }}
ANALYSIS_OUTCOME: ${{ steps.analysis.outcome }}
run: |
test "$FRONTEND_OUTCOME" = success
test "$ANALYSIS_OUTCOME" = success
runtime-contracts:
name: Runtime and CDP session contracts
needs: [resolve-refs, build-release-head]
runs-on: [self-hosted, moli-ci]
timeout-minutes: 30
container: rust:1.96.1-bookworm
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ needs.resolve-refs.outputs.head_sha }}
- name: Use HEAD release
uses: ./.github/actions/use-ci-release
with:
side: head
revision: ${{ needs.resolve-refs.outputs.head_sha }}
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
version: 0.11.14
python-version: "3.12"
enable-cache: true
cache-dependency-glob: moli-benchmark/uv.lock
- name: Install dependencies
run: uv sync --project moli-benchmark --locked
- name: Run deterministic fetch contracts
run: >-
uv run --project moli-benchmark --no-sync moli-benchmark synthetic
--moli-bin "$PWD/target/ci-bin/moli-release-head/moli"
--runs 5 --output-dir target/benchmark/runtime-synthetic
- name: Run long-lived CDP session contracts
run: >-
uv run --project moli-benchmark --no-sync moli-benchmark cdp-session
--moli-bin "$PWD/target/ci-bin/moli-release-head/moli"
--target moli-cdp --gate-target moli-cdp --runs 3
--output-dir target/benchmark/cdp-session
- name: Upload evidence
if: ${{ always() }}
continue-on-error: true
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: runtime-contract-results
path: target/benchmark
if-no-files-found: warn
retention-days: 7
agent-episode:
name: Agent episodes · Moli vs Chromium
needs: [resolve-refs, build-release-head]
runs-on: [self-hosted, moli-ci]
timeout-minutes: 45
container: mcr.microsoft.com/playwright/python:v1.58.0-noble
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ needs.resolve-refs.outputs.head_sha }}
- name: Use HEAD release
uses: ./.github/actions/use-ci-release
with:
side: head
revision: ${{ needs.resolve-refs.outputs.head_sha }}
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
version: 0.11.14
python-version: "3.12"
enable-cache: true
cache-dependency-glob: |
moli-benchmark/uv.lock
moli-cdp-smoke/uv.lock
- name: Install dependencies
run: |
uv sync --project moli-benchmark --locked
uv sync --project moli-cdp-smoke --locked
- name: Resolve pinned Chromium
id: chromium
run: |
chromium_bin=$(uv run --project moli-cdp-smoke --no-sync python -c \
'from playwright.sync_api import sync_playwright; p = sync_playwright().start(); print(p.chromium.executable_path); p.stop()')
test -x "$chromium_bin"
echo "binary=$chromium_bin" >> "$GITHUB_OUTPUT"
- name: Run fast cross-engine contract
run: >-
uv run --project moli-benchmark --no-sync moli-benchmark agent-episode
--moli-bin "$PWD/target/ci-bin/moli-release-head/moli"
--chrome-bin "${{ steps.chromium.outputs.binary }}"
--step-dwell-ms 0 --output-dir target/benchmark/agent-fast
- name: Run idle-resource episode
run: >-
uv run --project moli-benchmark --no-sync moli-benchmark agent-episode
--moli-bin "$PWD/target/ci-bin/moli-release-head/moli"
--chrome-bin "${{ steps.chromium.outputs.binary }}"
--output-dir target/benchmark/agent-resource
- name: Upload evidence
if: ${{ always() }}
continue-on-error: true
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: agent-episode-results
path: target/benchmark
if-no-files-found: warn
retention-days: 7
release-regression:
name: Release startup, size, and matrix A/B
needs: [resolve-refs, build-release-head, build-release-base]
runs-on: [self-hosted, moli-ci]
timeout-minutes: 60
container: rust:1.96.1-bookworm
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ needs.resolve-refs.outputs.head_sha }}
- name: Use HEAD release
uses: ./.github/actions/use-ci-release
with:
side: head
revision: ${{ needs.resolve-refs.outputs.head_sha }}
- name: Use base release
uses: ./.github/actions/use-ci-release
with:
side: base
revision: ${{ needs.resolve-refs.outputs.base_sha }}
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
version: 0.11.14
python-version: "3.12"
enable-cache: true
cache-dependency-glob: moli-benchmark/uv.lock
- run: uv sync --project moli-benchmark --locked
- name: Measure common-ancestor startup and package
continue-on-error: true
run: >-
uv run --project moli-benchmark --no-sync moli-benchmark startup
--moli-bin "$PWD/target/ci-bin/moli-release-base/moli"
--profile formal --output-dir target/release-regression/base-startup
- name: Require HEAD startup and package workflow
run: >-
uv run --project moli-benchmark --no-sync moli-benchmark startup
--moli-bin "$PWD/target/ci-bin/moli-release-head/moli"
--profile formal --baseline-report target/release-regression/base-startup
--output-dir target/release-regression/head-startup
- name: Measure common-ancestor concurrency matrix
continue-on-error: true
run: >-
uv run --project moli-benchmark --no-sync moli-benchmark synthetic-matrix
--moli-bin "$PWD/target/ci-bin/moli-release-base/moli"
--profile smoke --output-dir target/release-regression/base-matrix
- name: Require HEAD concurrency matrix
run: >-
uv run --project moli-benchmark --no-sync moli-benchmark synthetic-matrix
--moli-bin "$PWD/target/ci-bin/moli-release-head/moli"
--profile smoke --baseline-report target/release-regression/base-matrix
--output-dir target/release-regression/head-matrix
- name: Upload evidence
if: ${{ always() }}
continue-on-error: true
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: release-regression-results
# Package sizes stay in the JSON reports and image manifests. Upload
# report data only instead of duplicating the full rootfs archives.
path: |
target/release-regression/**/*.json
target/release-regression/**/*.csv
target/release-regression/**/*.md
if-no-files-found: warn
retention-days: 7
spider-fixture:
name: Deterministic Spider fixture
needs: [resolve-refs, build-release-head]
runs-on: [self-hosted, moli-ci]
timeout-minutes: 20
container: rust:1.96.1-bookworm
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ needs.resolve-refs.outputs.head_sha }}
- name: Use HEAD release
uses: ./.github/actions/use-ci-release
with:
side: head
revision: ${{ needs.resolve-refs.outputs.head_sha }}
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: 24.15.0
- run: npm ci --prefix moli-benchmark/browser-spider-local
- name: Require exact fixture contract
run: >-
node moli-benchmark/browser-spider-local/bench.mjs
--target moli
--moli-bin "$PWD/target/ci-bin/moli-release-head/moli"
--workers 1 --parallelism 1 --runs 1
--site-source fixture --goto-timeout-ms 1500 --server-hang-ms 3000
--sample-interval-ms 100 --assert-fixture-contract
--output-dir target/spider-fixture
- name: Upload evidence
if: ${{ always() }}
continue-on-error: true
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: spider-fixture-results
path: target/spider-fixture
if-no-files-found: warn
retention-days: 7
spider-bench:
name: Spider public-web A/B
needs: [resolve-refs, build-release-head, build-release-base]
if: >-
github.event_name != 'pull_request' ||
(github.event.pull_request.head.repo.full_name == github.repository &&
github.actor != 'dependabot[bot]')
runs-on: [self-hosted, moli-ci]
timeout-minutes: 90
container: rust:1.96.1-bookworm
env:
BENCHMARK_OUTPUT: spider-bench-results
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ needs.resolve-refs.outputs.head_sha }}
- name: Use HEAD release
uses: ./.github/actions/use-ci-release
with:
side: head
revision: ${{ needs.resolve-refs.outputs.head_sha }}
- name: Use base release
uses: ./.github/actions/use-ci-release
with:
side: base
revision: ${{ needs.resolve-refs.outputs.base_sha }}
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: 24.15.0
- run: npm ci --prefix moli-benchmark/browser-spider-local
- name: Run deterministic and public-web A/B
run: >-
node moli-benchmark/browser-spider-local/compare.mjs run
--base-bin "$PWD/target/ci-bin/moli-release-base/moli"
--head-bin "$PWD/target/ci-bin/moli-release-head/moli"
--base-sha "${{ needs.resolve-refs.outputs.base_sha }}"
--head-sha "${{ needs.resolve-refs.outputs.head_sha }}"
--execution-order "${{ needs.resolve-refs.outputs.execution_order }}"
--output-dir "$BENCHMARK_OUTPUT"
--run-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
- name: Render job summary
if: ${{ always() && !cancelled() }}
continue-on-error: true
run: |
if [[ -f "$BENCHMARK_OUTPUT/comment.md" ]]; then
cat "$BENCHMARK_OUTPUT/comment.md" >> "$GITHUB_STEP_SUMMARY"
fi
- name: Upload full evidence
if: ${{ always() && !cancelled() }}
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: spider-bench-results
path: ${{ env.BENCHMARK_OUTPUT }}
if-no-files-found: error
retention-days: 14
sequential-navigation-soak:
name: Sequential navigation soak · 200 A/B
needs: [resolve-refs, build-release-head, build-release-base]
if: >-
github.event_name != 'pull_request' ||
(github.event.pull_request.head.repo.full_name == github.repository &&
github.actor != 'dependabot[bot]')
runs-on: [self-hosted, moli-ci]
timeout-minutes: 90
container: rust:1.96.1-bookworm
env:
BENCHMARK_OUTPUT: sequential-navigation-soak-results
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ needs.resolve-refs.outputs.head_sha }}
- name: Use HEAD release
uses: ./.github/actions/use-ci-release
with:
side: head
revision: ${{ needs.resolve-refs.outputs.head_sha }}
- name: Use base release
uses: ./.github/actions/use-ci-release
with:
side: base
revision: ${{ needs.resolve-refs.outputs.base_sha }}
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
version: 0.11.14
python-version: "3.12"
enable-cache: true
cache-dependency-glob: moli-benchmark/uv.lock
- run: uv sync --project moli-benchmark --locked
- name: Run one-session public-web A/B
id: soak
shell: bash
env:
EXECUTION_ORDER: ${{ needs.resolve-refs.outputs.execution_order }}
run: |
set -euo pipefail
run_side() {
local side=$1
local binary=$2
set +e
uv run --project moli-benchmark --no-sync python \
moli-benchmark/scripts/fuzz-cdp-sequential-navigate.py \
--engine moli --moli-bin "$binary" \
--url https://www.csdn.net/ \
--url https://segmentfault.com/ \
--url https://huaban.com/ \
--url https://example.com/ \
--rounds 50 --network-diagnostics --navigation-resource-samples \
--response-timeout 15 --dcl-timeout 14 --load-timeout 20 \
--postcheck-timeout 5 --recovery-timeout 8 \
--output "$BENCHMARK_OUTPUT/$side.json"
local status=$?
set -e
echo "${side}_exit_code=$status" >> "$GITHUB_OUTPUT"
}
if [[ "$EXECUTION_ORDER" == head-first ]]; then
run_side head "$PWD/target/ci-bin/moli-release-head/moli"
run_side base "$PWD/target/ci-bin/moli-release-base/moli"
else
run_side base "$PWD/target/ci-bin/moli-release-base/moli"
run_side head "$PWD/target/ci-bin/moli-release-head/moli"
fi
- name: Compare resilience and memory evidence
id: comparison
if: ${{ always() && !cancelled() && steps.soak.outcome == 'success' }}
env:
PYTHONPATH: moli-benchmark
run: >-
uv run --project moli-benchmark --no-sync python
-m moli_benchmark.sequential_navigate_ci compare
--base-report "$BENCHMARK_OUTPUT/base.json"
--head-report "$BENCHMARK_OUTPUT/head.json"
--base-exit-code "${{ steps.soak.outputs.base_exit_code }}"
--head-exit-code "${{ steps.soak.outputs.head_exit_code }}"
--base-sha "${{ needs.resolve-refs.outputs.base_sha }}"
--head-sha "${{ needs.resolve-refs.outputs.head_sha }}"
--execution-order "${{ needs.resolve-refs.outputs.execution_order }}"
--expected-navigations 200
--output "$BENCHMARK_OUTPUT/comparison.json"
--comment-output "$BENCHMARK_OUTPUT/comment.md"
--run-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
--conclusion local
- name: Render job summary
if: ${{ always() && !cancelled() }}
continue-on-error: true
run: |
if [[ -f "$BENCHMARK_OUTPUT/comment.md" ]]; then
cat "$BENCHMARK_OUTPUT/comment.md" >> "$GITHUB_STEP_SUMMARY"
fi
- name: Upload full evidence
if: ${{ always() && !cancelled() }}
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: sequential-navigation-soak-results
path: ${{ env.BENCHMARK_OUTPUT }}
if-no-files-found: error
retention-days: 14
- name: Require complete HEAD resilience evidence
if: ${{ always() && !cancelled() && steps.comparison.outcome == 'success' }}
env:
PYTHONPATH: moli-benchmark
run: >-
uv run --project moli-benchmark --no-sync python
-m moli_benchmark.sequential_navigate_ci check
--input "$BENCHMARK_OUTPUT/comparison.json"