mirror of
https://github.com/lexmount/moli.git
synced 2026-09-23 00:01:25 +00:00
874 lines
33 KiB
YAML
874 lines
33 KiB
YAML
name: CI
|
|
|
|
on:
|
|
pull_request:
|
|
types: [opened, synchronize, reopened]
|
|
push:
|
|
branches:
|
|
- main
|
|
workflow_dispatch:
|
|
inputs:
|
|
base_ref:
|
|
description: Base commit or ref (defaults to the previous commit)
|
|
required: false
|
|
default: ""
|
|
head_ref:
|
|
description: Head commit or ref (defaults to the dispatched commit)
|
|
required: false
|
|
default: ""
|
|
|
|
concurrency:
|
|
group: ci-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
env:
|
|
CARGO_TERM_COLOR: always
|
|
CARGO_INCREMENTAL: "0"
|
|
GIT_CONFIG_COUNT: "1"
|
|
GIT_CONFIG_KEY_0: safe.directory
|
|
GIT_CONFIG_VALUE_0: ${{ github.workspace }}
|
|
|
|
jobs:
|
|
resolve-refs:
|
|
name: Resolve exact revisions
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
base_sha: ${{ steps.refs.outputs.base_sha }}
|
|
head_sha: ${{ steps.refs.outputs.head_sha }}
|
|
execution_order: ${{ steps.refs.outputs.execution_order }}
|
|
steps:
|
|
- name: Checkout exact HEAD
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
ref: ${{ github.event.pull_request.head.sha || inputs.head_ref || github.sha }}
|
|
|
|
- name: Resolve HEAD and common ancestor
|
|
id: refs
|
|
shell: bash
|
|
env:
|
|
TARGET_BASE_REF: ${{ github.event.pull_request.base.sha || inputs.base_ref }}
|
|
HEAD_REF: ${{ github.event.pull_request.head.sha || inputs.head_ref || github.sha }}
|
|
EVENT_BEFORE: ${{ github.event.before }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
head_sha=$(git rev-parse "${HEAD_REF}^{commit}")
|
|
target_base_ref=$TARGET_BASE_REF
|
|
if [[ -z "$target_base_ref" ]]; then
|
|
if [[ "$GITHUB_EVENT_NAME" == push \
|
|
&& -n "$EVENT_BEFORE" \
|
|
&& "$EVENT_BEFORE" != 0000000000000000000000000000000000000000 ]]; then
|
|
target_base_ref=$EVENT_BEFORE
|
|
else
|
|
target_base_ref="${head_sha}^"
|
|
fi
|
|
fi
|
|
|
|
target_base_sha=$(git rev-parse "${target_base_ref}^{commit}")
|
|
base_sha=$(git merge-base "$target_base_sha" "$head_sha")
|
|
|
|
if (( GITHUB_RUN_NUMBER % 2 == 0 )); then
|
|
execution_order=head-first
|
|
else
|
|
execution_order=base-first
|
|
fi
|
|
|
|
echo "base_sha=$base_sha" >> "$GITHUB_OUTPUT"
|
|
echo "head_sha=$head_sha" >> "$GITHUB_OUTPUT"
|
|
echo "execution_order=$execution_order" >> "$GITHUB_OUTPUT"
|
|
|
|
format:
|
|
name: Format
|
|
runs-on: ubuntu-latest
|
|
container: rust:1.96.1-bookworm
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Install rustfmt
|
|
run: rustup component add rustfmt
|
|
|
|
- name: Check formatting
|
|
run: cargo fmt --all --check
|
|
|
|
clippy:
|
|
name: Clippy
|
|
runs-on: ubuntu-latest
|
|
container: rust:1.96.1-bookworm
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Install AWS-LC build dependencies
|
|
run: |
|
|
apt-get update
|
|
apt-get install --yes --no-install-recommends cmake libclang-dev
|
|
|
|
- name: Install clippy
|
|
run: rustup component add clippy
|
|
|
|
- name: Check clippy
|
|
run: cargo clippy --workspace --all-targets --all-features -- -D warnings
|
|
|
|
nextest:
|
|
name: Nextest
|
|
runs-on: [self-hosted, moli-ci]
|
|
container:
|
|
image: rust:1.96.1-bookworm
|
|
# Docker seeds new named volumes from the pinned image. Keep toolchains
|
|
# and Cargo downloads local to each self-hosted Docker daemon. Bump both
|
|
# volume names when the image or Rust toolchain changes.
|
|
volumes:
|
|
- moli-ci-cargo-home-1-96-1-v1:/usr/local/cargo
|
|
- moli-ci-rustup-home-1-96-1-v1:/usr/local/rustup
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Install AWS-LC build dependencies
|
|
run: |
|
|
apt-get update
|
|
apt-get install --yes --no-install-recommends cmake libclang-dev
|
|
|
|
- name: Install cargo-nextest
|
|
uses: taiki-e/install-action@13608cbb45b01feb47ef444ab1a42dc41ad56f1a # v2.79.11
|
|
with:
|
|
tool: nextest@0.9.132
|
|
|
|
- name: Run full workspace tests
|
|
run: cargo nextest run --workspace --profile ci
|
|
|
|
windows-persistent-storage:
|
|
name: Windows persistent storage
|
|
runs-on: windows-2025
|
|
timeout-minutes: 25
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Install pinned Rust toolchain
|
|
shell: pwsh
|
|
run: |
|
|
$ErrorActionPreference = "Stop"
|
|
$PSNativeCommandUseErrorActionPreference = $true
|
|
$toolchain = (Get-Content rust-toolchain -Raw).Trim()
|
|
rustup toolchain install $toolchain --profile minimal --no-self-update
|
|
|
|
- name: Test Windows profile, OPFS, and storage paths
|
|
run: >-
|
|
cargo test --locked
|
|
-p moli-browser-profile
|
|
-p moli-opfs
|
|
-p moli-storage-service
|
|
|
|
python-harness-tests:
|
|
name: Python harness tests
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 20
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
|
with:
|
|
version: 0.11.14
|
|
python-version: "3.12"
|
|
enable-cache: true
|
|
cache-dependency-glob: |
|
|
moli-benchmark/uv.lock
|
|
moli-frontend-smoke/uv.lock
|
|
moli-cdp-smoke/uv.lock
|
|
moli-webdriver-smoke/uv.lock
|
|
|
|
- name: Install harness dependencies
|
|
run: |
|
|
uv sync --project moli-benchmark --locked
|
|
uv sync --project moli-frontend-smoke --locked
|
|
uv sync --project moli-cdp-smoke --locked
|
|
uv sync --project moli-webdriver-smoke --locked
|
|
|
|
- name: Test benchmark harness
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
for test_file in moli-benchmark/tests/test_*.py; do
|
|
case "$test_file" in
|
|
*/test_wpt_cross*.py) continue ;;
|
|
esac
|
|
uv run --project moli-benchmark --no-sync python "$test_file"
|
|
done
|
|
|
|
- name: Test frontend differential harness
|
|
run: uv run --project moli-frontend-smoke --no-sync pytest moli-frontend-smoke/tests
|
|
|
|
- name: Test CDP smoke diagnostics
|
|
run: >-
|
|
uv run --project moli-cdp-smoke --no-sync python -m unittest discover
|
|
--start-directory moli-cdp-smoke/tests
|
|
|
|
- name: Test WebDriver smoke diagnostics
|
|
run: >-
|
|
uv run --project moli-webdriver-smoke --no-sync python -m unittest discover
|
|
--start-directory moli-webdriver-smoke/tests
|
|
|
|
node-harness-tests:
|
|
name: Node harness and fixture tests
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 20
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
|
with:
|
|
node-version: 24.15.0
|
|
cache: npm
|
|
cache-dependency-path: |
|
|
moli-frontend-smoke/package-lock.json
|
|
moli-benchmark/browser-spider-local/package-lock.json
|
|
- name: Install dependencies
|
|
run: |
|
|
npm ci --prefix moli-frontend-smoke
|
|
npm ci --prefix moli-benchmark/browser-spider-local
|
|
- name: Validate frontend fixtures
|
|
working-directory: moli-frontend-smoke
|
|
run: |
|
|
npm run check:catalog
|
|
npm run test:node
|
|
npm run typecheck
|
|
npm run build
|
|
- name: Test Spider harness
|
|
working-directory: moli-benchmark/browser-spider-local
|
|
run: npm test
|
|
- name: Test trusted report helpers
|
|
run: node --test .github/scripts/*.test.cjs
|
|
|
|
build-release-head:
|
|
name: Build release · HEAD
|
|
needs: resolve-refs
|
|
runs-on: [self-hosted, moli-ci]
|
|
timeout-minutes: 60
|
|
container:
|
|
image: rust:1.96.1-bookworm
|
|
volumes:
|
|
- moli-ci-cargo-home-1-96-1-v1:/usr/local/cargo
|
|
- moli-ci-rustup-home-1-96-1-v1:/usr/local/rustup
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
persist-credentials: false
|
|
ref: ${{ needs.resolve-refs.outputs.head_sha }}
|
|
- name: Install AWS-LC build dependencies
|
|
run: |
|
|
apt-get update
|
|
apt-get install --yes --no-install-recommends cmake libclang-dev
|
|
- name: Build release binary
|
|
run: cargo build --release --locked -p moli
|
|
- name: Package release artifact
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
package_dir=target/ci-package/moli-release-head
|
|
mkdir -p "$package_dir"
|
|
install -m 0755 target/release/moli "$package_dir/moli"
|
|
printf '%s\n' '${{ needs.resolve-refs.outputs.head_sha }}' > "$package_dir/revision.txt"
|
|
(cd "$package_dir" && sha256sum moli > SHA256SUMS)
|
|
tar -C target/ci-package -czf target/moli-release-head.tar.gz moli-release-head
|
|
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
|
with:
|
|
name: moli-release-head
|
|
path: target/moli-release-head.tar.gz
|
|
compression-level: 0
|
|
if-no-files-found: error
|
|
retention-days: 3
|
|
|
|
build-release-base:
|
|
name: Build release · common ancestor
|
|
needs: resolve-refs
|
|
runs-on: [self-hosted, moli-ci]
|
|
timeout-minutes: 60
|
|
container:
|
|
image: rust:1.96.1-bookworm
|
|
volumes:
|
|
- moli-ci-cargo-home-1-96-1-v1:/usr/local/cargo
|
|
- moli-ci-rustup-home-1-96-1-v1:/usr/local/rustup
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
persist-credentials: false
|
|
ref: ${{ needs.resolve-refs.outputs.base_sha }}
|
|
- name: Install AWS-LC build dependencies
|
|
run: |
|
|
apt-get update
|
|
apt-get install --yes --no-install-recommends cmake libclang-dev
|
|
- name: Build release binary
|
|
run: cargo build --release --locked -p moli
|
|
- name: Package release artifact
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
package_dir=target/ci-package/moli-release-base
|
|
mkdir -p "$package_dir"
|
|
install -m 0755 target/release/moli "$package_dir/moli"
|
|
printf '%s\n' '${{ needs.resolve-refs.outputs.base_sha }}' > "$package_dir/revision.txt"
|
|
(cd "$package_dir" && sha256sum moli > SHA256SUMS)
|
|
tar -C target/ci-package -czf target/moli-release-base.tar.gz moli-release-base
|
|
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
|
with:
|
|
name: moli-release-base
|
|
path: target/moli-release-base.tar.gz
|
|
compression-level: 0
|
|
if-no-files-found: error
|
|
retention-days: 3
|
|
|
|
cdp-smoke:
|
|
name: CDP smoke
|
|
needs: [resolve-refs, build-release-head]
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 60
|
|
container: rust:1.96.1-bookworm
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
persist-credentials: false
|
|
ref: ${{ needs.resolve-refs.outputs.head_sha }}
|
|
- name: Use HEAD release
|
|
uses: ./.github/actions/use-ci-release
|
|
with:
|
|
side: head
|
|
revision: ${{ needs.resolve-refs.outputs.head_sha }}
|
|
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
|
with:
|
|
version: 0.11.14
|
|
python-version: "3.12"
|
|
enable-cache: true
|
|
cache-dependency-glob: moli-cdp-smoke/uv.lock
|
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
|
with:
|
|
node-version: 24.15.0
|
|
cache: npm
|
|
cache-dependency-path: moli-cdp-smoke/package-lock.json
|
|
- name: Install dependencies
|
|
run: |
|
|
uv sync --project moli-cdp-smoke --locked
|
|
npm ci --prefix moli-cdp-smoke
|
|
- name: Run full CDP smoke
|
|
run: |
|
|
export MOLI_BIN="$PWD/target/ci-bin/moli-release-head/moli"
|
|
uv run --project moli-cdp-smoke --no-sync moli-cdp-smoke \
|
|
--jobs 4 --output-dir target/smoke/cdp-ci
|
|
- name: Upload diagnostics
|
|
if: ${{ always() }}
|
|
continue-on-error: true
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
|
with:
|
|
name: cdp-smoke-diagnostics
|
|
path: target/smoke/cdp-ci
|
|
if-no-files-found: warn
|
|
retention-days: 7
|
|
|
|
webdriver-smoke:
|
|
name: WebDriver smoke
|
|
needs: [resolve-refs, build-release-head]
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 60
|
|
container: rust:1.96.1-bookworm
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
persist-credentials: false
|
|
ref: ${{ needs.resolve-refs.outputs.head_sha }}
|
|
- name: Use HEAD release
|
|
uses: ./.github/actions/use-ci-release
|
|
with:
|
|
side: head
|
|
revision: ${{ needs.resolve-refs.outputs.head_sha }}
|
|
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
|
with:
|
|
version: 0.11.14
|
|
python-version: "3.12"
|
|
enable-cache: true
|
|
cache-dependency-glob: moli-webdriver-smoke/uv.lock
|
|
- name: Install dependencies
|
|
run: uv sync --project moli-webdriver-smoke --locked
|
|
- name: Run full WebDriver smoke
|
|
shell: bash
|
|
run: |
|
|
export MOLI_BIN="$PWD/target/ci-bin/moli-release-head/moli"
|
|
mkdir -p target/smoke/webdriver-ci
|
|
set -o pipefail
|
|
uv run --project moli-webdriver-smoke --no-sync \
|
|
moli-webdriver-smoke --continue-on-failure \
|
|
2>&1 | tee target/smoke/webdriver-ci/run.log
|
|
- name: Upload diagnostics
|
|
if: ${{ always() }}
|
|
continue-on-error: true
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
|
with:
|
|
name: webdriver-smoke-diagnostics
|
|
path: target/smoke/webdriver-ci
|
|
if-no-files-found: warn
|
|
retention-days: 7
|
|
|
|
frontend-differential:
|
|
name: Frontend differential · 1020 cases
|
|
needs: [resolve-refs, build-release-head]
|
|
runs-on: [self-hosted, moli-ci]
|
|
timeout-minutes: 45
|
|
container: mcr.microsoft.com/playwright/python:v1.58.0-noble
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
persist-credentials: false
|
|
ref: ${{ needs.resolve-refs.outputs.head_sha }}
|
|
- name: Use HEAD release
|
|
uses: ./.github/actions/use-ci-release
|
|
with:
|
|
side: head
|
|
revision: ${{ needs.resolve-refs.outputs.head_sha }}
|
|
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
|
with:
|
|
version: 0.11.14
|
|
python-version: "3.12"
|
|
enable-cache: true
|
|
cache-dependency-glob: |
|
|
moli-frontend-smoke/uv.lock
|
|
moli-cdp-smoke/uv.lock
|
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
|
with:
|
|
node-version: 24.15.0
|
|
- name: Install dependencies and build fixtures
|
|
run: |
|
|
uv sync --project moli-frontend-smoke --locked
|
|
uv sync --project moli-cdp-smoke --locked
|
|
npm ci --prefix moli-frontend-smoke
|
|
npm run build --prefix moli-frontend-smoke
|
|
- name: Resolve pinned Chromium
|
|
id: chromium
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
chromium_bin=$(uv run --project moli-cdp-smoke --no-sync python -c \
|
|
'from playwright.sync_api import sync_playwright; p = sync_playwright().start(); print(p.chromium.executable_path); p.stop()')
|
|
test -x "$chromium_bin"
|
|
wrapper="$PWD/target/ci-bin/chromium-ci"
|
|
printf '%s\n' '#!/bin/sh' "exec \"$chromium_bin\" --no-sandbox \"\$@\"" > "$wrapper"
|
|
chmod 0755 "$wrapper"
|
|
echo "binary=$wrapper" >> "$GITHUB_OUTPUT"
|
|
- name: Run full differential
|
|
id: frontend
|
|
continue-on-error: true
|
|
# The full catalog runs on shared hosts; leave headroom for transient
|
|
# Chromium stalls without changing any readiness or comparison rules.
|
|
run: >-
|
|
uv run --project moli-frontend-smoke --no-sync moli-frontend-smoke
|
|
--chromium-bin "${{ steps.chromium.outputs.binary }}"
|
|
--moli-bin "$PWD/target/ci-bin/moli-release-head/moli"
|
|
--timeout-ms 30000
|
|
--reference-infrastructure-retries 1
|
|
--output target/frontend-smoke
|
|
- name: Validate frontend artifacts
|
|
id: analysis
|
|
if: ${{ always() && steps.frontend.outcome != 'skipped' }}
|
|
continue-on-error: true
|
|
run: >-
|
|
uv run --project moli-frontend-smoke --no-sync
|
|
moli-frontend-smoke-analyze results target/frontend-smoke
|
|
- name: Upload frontend evidence
|
|
if: ${{ always() && steps.frontend.outcome != 'skipped' }}
|
|
continue-on-error: true
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
|
with:
|
|
name: frontend-differential-results
|
|
path: target/frontend-smoke
|
|
if-no-files-found: warn
|
|
retention-days: 7
|
|
- name: Require differential and artifact validation
|
|
if: ${{ always() }}
|
|
env:
|
|
FRONTEND_OUTCOME: ${{ steps.frontend.outcome }}
|
|
ANALYSIS_OUTCOME: ${{ steps.analysis.outcome }}
|
|
run: |
|
|
test "$FRONTEND_OUTCOME" = success
|
|
test "$ANALYSIS_OUTCOME" = success
|
|
|
|
runtime-contracts:
|
|
name: Runtime and CDP session contracts
|
|
needs: [resolve-refs, build-release-head]
|
|
runs-on: [self-hosted, moli-ci]
|
|
timeout-minutes: 30
|
|
container: rust:1.96.1-bookworm
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
persist-credentials: false
|
|
ref: ${{ needs.resolve-refs.outputs.head_sha }}
|
|
- name: Use HEAD release
|
|
uses: ./.github/actions/use-ci-release
|
|
with:
|
|
side: head
|
|
revision: ${{ needs.resolve-refs.outputs.head_sha }}
|
|
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
|
with:
|
|
version: 0.11.14
|
|
python-version: "3.12"
|
|
enable-cache: true
|
|
cache-dependency-glob: moli-benchmark/uv.lock
|
|
- name: Install dependencies
|
|
run: uv sync --project moli-benchmark --locked
|
|
- name: Run deterministic fetch contracts
|
|
run: >-
|
|
uv run --project moli-benchmark --no-sync moli-benchmark synthetic
|
|
--moli-bin "$PWD/target/ci-bin/moli-release-head/moli"
|
|
--runs 5 --output-dir target/benchmark/runtime-synthetic
|
|
- name: Run long-lived CDP session contracts
|
|
run: >-
|
|
uv run --project moli-benchmark --no-sync moli-benchmark cdp-session
|
|
--moli-bin "$PWD/target/ci-bin/moli-release-head/moli"
|
|
--target moli-cdp --gate-target moli-cdp --runs 3
|
|
--output-dir target/benchmark/cdp-session
|
|
- name: Upload evidence
|
|
if: ${{ always() }}
|
|
continue-on-error: true
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
|
with:
|
|
name: runtime-contract-results
|
|
path: target/benchmark
|
|
if-no-files-found: warn
|
|
retention-days: 7
|
|
|
|
agent-episode:
|
|
name: Agent episodes · Moli vs Chromium
|
|
needs: [resolve-refs, build-release-head]
|
|
runs-on: [self-hosted, moli-ci]
|
|
timeout-minutes: 45
|
|
container: mcr.microsoft.com/playwright/python:v1.58.0-noble
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
persist-credentials: false
|
|
ref: ${{ needs.resolve-refs.outputs.head_sha }}
|
|
- name: Use HEAD release
|
|
uses: ./.github/actions/use-ci-release
|
|
with:
|
|
side: head
|
|
revision: ${{ needs.resolve-refs.outputs.head_sha }}
|
|
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
|
with:
|
|
version: 0.11.14
|
|
python-version: "3.12"
|
|
enable-cache: true
|
|
cache-dependency-glob: |
|
|
moli-benchmark/uv.lock
|
|
moli-cdp-smoke/uv.lock
|
|
- name: Install dependencies
|
|
run: |
|
|
uv sync --project moli-benchmark --locked
|
|
uv sync --project moli-cdp-smoke --locked
|
|
- name: Resolve pinned Chromium
|
|
id: chromium
|
|
run: |
|
|
chromium_bin=$(uv run --project moli-cdp-smoke --no-sync python -c \
|
|
'from playwright.sync_api import sync_playwright; p = sync_playwright().start(); print(p.chromium.executable_path); p.stop()')
|
|
test -x "$chromium_bin"
|
|
echo "binary=$chromium_bin" >> "$GITHUB_OUTPUT"
|
|
- name: Run fast cross-engine contract
|
|
run: >-
|
|
uv run --project moli-benchmark --no-sync moli-benchmark agent-episode
|
|
--moli-bin "$PWD/target/ci-bin/moli-release-head/moli"
|
|
--chrome-bin "${{ steps.chromium.outputs.binary }}"
|
|
--step-dwell-ms 0 --output-dir target/benchmark/agent-fast
|
|
- name: Run idle-resource episode
|
|
run: >-
|
|
uv run --project moli-benchmark --no-sync moli-benchmark agent-episode
|
|
--moli-bin "$PWD/target/ci-bin/moli-release-head/moli"
|
|
--chrome-bin "${{ steps.chromium.outputs.binary }}"
|
|
--output-dir target/benchmark/agent-resource
|
|
- name: Upload evidence
|
|
if: ${{ always() }}
|
|
continue-on-error: true
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
|
with:
|
|
name: agent-episode-results
|
|
path: target/benchmark
|
|
if-no-files-found: warn
|
|
retention-days: 7
|
|
|
|
release-regression:
|
|
name: Release startup, size, and matrix A/B
|
|
needs: [resolve-refs, build-release-head, build-release-base]
|
|
runs-on: [self-hosted, moli-ci]
|
|
timeout-minutes: 60
|
|
container: rust:1.96.1-bookworm
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
persist-credentials: false
|
|
ref: ${{ needs.resolve-refs.outputs.head_sha }}
|
|
- name: Use HEAD release
|
|
uses: ./.github/actions/use-ci-release
|
|
with:
|
|
side: head
|
|
revision: ${{ needs.resolve-refs.outputs.head_sha }}
|
|
- name: Use base release
|
|
uses: ./.github/actions/use-ci-release
|
|
with:
|
|
side: base
|
|
revision: ${{ needs.resolve-refs.outputs.base_sha }}
|
|
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
|
with:
|
|
version: 0.11.14
|
|
python-version: "3.12"
|
|
enable-cache: true
|
|
cache-dependency-glob: moli-benchmark/uv.lock
|
|
- run: uv sync --project moli-benchmark --locked
|
|
- name: Measure common-ancestor startup and package
|
|
continue-on-error: true
|
|
run: >-
|
|
uv run --project moli-benchmark --no-sync moli-benchmark startup
|
|
--moli-bin "$PWD/target/ci-bin/moli-release-base/moli"
|
|
--profile formal --output-dir target/release-regression/base-startup
|
|
- name: Require HEAD startup and package workflow
|
|
run: >-
|
|
uv run --project moli-benchmark --no-sync moli-benchmark startup
|
|
--moli-bin "$PWD/target/ci-bin/moli-release-head/moli"
|
|
--profile formal --baseline-report target/release-regression/base-startup
|
|
--output-dir target/release-regression/head-startup
|
|
- name: Measure common-ancestor concurrency matrix
|
|
continue-on-error: true
|
|
run: >-
|
|
uv run --project moli-benchmark --no-sync moli-benchmark synthetic-matrix
|
|
--moli-bin "$PWD/target/ci-bin/moli-release-base/moli"
|
|
--profile smoke --output-dir target/release-regression/base-matrix
|
|
- name: Require HEAD concurrency matrix
|
|
run: >-
|
|
uv run --project moli-benchmark --no-sync moli-benchmark synthetic-matrix
|
|
--moli-bin "$PWD/target/ci-bin/moli-release-head/moli"
|
|
--profile smoke --baseline-report target/release-regression/base-matrix
|
|
--output-dir target/release-regression/head-matrix
|
|
- name: Upload evidence
|
|
if: ${{ always() }}
|
|
continue-on-error: true
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
|
with:
|
|
name: release-regression-results
|
|
# Package sizes stay in the JSON reports and image manifests. Upload
|
|
# report data only instead of duplicating the full rootfs archives.
|
|
path: |
|
|
target/release-regression/**/*.json
|
|
target/release-regression/**/*.csv
|
|
target/release-regression/**/*.md
|
|
if-no-files-found: warn
|
|
retention-days: 7
|
|
|
|
spider-fixture:
|
|
name: Deterministic Spider fixture
|
|
needs: [resolve-refs, build-release-head]
|
|
runs-on: [self-hosted, moli-ci]
|
|
timeout-minutes: 20
|
|
container: rust:1.96.1-bookworm
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
persist-credentials: false
|
|
ref: ${{ needs.resolve-refs.outputs.head_sha }}
|
|
- name: Use HEAD release
|
|
uses: ./.github/actions/use-ci-release
|
|
with:
|
|
side: head
|
|
revision: ${{ needs.resolve-refs.outputs.head_sha }}
|
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
|
with:
|
|
node-version: 24.15.0
|
|
- run: npm ci --prefix moli-benchmark/browser-spider-local
|
|
- name: Require exact fixture contract
|
|
run: >-
|
|
node moli-benchmark/browser-spider-local/bench.mjs
|
|
--target moli
|
|
--moli-bin "$PWD/target/ci-bin/moli-release-head/moli"
|
|
--workers 1 --parallelism 1 --runs 1
|
|
--site-source fixture --goto-timeout-ms 1500 --server-hang-ms 3000
|
|
--sample-interval-ms 100 --assert-fixture-contract
|
|
--output-dir target/spider-fixture
|
|
- name: Upload evidence
|
|
if: ${{ always() }}
|
|
continue-on-error: true
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
|
with:
|
|
name: spider-fixture-results
|
|
path: target/spider-fixture
|
|
if-no-files-found: warn
|
|
retention-days: 7
|
|
|
|
spider-bench:
|
|
name: Spider public-web A/B
|
|
needs: [resolve-refs, build-release-head, build-release-base]
|
|
if: >-
|
|
github.event_name != 'pull_request' ||
|
|
(github.event.pull_request.head.repo.full_name == github.repository &&
|
|
github.actor != 'dependabot[bot]')
|
|
runs-on: [self-hosted, moli-ci]
|
|
timeout-minutes: 90
|
|
container: rust:1.96.1-bookworm
|
|
env:
|
|
BENCHMARK_OUTPUT: spider-bench-results
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
persist-credentials: false
|
|
ref: ${{ needs.resolve-refs.outputs.head_sha }}
|
|
- name: Use HEAD release
|
|
uses: ./.github/actions/use-ci-release
|
|
with:
|
|
side: head
|
|
revision: ${{ needs.resolve-refs.outputs.head_sha }}
|
|
- name: Use base release
|
|
uses: ./.github/actions/use-ci-release
|
|
with:
|
|
side: base
|
|
revision: ${{ needs.resolve-refs.outputs.base_sha }}
|
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
|
with:
|
|
node-version: 24.15.0
|
|
- run: npm ci --prefix moli-benchmark/browser-spider-local
|
|
- name: Run deterministic and public-web A/B
|
|
run: >-
|
|
node moli-benchmark/browser-spider-local/compare.mjs run
|
|
--base-bin "$PWD/target/ci-bin/moli-release-base/moli"
|
|
--head-bin "$PWD/target/ci-bin/moli-release-head/moli"
|
|
--base-sha "${{ needs.resolve-refs.outputs.base_sha }}"
|
|
--head-sha "${{ needs.resolve-refs.outputs.head_sha }}"
|
|
--execution-order "${{ needs.resolve-refs.outputs.execution_order }}"
|
|
--output-dir "$BENCHMARK_OUTPUT"
|
|
--run-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
|
- name: Render job summary
|
|
if: ${{ always() && !cancelled() }}
|
|
continue-on-error: true
|
|
run: |
|
|
if [[ -f "$BENCHMARK_OUTPUT/comment.md" ]]; then
|
|
cat "$BENCHMARK_OUTPUT/comment.md" >> "$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
- name: Upload full evidence
|
|
if: ${{ always() && !cancelled() }}
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
|
with:
|
|
name: spider-bench-results
|
|
path: ${{ env.BENCHMARK_OUTPUT }}
|
|
if-no-files-found: error
|
|
retention-days: 14
|
|
|
|
sequential-navigation-soak:
|
|
name: Sequential navigation soak · 200 A/B
|
|
needs: [resolve-refs, build-release-head, build-release-base]
|
|
if: >-
|
|
github.event_name != 'pull_request' ||
|
|
(github.event.pull_request.head.repo.full_name == github.repository &&
|
|
github.actor != 'dependabot[bot]')
|
|
runs-on: [self-hosted, moli-ci]
|
|
timeout-minutes: 90
|
|
container: rust:1.96.1-bookworm
|
|
env:
|
|
BENCHMARK_OUTPUT: sequential-navigation-soak-results
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
persist-credentials: false
|
|
ref: ${{ needs.resolve-refs.outputs.head_sha }}
|
|
- name: Use HEAD release
|
|
uses: ./.github/actions/use-ci-release
|
|
with:
|
|
side: head
|
|
revision: ${{ needs.resolve-refs.outputs.head_sha }}
|
|
- name: Use base release
|
|
uses: ./.github/actions/use-ci-release
|
|
with:
|
|
side: base
|
|
revision: ${{ needs.resolve-refs.outputs.base_sha }}
|
|
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
|
with:
|
|
version: 0.11.14
|
|
python-version: "3.12"
|
|
enable-cache: true
|
|
cache-dependency-glob: moli-benchmark/uv.lock
|
|
- run: uv sync --project moli-benchmark --locked
|
|
- name: Run one-session public-web A/B
|
|
id: soak
|
|
shell: bash
|
|
env:
|
|
EXECUTION_ORDER: ${{ needs.resolve-refs.outputs.execution_order }}
|
|
run: |
|
|
set -euo pipefail
|
|
run_side() {
|
|
local side=$1
|
|
local binary=$2
|
|
set +e
|
|
uv run --project moli-benchmark --no-sync python \
|
|
moli-benchmark/scripts/fuzz-cdp-sequential-navigate.py \
|
|
--engine moli --moli-bin "$binary" \
|
|
--url https://www.csdn.net/ \
|
|
--url https://segmentfault.com/ \
|
|
--url https://huaban.com/ \
|
|
--url https://example.com/ \
|
|
--rounds 50 --network-diagnostics --navigation-resource-samples \
|
|
--response-timeout 15 --dcl-timeout 14 --load-timeout 20 \
|
|
--postcheck-timeout 5 --recovery-timeout 8 \
|
|
--output "$BENCHMARK_OUTPUT/$side.json"
|
|
local status=$?
|
|
set -e
|
|
echo "${side}_exit_code=$status" >> "$GITHUB_OUTPUT"
|
|
}
|
|
if [[ "$EXECUTION_ORDER" == head-first ]]; then
|
|
run_side head "$PWD/target/ci-bin/moli-release-head/moli"
|
|
run_side base "$PWD/target/ci-bin/moli-release-base/moli"
|
|
else
|
|
run_side base "$PWD/target/ci-bin/moli-release-base/moli"
|
|
run_side head "$PWD/target/ci-bin/moli-release-head/moli"
|
|
fi
|
|
- name: Compare resilience and memory evidence
|
|
id: comparison
|
|
if: ${{ always() && !cancelled() && steps.soak.outcome == 'success' }}
|
|
env:
|
|
PYTHONPATH: moli-benchmark
|
|
run: >-
|
|
uv run --project moli-benchmark --no-sync python
|
|
-m moli_benchmark.sequential_navigate_ci compare
|
|
--base-report "$BENCHMARK_OUTPUT/base.json"
|
|
--head-report "$BENCHMARK_OUTPUT/head.json"
|
|
--base-exit-code "${{ steps.soak.outputs.base_exit_code }}"
|
|
--head-exit-code "${{ steps.soak.outputs.head_exit_code }}"
|
|
--base-sha "${{ needs.resolve-refs.outputs.base_sha }}"
|
|
--head-sha "${{ needs.resolve-refs.outputs.head_sha }}"
|
|
--execution-order "${{ needs.resolve-refs.outputs.execution_order }}"
|
|
--expected-navigations 200
|
|
--output "$BENCHMARK_OUTPUT/comparison.json"
|
|
--comment-output "$BENCHMARK_OUTPUT/comment.md"
|
|
--run-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
|
--conclusion local
|
|
- name: Render job summary
|
|
if: ${{ always() && !cancelled() }}
|
|
continue-on-error: true
|
|
run: |
|
|
if [[ -f "$BENCHMARK_OUTPUT/comment.md" ]]; then
|
|
cat "$BENCHMARK_OUTPUT/comment.md" >> "$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
- name: Upload full evidence
|
|
if: ${{ always() && !cancelled() }}
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
|
with:
|
|
name: sequential-navigation-soak-results
|
|
path: ${{ env.BENCHMARK_OUTPUT }}
|
|
if-no-files-found: error
|
|
retention-days: 14
|
|
- name: Require complete HEAD resilience evidence
|
|
if: ${{ always() && !cancelled() && steps.comparison.outcome == 'success' }}
|
|
env:
|
|
PYTHONPATH: moli-benchmark
|
|
run: >-
|
|
uv run --project moli-benchmark --no-sync python
|
|
-m moli_benchmark.sequential_navigate_ci check
|
|
--input "$BENCHMARK_OUTPUT/comparison.json"
|