mirror of
https://github.com/lexmount/moli.git
synced 2026-09-24 16:01:31 +00:00
212 lines
8.0 KiB
YAML
212 lines
8.0 KiB
YAML
name: Sequential Navigation Soak
|
|
|
|
on:
|
|
pull_request:
|
|
types: [opened, synchronize, reopened]
|
|
workflow_dispatch:
|
|
inputs:
|
|
base_ref:
|
|
description: Base commit or ref
|
|
required: false
|
|
default: main
|
|
head_ref:
|
|
description: Head commit or ref (defaults to the dispatched commit)
|
|
required: false
|
|
default: ""
|
|
|
|
concurrency:
|
|
group: sequential-navigation-soak-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
# Candidate code builds and runs without a token capable of writing to GitHub.
|
|
# A separate workflow_run workflow renders the artifact from the default branch.
|
|
permissions:
|
|
contents: read
|
|
|
|
env:
|
|
CARGO_TERM_COLOR: always
|
|
CARGO_INCREMENTAL: "0"
|
|
|
|
jobs:
|
|
compare:
|
|
name: 200 navigations · Base vs HEAD
|
|
if: >-
|
|
github.event_name != 'pull_request' ||
|
|
(github.event.pull_request.head.repo.full_name == github.repository &&
|
|
github.actor != 'dependabot[bot]')
|
|
runs-on: [self-hosted, moli-ci]
|
|
timeout-minutes: 90
|
|
container: rust:1.96.1-bookworm
|
|
env:
|
|
BENCHMARK_OUTPUT: sequential-navigation-soak-results
|
|
BINARY_DIR: /tmp/moli-sequential-navigation-binaries
|
|
TARGET_BASE_REF: ${{ github.event.pull_request.base.sha || inputs.base_ref || github.event.repository.default_branch }}
|
|
HEAD_REF: ${{ github.event.pull_request.head.sha || inputs.head_ref || github.sha }}
|
|
steps:
|
|
# The exact HEAD harness drives both binaries, so an A/B run uses one
|
|
# report schema and one URL order. This job has read-only permissions.
|
|
- name: Checkout exact HEAD harness
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
ref: ${{ github.event.pull_request.head.sha || inputs.head_ref || github.sha }}
|
|
|
|
- name: Resolve commits and initialize evidence
|
|
id: refs
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
git config --global --add safe.directory "$(pwd -P)"
|
|
|
|
mkdir -p "$BENCHMARK_OUTPUT"
|
|
printf '%s\n' '{"schema":"moli.sequential-navigation.run-metadata.v1","status":"initializing"}' \
|
|
> "$BENCHMARK_OUTPUT/run-metadata.json"
|
|
|
|
target_base_sha=$(git rev-parse "${TARGET_BASE_REF}^{commit}")
|
|
head_sha=$(git rev-parse "${HEAD_REF}^{commit}")
|
|
base_sha=$(git merge-base "$target_base_sha" "$head_sha")
|
|
git merge-base --is-ancestor "$base_sha" "$target_base_sha"
|
|
git merge-base --is-ancestor "$base_sha" "$head_sha"
|
|
echo "target_base_sha=$target_base_sha" >> "$GITHUB_OUTPUT"
|
|
echo "base_sha=$base_sha" >> "$GITHUB_OUTPUT"
|
|
echo "head_sha=$head_sha" >> "$GITHUB_OUTPUT"
|
|
|
|
if (( GITHUB_RUN_NUMBER % 2 == 0 )); then
|
|
execution_order=head-first
|
|
else
|
|
execution_order=base-first
|
|
fi
|
|
echo "execution_order=$execution_order" >> "$GITHUB_OUTPUT"
|
|
|
|
printf '{\n "schema": "moli.sequential-navigation.run-metadata.v1",\n "status": "resolved",\n "target_base": { "sha": "%s" },\n "base": { "sha": "%s", "role": "common-ancestor" },\n "head": { "sha": "%s" },\n "harness": { "sha": "%s", "role": "exact-head" },\n "execution_order": "%s",\n "navigation_count": 200\n}\n' \
|
|
"$target_base_sha" "$base_sha" "$head_sha" "$head_sha" \
|
|
"$execution_order" > "$BENCHMARK_OUTPUT/run-metadata.json"
|
|
|
|
- name: Build common-ancestor base and exact HEAD release binaries
|
|
shell: bash
|
|
env:
|
|
BASE_SHA: ${{ steps.refs.outputs.base_sha }}
|
|
HEAD_SHA: ${{ steps.refs.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
mkdir -p "$BINARY_DIR"
|
|
|
|
restore_harness() {
|
|
git checkout --detach "$HEAD_SHA"
|
|
}
|
|
trap restore_harness EXIT
|
|
|
|
git checkout --detach "$BASE_SHA"
|
|
cargo build --release --locked -p moli
|
|
install -m 755 target/release/moli "$BINARY_DIR/moli-base"
|
|
|
|
git checkout --detach "$HEAD_SHA"
|
|
cargo build --release --locked -p moli
|
|
install -m 755 target/release/moli "$BINARY_DIR/moli-head"
|
|
|
|
restore_harness
|
|
trap - EXIT
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
|
with:
|
|
version: 0.11.14
|
|
python-version: "3.12"
|
|
enable-cache: true
|
|
cache-dependency-glob: moli-benchmark/uv.lock
|
|
|
|
- name: Install benchmark dependencies
|
|
run: uv sync --project moli-benchmark --locked
|
|
|
|
- name: Run one-session public-web A/B
|
|
id: soak
|
|
shell: bash
|
|
env:
|
|
EXECUTION_ORDER: ${{ steps.refs.outputs.execution_order }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
run_side() {
|
|
local side=$1
|
|
local binary=$2
|
|
local report="$BENCHMARK_OUTPUT/$side.json"
|
|
set +e
|
|
uv run --project moli-benchmark --no-sync python \
|
|
moli-benchmark/scripts/fuzz-cdp-sequential-navigate.py \
|
|
--engine moli \
|
|
--moli-bin "$binary" \
|
|
--url https://www.csdn.net/ \
|
|
--url https://segmentfault.com/ \
|
|
--url https://huaban.com/ \
|
|
--url https://example.com/ \
|
|
--rounds 50 \
|
|
--network-diagnostics \
|
|
--navigation-resource-samples \
|
|
--response-timeout 15 \
|
|
--dcl-timeout 14 \
|
|
--load-timeout 20 \
|
|
--postcheck-timeout 5 \
|
|
--recovery-timeout 8 \
|
|
--output "$report"
|
|
local status=$?
|
|
set -e
|
|
echo "${side}_exit_code=$status" >> "$GITHUB_OUTPUT"
|
|
}
|
|
|
|
if [[ "$EXECUTION_ORDER" == head-first ]]; then
|
|
run_side head "$BINARY_DIR/moli-head"
|
|
run_side base "$BINARY_DIR/moli-base"
|
|
else
|
|
run_side base "$BINARY_DIR/moli-base"
|
|
run_side head "$BINARY_DIR/moli-head"
|
|
fi
|
|
|
|
- name: Compare resilience and memory evidence
|
|
id: comparison
|
|
if: ${{ always() && !cancelled() && steps.soak.outcome == 'success' }}
|
|
env:
|
|
PYTHONPATH: moli-benchmark
|
|
run: >-
|
|
uv run --project moli-benchmark --no-sync python
|
|
-m moli_benchmark.sequential_navigate_ci compare
|
|
--base-report "$BENCHMARK_OUTPUT/base.json"
|
|
--head-report "$BENCHMARK_OUTPUT/head.json"
|
|
--base-exit-code "${{ steps.soak.outputs.base_exit_code }}"
|
|
--head-exit-code "${{ steps.soak.outputs.head_exit_code }}"
|
|
--base-sha "${{ steps.refs.outputs.base_sha }}"
|
|
--head-sha "${{ steps.refs.outputs.head_sha }}"
|
|
--execution-order "${{ steps.refs.outputs.execution_order }}"
|
|
--expected-navigations 200
|
|
--output "$BENCHMARK_OUTPUT/comparison.json"
|
|
--comment-output "$BENCHMARK_OUTPUT/comment.md"
|
|
--run-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
|
--conclusion local
|
|
|
|
- name: Render Actions job summary
|
|
if: ${{ always() && !cancelled() }}
|
|
continue-on-error: true
|
|
shell: bash
|
|
run: |
|
|
if [[ -f "$BENCHMARK_OUTPUT/comment.md" ]]; then
|
|
cat "$BENCHMARK_OUTPUT/comment.md" >> "$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
|
|
- name: Upload full soak evidence
|
|
if: ${{ always() && !cancelled() }}
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
|
with:
|
|
name: sequential-navigation-soak-results
|
|
path: ${{ env.BENCHMARK_OUTPUT }}
|
|
if-no-files-found: error
|
|
retention-days: 14
|
|
|
|
- name: Require complete HEAD resilience evidence
|
|
if: ${{ always() && !cancelled() && steps.comparison.outcome == 'success' }}
|
|
env:
|
|
PYTHONPATH: moli-benchmark
|
|
run: >-
|
|
uv run --project moli-benchmark --no-sync python
|
|
-m moli_benchmark.sequential_navigate_ci check
|
|
--input "$BENCHMARK_OUTPUT/comparison.json"
|