starred/moli
1
0
Fork 0
mirror of https://github.com/lexmount/moli.git synced 2026-11-01 00:07:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Public preview

Browse Source
This commit is contained in:
ldm0
2026-08-11 00:10:12 +08:00
commit 27b5135cb6
6009 changed files with 2373646 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.cargo/config.toml
+12
View File
@@ -0,0 +1,12 @@
# Most local tests spin up direct in-process HTTP servers and assert on
# exact request/response behavior. In this environment, inheriting the
# shell proxy settings would route those test requests through the host
# proxy and cause broad unrelated failures, so test builds explicitly
# clear curl's proxy when no proxy override was requested.
[env]
all_proxy = { value = "", force = true }
http_proxy = { value = "", force = true }
https_proxy = { value = "", force = true }
ALL_PROXY = { value = "", force = true }
HTTP_PROXY = { value = "", force = true }
HTTPS_PROXY = { value = "", force = true }
.config/nextest.toml
+6
View File
@@ -0,0 +1,6 @@
[profile.ci]
fail-fast = false
retries = 2
status-level = "fail"
final-status-level = "fail"
success-output = "never"
.github/workflows/ci.yml
+191
View File
@@ -0,0 +1,191 @@
name: CI
on:
pull_request:
# The closed event exists only to supersede stale queued/running PR CI.
types: [opened, synchronize, reopened, closed]
push:
branches:
- master
workflow_dispatch:
concurrency:
# github.ref changes to the base branch when a PR is merged, so use the
# stable PR number to keep its closed event in the original PR CI group.
group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
permissions:
contents: read
env:
CARGO_TERM_COLOR: always
# CI jobs use clean, one-shot builds, so incremental compiler state is not reused.
CARGO_INCREMENTAL: "0"
# actions/checkout registers safe.directory through a temporary HOME that is
# not retained by these container jobs. Keep Git trust scoped to this exact
# checkout so vergen-gitcl can read the build revision without weakening the
# container's policy for any other path.
GIT_CONFIG_COUNT: "1"
GIT_CONFIG_KEY_0: safe.directory
GIT_CONFIG_VALUE_0: ${{ github.workspace }}
jobs:
format:
name: Format
if: github.event_name != 'pull_request' || github.event.action != 'closed'
runs-on: [self-hosted, moli-ci]
container: rust:1.96.1-bookworm
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install rustfmt
run: rustup component add rustfmt
- name: Check formatting
run: cargo fmt --all --check
clippy:
name: Clippy
if: github.event_name != 'pull_request' || github.event.action != 'closed'
runs-on: [self-hosted, moli-ci]
container:
image: rust:1.96.1-bookworm
env:
CARGO_HOME: /cargo-home
# Keep Cargo's cache and its root-level lock files together across job containers.
# Build artifacts remain in the checkout's target directory and are not persisted.
volumes:
- moli-ci-cargo-home-v1:/cargo-home
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install clippy
run: rustup component add clippy
- name: Check clippy
run: cargo clippy --workspace --all-targets --all-features -- -D warnings
nextest:
name: Nextest
if: github.event_name != 'pull_request' || github.event.action != 'closed'
runs-on: [self-hosted, moli-ci]
container:
image: rust:1.96.1-bookworm
env:
CARGO_HOME: /cargo-home
# Use the same Cargo home as Clippy so concurrent Cargo processes share cache locks.
volumes:
- moli-ci-cargo-home-v1:/cargo-home
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install cargo-nextest
uses: taiki-e/install-action@13608cbb45b01feb47ef444ab1a42dc41ad56f1a # v2.79.11
with:
tool: nextest@0.9.132
# Keep one unpartitioned run so full-suite process and resource interactions
# remain visible. The ci profile owns fail-fast and retry behavior.
- name: Run full workspace tests
run: cargo nextest run --workspace --profile ci
protocol-smoke:
name: CDP and WebDriver smoke
if: github.event_name != 'pull_request' || github.event.action != 'closed'
runs-on: [self-hosted, moli-ci]
timeout-minutes: 60
container:
image: rust:1.96.1-bookworm
env:
CARGO_HOME: /cargo-home
volumes:
- moli-ci-cargo-home-v1:/cargo-home
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install uv
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
version: 0.11.14
python-version: "3.12"
enable-cache: true
cache-dependency-glob: |
moli-cdp-smoke/uv.lock
moli-webdriver-smoke/uv.lock
- name: Install Node.js
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: 24.15.0
cache: npm
cache-dependency-path: moli-cdp-smoke/package-lock.json
- name: Install smoke dependencies
run: |
uv sync --project moli-cdp-smoke --locked
uv sync --project moli-webdriver-smoke --locked
npm ci --prefix moli-cdp-smoke
- name: Build moli
id: build_moli
run: cargo build --release -p moli
- name: Test WebDriver smoke diagnostics
run: >-
uv run --project moli-webdriver-smoke --no-sync
python -m unittest discover
--start-directory moli-webdriver-smoke/tests
- name: Run default CDP smoke
id: cdp_default
continue-on-error: true
env:
MOLI_BIN: ${{ github.workspace }}/target/release/moli
run: uv run --project moli-cdp-smoke --no-sync moli-cdp-smoke
- name: Run Puppeteer CDP smoke
id: cdp_puppeteer
if: ${{ !cancelled() && steps.build_moli.outcome == 'success' }}
continue-on-error: true
env:
MOLI_BIN: ${{ github.workspace }}/target/release/moli
run: uv run --project moli-cdp-smoke --no-sync moli-cdp-smoke --group puppeteer
- name: Run full WebDriver smoke
id: webdriver
if: ${{ !cancelled() && steps.build_moli.outcome == 'success' }}
continue-on-error: true
env:
MOLI_BIN: ${{ github.workspace }}/target/release/moli
run: >-
uv run --project moli-webdriver-smoke --no-sync
moli-webdriver-smoke
--group classic,bidi,selenium,semantics,url-policy,navigation-errors
--continue-on-failure
- name: Require all protocol smoke suites
if: ${{ always() && !cancelled() && steps.build_moli.outcome == 'success' }}
env:
CDP_DEFAULT_OUTCOME: ${{ steps.cdp_default.outcome }}
CDP_PUPPETEER_OUTCOME: ${{ steps.cdp_puppeteer.outcome }}
WEBDRIVER_OUTCOME: ${{ steps.webdriver.outcome }}
run: |
failed=0
if [ "$CDP_DEFAULT_OUTCOME" != success ]; then
echo "default CDP smoke: $CDP_DEFAULT_OUTCOME"
failed=1
fi
if [ "$CDP_PUPPETEER_OUTCOME" != success ]; then
echo "Puppeteer CDP smoke: $CDP_PUPPETEER_OUTCOME"
failed=1
fi
if [ "$WEBDRIVER_OUTCOME" != success ]; then
echo "WebDriver smoke: $WEBDRIVER_OUTCOME"
failed=1
fi
exit "$failed"
.github/workflows/spider-bench-comment.yml
+120
View File
@@ -0,0 +1,120 @@
name: Spider Bench Comment
on:
workflow_run:
workflows: [Spider Bench]
types: [completed]
# This trusted workflow runs from the default branch. It never executes code
# from the PR checkout and only renders bounded numeric fields from the artifact.
permissions:
actions: read
contents: read
pull-requests: write
jobs:
comment:
name: Update PR benchmark comment
if: >-
github.event.workflow_run.event == 'pull_request' &&
github.event.workflow_run.conclusion != 'cancelled' &&
github.event.workflow_run.pull_requests[0].head.repo.id == github.event.repository.id &&
github.event.workflow_run.actor.login != 'dependabot[bot]'
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout trusted comment renderer
uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ github.event.repository.default_branch }}
- name: Download benchmark evidence
id: evidence
continue-on-error: true
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
name: spider-bench-results
path: spider-bench-results
github-token: ${{ secrets.GITHUB_TOKEN }}
run-id: ${{ github.event.workflow_run.id }}
- name: Install Node.js
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: 24.15.0
- name: Render trusted PR comment
id: render
env:
RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}
CONCLUSION: ${{ github.event.workflow_run.conclusion }}
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
run: |
if [[ "$EVIDENCE_OUTCOME" == success && -f spider-bench-results/comparison.json ]]; then
node moli-benchmark/browser-spider-local/compare.mjs comment \
--input spider-bench-results/comparison.json \
--run-url "$RUN_URL" \
--conclusion "$CONCLUSION" \
--output spider-bench-comment.md
echo "has_comparison=true" >> "$GITHUB_OUTPUT"
else
node moli-benchmark/browser-spider-local/compare.mjs infrastructure-comment \
--run-url "$RUN_URL" \
--conclusion "$CONCLUSION" \
--output spider-bench-comment.md
echo "has_comparison=false" >> "$GITHUB_OUTPUT"
fi
- name: Post PR benchmark comment
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
env:
COMMENT_PATH: spider-bench-comment.md
COMPARISON_PATH: spider-bench-results/comparison.json
HAS_COMPARISON: ${{ steps.render.outputs.has_comparison }}
with:
script: |
const fs = require('fs');
const run = context.payload.workflow_run;
const association = run.pull_requests?.[0];
if (!association) {
core.notice('No pull request is associated with this workflow run.');
return;
}
const pull = await github.rest.pulls.get({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: association.number,
});
if (pull.data.head.sha !== association.head.sha) {
core.notice('Skipping a stale Spider Bench result for an older PR head.');
return;
}
if (process.env.HAS_COMPARISON === 'true') {
const comparison = JSON.parse(fs.readFileSync(process.env.COMPARISON_PATH, 'utf8'));
const compared = await github.rest.repos.compareCommitsWithBasehead({
owner: context.repo.owner,
repo: context.repo.repo,
basehead: `${association.base.sha}...${association.head.sha}`,
});
const expectedCommonAncestor = compared.data.merge_base_commit?.sha;
if (
comparison.head?.sha !== association.head.sha ||
comparison.base?.sha !== expectedCommonAncestor
) {
core.setFailed(
'Spider Bench artifact identities do not match the PR head and common ancestor.'
);
return;
}
}
const body = fs.readFileSync(process.env.COMMENT_PATH, 'utf8');
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: association.number,
body,
});
.github/workflows/spider-bench.yml
+162
View File
@@ -0,0 +1,162 @@
name: Spider Bench
on:
pull_request:
types: [opened, synchronize, reopened]
workflow_dispatch:
inputs:
base_ref:
description: Base commit or ref
required: false
default: master
head_ref:
description: Head commit or ref (defaults to the dispatched commit)
required: false
default: ""
concurrency:
group: spider-bench-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
# PR code builds and runs without a token capable of writing back to GitHub.
# The separate Spider Bench Comment workflow consumes the resulting artifact.
permissions:
contents: read
env:
CARGO_TERM_COLOR: always
CARGO_INCREMENTAL: "0"
jobs:
compare:
name: Base vs HEAD
if: >-
github.event_name != 'pull_request' ||
(github.event.pull_request.head.repo.full_name == github.repository &&
github.actor != 'dependabot[bot]')
runs-on: [self-hosted, moli-ci]
timeout-minutes: 90
container:
image: rust:1.96.1-bookworm
env:
CARGO_HOME: /cargo-home
volumes:
- moli-ci-cargo-home-v1:/cargo-home
env:
BENCHMARK_OUTPUT: spider-bench-results
BINARY_DIR: /tmp/moli-spider-bench-binaries
TARGET_BASE_REF: ${{ github.event.pull_request.base.sha || inputs.base_ref || github.event.repository.default_branch }}
HEAD_REF: ${{ github.event.pull_request.head.sha || inputs.head_ref || github.sha }}
steps:
# The benchmark harness is infrastructure, not part of the candidate
# being measured. Keep it on the trusted target base even when the PR
# branch predates the harness itself.
- name: Checkout trusted target-base harness
uses: actions/checkout@v4
with:
fetch-depth: 0
persist-credentials: false
ref: ${{ github.event.pull_request.base.sha || inputs.base_ref || github.event.repository.default_branch }}
- name: Resolve commits and initialize evidence
id: refs
shell: bash
run: |
set -euo pipefail
# actions/checkout writes safe.directory through a temporary HOME.
# Re-register it in the container user's persistent HOME before the
# first git command run by this job.
git config --global --add safe.directory "$(pwd -P)"
mkdir -p "$BENCHMARK_OUTPUT"
printf '%s\n' '{"schema":"moli.browser-spider.run-metadata.v1","status":"initializing"}' \
> "$BENCHMARK_OUTPUT/run-metadata.json"
target_base_sha=$(git rev-parse "${TARGET_BASE_REF}^{commit}")
head_sha=$(git rev-parse "${HEAD_REF}^{commit}")
base_sha=$(git merge-base "$target_base_sha" "$head_sha")
git merge-base --is-ancestor "$base_sha" "$target_base_sha"
git merge-base --is-ancestor "$base_sha" "$head_sha"
echo "target_base_sha=$target_base_sha" >> "$GITHUB_OUTPUT"
echo "base_sha=$base_sha" >> "$GITHUB_OUTPUT"
echo "head_sha=$head_sha" >> "$GITHUB_OUTPUT"
if (( GITHUB_RUN_NUMBER % 2 == 0 )); then
execution_order=head-first
else
execution_order=base-first
fi
echo "execution_order=$execution_order" >> "$GITHUB_OUTPUT"
printf '{\n "schema": "moli.browser-spider.run-metadata.v1",\n "status": "resolved",\n "target_base": { "sha": "%s" },\n "base": { "sha": "%s", "role": "common-ancestor" },\n "head": { "sha": "%s" },\n "harness": { "sha": "%s" },\n "execution_order": "%s"\n}\n' \
"$target_base_sha" "$base_sha" "$head_sha" "$target_base_sha" \
"$execution_order" > "$BENCHMARK_OUTPUT/run-metadata.json"
- name: Build common-ancestor base and exact HEAD release binaries
shell: bash
env:
TARGET_BASE_SHA: ${{ steps.refs.outputs.target_base_sha }}
BASE_SHA: ${{ steps.refs.outputs.base_sha }}
HEAD_SHA: ${{ steps.refs.outputs.head_sha }}
run: |
set -euo pipefail
mkdir -p "$BINARY_DIR"
restore_harness() {
git checkout --detach "$TARGET_BASE_SHA"
}
trap restore_harness EXIT
git checkout --detach "$BASE_SHA"
cargo build --release --locked -p moli
install -m 755 target/release/moli "$BINARY_DIR/moli-base"
git checkout --detach "$HEAD_SHA"
cargo build --release --locked -p moli
install -m 755 target/release/moli "$BINARY_DIR/moli-head"
restore_harness
trap - EXIT
- name: Install Node.js
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: 24.15.0
cache: npm
cache-dependency-path: moli-benchmark/browser-spider-local/package-lock.json
- name: Install Spider Bench dependencies
run: npm ci --prefix moli-benchmark/browser-spider-local
- name: Run deterministic and public-web A/B
env:
EXECUTION_ORDER: ${{ steps.refs.outputs.execution_order }}
run: >-
node moli-benchmark/browser-spider-local/compare.mjs run
--base-bin "$BINARY_DIR/moli-base"
--head-bin "$BINARY_DIR/moli-head"
--base-sha "${{ steps.refs.outputs.base_sha }}"
--head-sha "${{ steps.refs.outputs.head_sha }}"
--execution-order "$EXECUTION_ORDER"
--output-dir "$BENCHMARK_OUTPUT"
--run-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
- name: Render Actions job summary
if: ${{ always() && !cancelled() }}
continue-on-error: true
shell: bash
env:
COMMENT_PATH: ${{ env.BENCHMARK_OUTPUT }}/comment.md
run: |
if [[ -f "$COMMENT_PATH" ]]; then
cat "$COMMENT_PATH" >> "$GITHUB_STEP_SUMMARY"
fi
- name: Upload full benchmark evidence
if: ${{ always() && !cancelled() }}
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: spider-bench-results
path: ${{ env.BENCHMARK_OUTPUT }}
if-no-files-found: error
retention-days: 14
.gitignore
+29
View File
@@ -0,0 +1,29 @@
/target*
/docs/wild-web-report/.wild-web-*
/docs/wild-web-report/.compare-engines-run
/docs/wild-web-report/.compare-engines-indexed.txt
/docs/wild-web-report/.curl-reachability-*.tsv
/docs/wild-web-report/.engine-comparison-last-run.txt
/docs/wild-web-report/wild-web-report-*.md
/docs/wild-web-report/wild-web-report-*.tsv
/docs/wild-web-report/engine-comparison-*.md
/docs/wild-web-report/engine-comparison-*.tsv
/.vscode
/.codex
/.claude
/.codex-tmp
/node_modules
/tmp
/vendor/*
!/vendor/v8-146.9.0/
!/vendor/v8-146.9.0/**
/.moli
__pycache__
*.pyc
/moli-playground/chatgpt/.profile/
# Accidental nested benchmark output when moli-benchmark is run from its
# own directory with a repo-relative output path.
/moli-benchmark/moli-benchmark/
node_modules
Cargo.lock
Generated
+5870
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+111
View File
@@ -0,0 +1,111 @@
[workspace]
resolver = "3"
members = [
"moli",
"moli-bounded-buffer",
"moli-bounded-json",
"moli-broadcast-channel",
"moli-browser-profile",
"moli-canvas",
"moli-charset-parser",
"moli-cookie-cache",
"moli-protocol-cdp",
"moli-cookie-store",
"moli-css-parse",
"moli-crypto",
"moli-curl",
"moli-dns-resolver",
"moli-protocol",
"moli-encoding",
"moli-core",
"moli-import-map",
"moli-image",
"moli-indexeddb",
"moli-local-executor",
"moli-layout",
"moli-message-port",
"moli-multipart",
"moli-xpath",
"moli-module-script-tree",
"moli-module-syntax",
"moli-owner-queue",
"moli-opfs",
"moli-dom",
"moli-parser",
"moli-paint",
"moli-page-types",
"moli-process-signal",
"moli-renderer-v8",
"moli-script",
"moli-selector",
"moli-site",
"moli-shared-worker",
"moli-storage-service",
"moli-storage-key",
"moli-streams",
"moli-structured-clone",
"moli-stylesheet-blocking",
"moli-svg",
"moli-test-support",
"moli-url",
"moli-url-policy",
"moli-v8-init",
"moli-v8-platform",
"moli-v8-test-util",
"moli-v8-util",
"moli-web-mime",
"moli-web-errors",
"moli-protocol-webdriver-bidi",
"moli-protocol-webdriver-classic",
"moli-webcrypto",
"moli-webapi-declare",
"moli-webapi-declare-derive",
"moli-webidl",
"moli-webidl-callback",
"moli-webidl-derive",
"moli-cookie-jar",
"moli-websocket",
"moli-fetch",
"moli-file-api",
"moli-geometry",
"moli-header-field",
"moli-html-input-temporal",
"moli-time",
"moli-trace",
"moli-http-cache",
"moli-wpt-compat"
]
[workspace.package]
license = "MIT OR Apache-2.0"
[patch.crates-io]
curl = { git = "https://github.com/ldm0/curl-rust", branch = "lightmount" }
curl-sys = { git = "https://github.com/ldm0/curl-rust", branch = "lightmount" }
cookie = { git = "https://github.com/ldm0/cookie-rs", branch = "priority" }
v8 = { path = "vendor/v8-146.9.0" }
selectors = { git = "https://github.com/ldm0/stylo", rev = "74cd014af14d84db1fc8faed85086895012fc687" }
stylo = { git = "https://github.com/ldm0/stylo", rev = "74cd014af14d84db1fc8faed85086895012fc687" }
stylo_atoms = { git = "https://github.com/ldm0/stylo", rev = "74cd014af14d84db1fc8faed85086895012fc687" }
stylo_dom = { git = "https://github.com/ldm0/stylo", rev = "74cd014af14d84db1fc8faed85086895012fc687" }
stylo_static_prefs = { git = "https://github.com/ldm0/stylo", rev = "74cd014af14d84db1fc8faed85086895012fc687" }
stylo_traits = { git = "https://github.com/ldm0/stylo", rev = "74cd014af14d84db1fc8faed85086895012fc687" }
taffy = { git = "https://github.com/ldm0/taffy", rev = "28dba60e85046442fe66fcca1ef976ca7953852e" }
[workspace.lints.clippy]
disallowed_methods = "deny"
disallowed_types = "deny"
enum_variant_names = "allow"
result_unit_err = "allow"
too_many_arguments = "allow"
type_complexity = "allow"
[profile.dev]
panic = "abort"
[profile.release]
panic = "abort"
strip = false
[profile.test]
debug = 1
LICENSE
+10
View File
@@ -0,0 +1,10 @@
Moli is licensed under either of the following licenses, at your option:
* Apache License, Version 2.0 (LICENSE-APACHE)
* MIT License (LICENSE-MIT)
SPDX-License-Identifier: MIT OR Apache-2.0
Files and directories carrying their own license notices are not relicensed
by this statement. Those components remain governed by their respective
licenses and notices.
LICENSE-APACHE
+201
View File
@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
LICENSE-MIT
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2026 Moli contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
README.md
+305
View File
@@ -0,0 +1,305 @@
<p align="center">
<img
src="assets/moli-browser-banner.png"
alt="Moli Browser — Fast, Lightweight, Headless"
width="1086"
/>
</p>
<h1 align="center">Moli</h1>
<p align="center">
<strong>A browser engine built for AI agents: DOM-first, pixels on demand.</strong>
</p>
<p align="center">
Real JavaScript, DOM, and browser APIs · One-shot, non-retained layout and CPU rendering ·
CLI, MCP, CDP, WebDriver Classic, and WebDriver BiDi
</p>
Moli is a headless browser built from the ground up for web extraction,
automation, and agent workloads. It runs real page JavaScript on V8, maintains
a native DOM and CSS state, implements browser storage and networking, and
exposes the page through structured interfaces before asking an agent to reason
over pixels.
The central idea is simple: **DOM and structured page data are available by
default; pixels are rendered only when requested.** Reading a document,
extracting Markdown, inspecting an accessibility tree, or executing JavaScript
does not need to keep a GUI browser's layout, paint, and compositor machinery
alive. When geometry or pixels do matter, Moli constructs a fresh layout pass
and can produce a software-rendered frame on demand.
## Why Moli
Conventional browsers assume that every loaded page may need to be displayed,
animated, or scrolled at any moment. They therefore keep visual machinery ready
for the next frame—even when an automation task only wants a title, a list of
links, or the result of a JavaScript expression.
An AI agent usually works differently:
1. **Understand the page.** Read DOM structure, text, links, forms, network
responses, storage, and JavaScript state.
2. **Act on the page.** Use DOM-backed controls and browser APIs to fill,
select, submit, navigate, or execute code.
3. **Look at the page when necessary.** Ask for geometry when spatial position
matters, or pixels when a screenshot is the best source of truth.
Moli performs only the visual work required by the current request:
| Agent request | Work performed by Moli |
| --- | --- |
| Extract HTML or Markdown, query the DOM, run JavaScript, or inspect network and storage | Use the browser runtime directly; do not run real layout or paint |
| Read an element box, hit-test a point, or send coordinate input | Build one complete layout pass and keep only the latest owned geometry snapshot |
| Capture a screenshot or refresh a screencast | Rebuild from the current DOM and style, software-render one fresh frame, then discard pass-local visual state |
This is not a browser with rendering removed. Moli still has a real DOM, V8,
CSS state, layout, text shaping, hit-testing, and software paint. The difference
is when those systems run and how long their intermediate state survives: DOM
and browser state are the default interface; layout and pixels are paid for
only when the agent asks for them.
That cost model matters for high-density crawling, browser-use agents,
retrieval pipelines, evaluation environments, and RL workloads, where startup
time, idle memory, and per-page resident state determine how many browser
sessions a machine can support.
## On-demand, non-retained rendering
**Moli does not keep a visual world alive just in case.** The Native DOM and
Stylo state are the single source of truth. DOM and structured-data operations
read that state directly; layout and pixels are constructed only when an
operation actually needs them.
```mermaid
flowchart LR
source["Native DOM + Stylo<br/>single source of truth"]
source --> semantic["DOM-first path<br/>HTML · Markdown · JavaScript<br/>no real layout or paint"]
source -->|geometry or visual demand| boxes["Fresh BoxBuilder"]
boxes --> layout["Taffy layout<br/>Parley text"]
layout --> projection["Immutable output projection"]
projection -->|geometry| geometry["Latest owned LayoutPassOutput<br/>hit-test · rendered text"]
projection -->|screenshot or screencast| paint["Fresh PaintSnapshot<br/>CPU software raster"]
projection -. pass ends .-> drop["Drop box tree, layout world,<br/>text and paint working state"]
```
The default server uses `LayoutPolicy::Mock` and never constructs the real
layout or paint pipeline. `--layout` selects `LayoutPolicy::OnDemand`: it makes
real geometry, hit-testing, coordinate input, screenshots, and screencast
available, but it does **not** turn Moli into a continuously rendering browser.
A cold geometry request rebuilds layout once and keeps only the latest owned
geometry snapshot. Ordinary geometry reads may reuse that sampled output;
screenshot and screencast refreshes rebuild from the current DOM and style and
produce a fresh frame. Even screencast is a low-frequency repetition of the
same one-shot pipeline, not a retained 60 FPS compositor.
As a result, extraction, DOM inspection, JavaScript execution, and most agent
actions avoid both the CPU cost of producing unused frames and the resident
memory cost of retained layout, paint, and compositor state.
## What works today
- **Real web runtime.** Streaming HTML parsing, native DOM ownership, V8-backed
JavaScript, modules, timers, microtasks, events, iframe and worker surfaces,
CSS cascade/computed style, Fetch/XHR, WebSocket, cookies, WebCrypto, and
profile-scoped storage including localStorage, IndexedDB, and OPFS.
- **Extraction-first outputs.** HTML, Markdown, JSON, text semantic trees,
frame-aware serialization, selector/script/response waits, and network
tracing are available directly from the CLI.
- **Agent-native MCP server.** The stdio server exposes navigation, Markdown,
links, JavaScript evaluation, semantic trees, interactive-element discovery,
node inspection, form actions, keyboard input, hover, and scrolling.
- **One automation binary.** CDP, WebDriver Classic, and WebDriver BiDi reuse
the same browser kernel and typed scheduler model. There is no separate
ChromeDriver, geckodriver, or browser installation to coordinate.
- **Real visual surfaces on demand.** With `--layout`, Moli uses fresh box
construction, Taffy layout, Parley text layout, layout-backed hit-testing and
input, current-viewport screenshots, and low-frequency DevTools screencast
frames rendered on the CPU.
- **Operational controls.** Profiles, cookies, HTTP cache, proxies, resource
families, connection limits, timeouts, private-network policy, user-agent
overrides, structured logging, and network diagnostics are explicit runtime
options.
<p align="center">
<a href="assets/moli-devtools-rust-lang.png">
<img
src="assets/moli-devtools-rust-lang.png"
alt="Chrome DevTools connected to Moli while inspecting and rendering rust-lang.org"
width="1200"
/>
</a>
</p>
<p align="center">
<sub>Chrome DevTools connected to Moli: rendered page, live DOM, CSS, and geometry from the same browser runtime.</sub>
</p>
## Quick start
The repository pins its Rust toolchain. Build the current development binary
from the workspace root:
```bash
cargo build --release -p moli
```
### Extract a page
Return the rendered document as Markdown after Moli's default completion
strategy:
```bash
./target/release/moli fetch \
--dump markdown \
--wait-until done \
https://example.com
```
Return a compact, model-friendly semantic tree instead:
```bash
./target/release/moli fetch \
--dump semantic_tree_text \
--wait-selector body \
https://example.com
```
`fetch --help` lists the available output formats, lifecycle waits, response
waits, profiles, proxy controls, resource policies, and tracing options.
### Start the automation server
```bash
# Basic automation server for DOM-first workloads
./target/release/moli serve
# Enable real geometry, coordinate input, and screenshot/screencast surfaces
./target/release/moli serve --layout
# Also fetch optional image, font, audio, video, media, and text-track resources
./target/release/moli serve --layout --resource
```
The same endpoint serves CDP discovery/WebSocket connections, WebDriver
Classic HTTP routes, and WebDriver BiDi connections. Playwright can connect to
the CDP endpoint directly:
```js
import { chromium } from "playwright";
const browser = await chromium.connectOverCDP("http://127.0.0.1:9222");
const context = browser.contexts()[0];
const page = context.pages()[0] ?? await context.newPage();
await page.goto("https://example.com");
console.log(await page.locator("body").innerText());
await browser.close();
```
## Cost controls
Moli keeps expensive browser work explicit rather than silently enabling it:
| Mode or option | Behavior |
| --- | --- |
| Default | `LayoutPolicy::Mock`; deterministic compatibility geometry, no real layout or paint |
| `--layout` | `LayoutPolicy::OnDemand`; real layout, geometry, hit-testing, coordinate input, screenshots, and screencast |
| `--resource` | Fetch all optional visual/media resource families |
| `--image`, `--font`, `--audio`, `--video`, `--media`, `--text-track` | Enable only the named optional resource family |
| `--profile-dir`, `--http-cache-dir`, `--cookie-file` | Opt into the persistence boundary required by the workload |
Layout is sampled rather than continuously retained. A cold geometry demand
builds a complete pass from the current Native DOM and Stylo state, and Moli
keeps only the latest owned `LayoutPassOutput`. Ordinary geometry getters may
reuse that sampled output after later mutations; screenshot and screencast
refreshes always produce a fresh frame.
## Architecture
Moli is a browser kernel, not a Chromium wrapper. Its public interfaces converge
on one Rust runtime and one set of ownership/lifecycle rules:
The implementation deliberately builds on strong Rust and browser-engine
components:
- `libcurl` for the network transport and multi-request runtime;
- `html5ever` for HTML parsing;
- `rusty_v8` and V8 for JavaScript execution;
- Servo/Stylo foundations for selectors, cascade, and computed style;
- Taffy and Parley for box and text layout;
- AnyRender/Vello CPU, `usvg`, and the Rust image ecosystem for software
rendering.
Native DOM and Moli's Stylo integration remain the unique document/style
owners. Every real refresh rebuilds layout from that source of truth, projects
the result into DOM-neutral immutable data, and destroys pass-local layout and
paint state when the operation completes. There is no incremental layout tree,
damage graph, retained display list, GPU compositor, or persistent window.
For the subsystem ownership map, see
[workspace-crate-map-current.md](docs/workspace-crate-map-current.md).
## Evidence
Moli is exercised against real sites, real automation clients, focused
Chromium/WPT behavior, and a large nextest regression suite. Two recorded
snapshots illustrate the intended operating point.
### Mixed public-web crawl
This run used 192 public URLs across major Chinese and international sites. A
page counted as successful only when it produced useful post-JavaScript
content; an HTTP 200, challenge page, login wall, empty response, or app shell
did not pass.
| Engine | Useful pages | Success rate | Median time | Median RSS |
| --- | ---: | ---: | ---: | ---: |
| **Moli** | **103** | **53.6%** | **1.43 s** | **73 MiB** |
| Chrome Headless | 101 | 52.6% | 1.43 s | 773 MiB |
| Lightpanda | 85 | 44.3% | 0.97 s | 40 MiB |
| Obscura | 57 | 29.7% | 1.30 s | 39 MiB |
### Sampled internal agent episode
| Metric | Moli | Chromium |
| --- | ---: | ---: |
| CDP ready | 34.85 ms | 169.37 ms |
| Episode active p50 | 33.40 ms | 57.13 ms |
| Peak PSS | 102.46 MiB | 348.82 MiB |
| Peak processes / threads | 1 / 24 | 11 / 123 |
Across the current WPT selection used to guard Moli's agent-browser scope, one
complete run recorded **1.612 million passing tests**.
## Project scope
Moli is in active development. It is designed to be a practical agent browser,
not a drop-in replacement for every Chrome feature.
Current intentional boundaries include:
- no GUI browser, persistent window, GPU compositor, or retained multi-frame
paint architecture;
- no promise of Chrome pixel parity or high-fidelity Canvas, WebGL, and media
playback;
- selected CDP, WebDriver Classic, and WebDriver BiDi coverage rather than
complete protocol parity;
- current-viewport software screenshots under `--layout`, without treating PDF
generation or every Chrome screenshot mode as implemented;
- resource loading, geometry freshness, and visual cost remain explicit policy
choices rather than always-on behavior.
Unsupported protocol paths should fail explicitly. They must not silently
pretend that a browser action, event, network observation, or visual result
occurred.
## License
Unless a file or directory carries a different notice, Moli is licensed under
either the [Apache License 2.0](LICENSE-APACHE) or the
[MIT License](LICENSE-MIT), at your option. Separately licensed third-party
components and fixtures retain their own licenses and notices.
assets/moli-browser-banner.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.4 MiB

assets/moli-devtools-rust-lang.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.4 MiB

clippy.toml
+28
View File
@@ -0,0 +1,28 @@
disallowed-types = [
# Use the faster and simpler non-poisonable primitives in `parking_lot` instead.
{ path = "std::sync::Mutex", reason = "Use parking_lot::Mutex for synchronous shared state. Use tokio::sync::Mutex only when a lock must be held across async suspension." },
{ path = "std::sync::MutexGuard", reason = "std::sync::Mutex poisoning is not part of Moli's lock semantics; use parking_lot::MutexGuard via parking_lot::Mutex instead." },
{ path = "std::sync::RwLock", reason = "Use parking_lot::RwLock for synchronous shared state. Use tokio::sync::RwLock only when a lock must be held across async suspension." },
{ path = "std::sync::RwLockReadGuard", reason = "std::sync::RwLock poisoning is not part of Moli's lock semantics; use parking_lot::RwLockReadGuard via parking_lot::RwLock instead." },
{ path = "std::sync::RwLockWriteGuard", reason = "std::sync::RwLock poisoning is not part of Moli's lock semantics; use parking_lot::RwLockWriteGuard via parking_lot::RwLock instead." },
{ path = "std::sync::Condvar", reason = "Use parking_lot::Condvar with parking_lot locks; avoid std condition variables and poisoning semantics." },
{ path = "std::sync::PoisonError", reason = "Moli synchronous locks intentionally do not use poisoning recovery paths; use parking_lot synchronization primitives instead." },
]
disallowed-methods = [
{ path = "std::sync::Mutex::new", reason = "Use parking_lot::Mutex::new for synchronous shared state." },
{ path = "std::sync::Mutex::lock", reason = "Use parking_lot::Mutex::lock; do not introduce std mutex poisoning semantics." },
{ path = "std::sync::Mutex::try_lock", reason = "Use parking_lot::Mutex::try_lock; do not introduce std mutex poisoning semantics." },
{ path = "std::sync::RwLock::new", reason = "Use parking_lot::RwLock::new for synchronous shared state." },
{ path = "std::sync::RwLock::read", reason = "Use parking_lot::RwLock::read; do not introduce std rwlock poisoning semantics." },
{ path = "std::sync::RwLock::write", reason = "Use parking_lot::RwLock::write; do not introduce std rwlock poisoning semantics." },
{ path = "std::sync::RwLock::try_read", reason = "Use parking_lot::RwLock::try_read; do not introduce std rwlock poisoning semantics." },
{ path = "std::sync::RwLock::try_write", reason = "Use parking_lot::RwLock::try_write; do not introduce std rwlock poisoning semantics." },
{ path = "std::sync::Condvar::new", reason = "Use parking_lot::Condvar::new with parking_lot locks." },
{ path = "std::sync::Condvar::wait", reason = "Use parking_lot::Condvar::wait with parking_lot locks." },
{ path = "std::sync::Condvar::wait_timeout", reason = "Use parking_lot::Condvar::wait_for or wait_until with parking_lot locks." },
{ path = "std::sync::Condvar::wait_while", reason = "Use parking_lot::Condvar with explicit predicate loops." },
{ path = "std::sync::Condvar::wait_timeout_while", reason = "Use parking_lot::Condvar with explicit timed predicate loops." },
{ path = "std::sync::Condvar::notify_one", reason = "Use parking_lot::Condvar::notify_one with parking_lot locks." },
{ path = "std::sync::Condvar::notify_all", reason = "Use parking_lot::Condvar::notify_all with parking_lot locks." },
]
moli-benchmark/.gitignore
+8
View File
@@ -0,0 +1,8 @@
/results/
/benchmarks/
/logs/
/.moli/
/.venv/
/.pytest_cache/
/**/*.pyc
/**/__pycache__/
moli-benchmark/README.md
+188
View File
@@ -0,0 +1,188 @@
# moli-benchmark
Python benchmark harness for the Moli benchmark standard in `docs/moli-benchmark-standard.md`.
This project writes development benchmark artifacts under `moli-benchmark/results/<timestamp>/` by default. Use `--report-date YYYY-MM-DD` for formal artifacts under `benchmarks/results/YYYY-MM-DD/`. Most suites run their own Python harnesses; Suite C WPT archival intentionally shells out to `cargo nextest run -p moli-core --test wpt_compat --release --no-fail-fast` so WPT reports use the same release profile as normal repository verification:
- `environment.json`
- `versions.json`
- `summary.json`
- `summary.md`
- `publish-readiness.json`
- `report-data.json`
- `index.html`
- optional `report-diff.{json,csv}` when `--baseline-report` is provided
- `startup/runs.{csv,json}`, `startup/gate-rows.json`, and `startup/summary.json`
- `synthetic/runs.{csv,json}` and `synthetic/summary.json`
- `synthetic-matrix/matrix.{csv,json}`, `synthetic-matrix/gate-rows.json`, `synthetic-matrix/run-summaries.json`, and `synthetic-matrix/summary.json`
- `wpt/moli-wpt-compat-report.json`, `wpt/by-tag.csv`, optional `wpt/diff.{csv,json}`, and `wpt/summary.md`
- `cdp-smoke/group-listing.json`, `cdp-smoke/preflight.json`, `cdp-smoke/client-rows.json`, `cdp-smoke/moli-cdp-smoke.json`, and `cdp-smoke/summary.md`
- `synthetic-compare/runs.{csv,json}`, `synthetic-compare/summary.json`, and top-level `index.html`
- `cdp-session/runs.{csv,json}` for long-lived CDP sessions.
- `agent-episode/report-data.json`, offline `agent-episode/index.html`, raw
episode/step rows, resource timelines, phase markers, and bounded failure
artifacts for deterministic RL-shaped CDP workflows.
- `crawler/raw-runs.csv` for local multi-page crawler runs.
- `amiibo-crawler/raw-runs.csv` for the Python raw-CDP Amiibo crawler.
- `wild-web/raw-runs.csv`, `wild-web/failures/`, and optional `wild-web/replay/` captures for real-site seed classification, extraction assertions, failure snapshots, and explicit replay fixtures.
- `top-sites/raw-runs.csv`, `top-sites/runs.json`, `top-sites/summary.json`, and `top-sites/failures/` for public-web fetch/DCL benchmarks. The default source remains `docs/chinese-community-top100-websites.md` (`quick` = top 20, `full` = top 100). `--target moli-cdp`, `--target lightpanda-cdp`, and `--target obscura-cdp` run the same URL set through each engine's CDP server and dump DOM after the CDP DOMContentLoaded signal. `--target moli-full` and `--target moli-full-cdp` reuse the Moli binary with `--layout --resource`; ordinary Moli targets keep the product-default Mock layout and disabled optional resources. `--source webfetch-mix --profile webfetch` combines 100 mixed Chinese/global top sites with observed longtail WebFetch URL paths from `docs/webfetch-longtail-seed-list.md`. `--source render-quality` uses concrete article/document URLs from `docs/render-quality-seed-list.md` for rendered-DOM quality regression checks. `--source legacy-encoding` uses non-UTF-8 pages from `docs/legacy-encoding-websites-seed-list.md`.
- `render-compare/raw-runs.csv`, `render-compare/runs.json`, `render-compare/fetch-runs.json`, `render-compare/baseline-runs.json`, `render-compare/baseline-sites.{csv,json}`, `render-compare/summary.json`, and `render-compare/failures/` for baseline-relative public-web render quality checks. The suite defaults to Chrome as the baseline, first runs the full baseline URL set, filters to `baseline-usable` pages, then runs target browsers only for that evaluated set. It compares target visible text with character n-gram containment, key phrase hit rate, visible text ratio, and a combined render quality score. Rows can distinguish `render-match`, `render-partial`, `render-mismatch`, `state-only-content`, and fetch-level failures. Baseline-unusable or baseline-thin pages are recorded in `baseline-sites` and skipped before target scoring.
`versions.json` records detected engine binaries for `moli`, `lightpanda`, `chrome`, and `obscura`. Suite result rows use engine/driver target variants: CLI fetch targets use `moli`, `moli-full`, `lightpanda`, and `obscura`; public-web DCL targets also support `moli-cdp`, `moli-full-cdp`, `lightpanda-cdp`, `obscura-cdp`, and `chrome` with `driver=cdp-dcl`; synthetic CDP suites use `moli-cdp`, `moli-full-cdp`, `lightpanda-cdp`, `chrome-cdp`, and `obscura-cdp`. Every row and target summary carries `engine`, `driver`, `label`, and `binary_key`, so `report-data.json` is independent of the HTML renderer. Defaults come from `MOLI_BIN` / `LIGHTPANDA_BIN` / `CHROME_BIN` / `OBSCURA_BIN`, repository `target/release/moli`, and then `PATH`. Obscura local fixture runs use an external-looking global IPv6 fixture URL when one is available, so Obscura does not see a `127.0.0.1` URL.
The first landing slice covers:
- Suite E startup/deploy subset: binary size, stripped binary size, tar.gz package size, SHA256, daemonless minimal rootfs image tar/tar.gz size, `moli serve` readiness, optional CDP first page, optional warm-process CDP page creation, optional idle footprint, `moli fetch about:blank`, local JS fetch startup, `/usr/bin/time -v` raw artifacts when GNU time is available, cgroup/procfs raw artifacts, and structured formal gate rows.
- Suite B synthetic subset: `static-html`, `js-xhr-fetch`, `dynamic-script`, `dom-heavy`, `storage-cookie`, `forms-events` through a local Python fixture server.
- Suite B synthetic matrix: repeats the synthetic suite across a concurrency matrix and records median drift for stability checks. The `formal` profile expands default runs/repeats to the P0 floor from `docs/moli-benchmark-standard.md` and emits structured `gate-rows.json`.
- Suite D CDP smoke archival: runs or archives `moli-cdp-smoke` JSON output. `cdp-smoke --profile formal` requires raw CDP, Playwright, and Puppeteer client coverage; Puppeteer coverage requires local `node` and `puppeteer-core`.
- Suite C WPT archival: runs local WPT compat per-fixture reports through release nextest, or collects existing reports with overall/per-tag pass-rate summaries and optional baseline diffing.
- Synthetic horizontal compare: runs the same fixture set across fetch-style variants `moli`, `moli-full`, `lightpanda`, `chrome`, and `obscura`, then renders a top-level professional static `index.html` report with KPI cards, P0 scorecard, headline charts, drilldown tables, and artifact links.
- CDP session compare: starts each engine as a CDP endpoint through `moli-cdp`, `moli-full-cdp`, `lightpanda-cdp`, `chrome-cdp`, and `obscura-cdp`, navigates multiple cases through a reused page session, and records compact console / JavaScript exception / network failure traces under `cdp-session/traces/` when failures or error events occur.
- Crawler and wild-web suites: separate local many-page crawling from real external seed classification.
- Amiibo crawler suite: uses a Python raw-CDP crawler against the Lightpanda demo Amiibo site so the real 933-page workload can be run from the same benchmark harness.
- Synthetic fixture cases are split by domain under `moli_benchmark/synthetic_case_groups/`; current groups are `basic`, `modules`, `dom`, and `io`.
- `synthetic-compare` exits on the selected gate target only. The default gate target is `moli`; competitor failures stay visible in JSON/CSV/HTML without blocking report generation. Obscura is included as a local adapter; the benchmark harness gives Obscura an external-looking global IPv6 fixture URL instead of a `127.0.0.1` URL.
Resource sampling runs at 100 ms and records process-tree PSS when `/proc/<pid>/smaps_rollup` is available, RSS fallback, and aggregate `ps` CPU percentage. Startup cases also request `/usr/bin/time -v` artifacts under `startup/time/`; if GNU time is unavailable, the runner archives an explicit unavailability marker instead of silently omitting the evidence. Startup cases also archive `/proc/<pid>/cgroup` plus common cgroup v2/v1 files under `startup/cgroup/` when the environment exposes them. Image size artifacts are written under `startup/image-size/` by packaging the benchmark binary plus `ldd`-discovered dynamic dependencies into a minimal rootfs tar and tar.gz without requiring a container daemon. Startup rows explicitly record `process_cache_mode` and `kernel_cache_mode`; `--drop-os-cache` attempts `/proc/sys/vm/drop_caches` as root and otherwise records an unavailable marker under `startup/cache/`.
The HTML report is a Chart.js static dashboard generated from `report-data.json`. The JSON file is the renderer-independent contract; `index.html` embeds the same payload so the report can still be opened directly from disk without a local web server. When `synthetic-compare` and `cdp-session` are present together, `report-data.json` also includes a derived `horizontal_comparisons[0]` entry named `web-scraping-variants` that joins shared cases across fetch-style and CDP variants for one side-by-side chart/table view.
The standalone `browser-spider-local/bench.mjs` runner samples at 500 ms by
default. Its sampler runs in a Node worker thread, derives interval CPU from
`/proc` tick deltas, and records complete browser process-tree RSS/PSS alongside
case and site markers. Every run automatically writes
`resource-samples.{json,csv}`, `report-data.json`, and a resource/correctness
dashboard at `index.html`. The dashboard combines resource timelines, case
quality/resource comparisons, process topology, sampler health, site latency,
outcome distribution, and per-site diagnostics. Use
`--no-resource-sampling` for an explicit
sampling-free run, or `--sample-interval-ms N` to select a custom interval
(minimum 100 ms).
The detailed contract is in
`docs/browser-spider-resource-sampling-report-design-2026-07-26.md`.
Pull requests from branches in this repository also run an exact base/HEAD
Spider Bench comparison in GitHub Actions. The deterministic local fixture is
reported as a correctness and stale-page-leakage diagnostic, while the public
48-site run provides noisier performance and compatibility evidence. Neither
result fails the PR: the check fails only when the observer cannot produce its
artifact. A trusted follow-up workflow creates a new PR comment for each
completed current-head run and links the full `spider-bench-results` artifact.
The execution and permission model is documented in
`docs/browser-spider-ci-pr-benchmark-2026-08-03.md`.
Expected rows are derived from the selected workload. Public sites default to
five rows each (240 for the full 48-site set); deterministic fixture routes
declare whether they intentionally produce five rows or no rows (40 total).
Consequently fixture failures, `--site-limit`, and full48 no longer share a
misleading hard-coded denominator. The PR comment opens the public result first
and includes site coverage, bounded outcome counts, and per-category rows.
For a compact leadership-facing local comparison report, use:
```bash
uv run moli-benchmark run --profile horizontal --timeout 10
```
This preset runs `synthetic-compare` and `cdp-session` against the default target matrix with 10 runs per case unless `--runs` is explicitly provided. It is designed to produce the ten-way `moli` / `moli-cdp` / `moli-full` / `moli-full-cdp` / `lightpanda` / `lightpanda-cdp` / `chrome` / `chrome-cdp` / `obscura` / `obscura-cdp` comparison view quickly. It remains an investigation report until the independent P0 formal gates in `publish-readiness.json` also pass.
## Running
Build moli first:
```bash
cargo build --release
```
Run the default initial suite set:
```bash
cd moli-benchmark
python3 -m moli_benchmark run
```
Write a formal report directory. Generated date directories under `benchmarks/results/` are ignored by git and should be uploaded by CI or release tooling:
```bash
python3 -m moli_benchmark run --report-date 2026-05-07
```
Run focused suites:
```bash
python3 -m moli_benchmark startup --runs 5 --timeout 30
uv run moli-benchmark startup --runs 5 --include-cdp-first-page --include-cdp-warm-pages --cdp-warm-pages 10 --idle-seconds 1 --idle-seconds 5 --idle-seconds 30 --timeout 30
uv run moli-benchmark startup --profile formal --timeout 30
sudo uv run moli-benchmark startup --runs 5 --drop-os-cache --timeout 30
python3 -m moli_benchmark synthetic --case static-html --case js-xhr-fetch --runs 5 --concurrency 5 --timeout 30
python3 -m moli_benchmark synthetic-matrix --case static-html --runs 5 --matrix-concurrency 1 --matrix-concurrency 5 --matrix-repeats 2 --timeout 30
python3 -m moli_benchmark synthetic-matrix --profile formal --timeout 30
python3 -m moli_benchmark synthetic-compare --runs 5 --concurrency 3 --timeout 30
python3 -m moli_benchmark synthetic-compare --gate-target moli --runs 5 --concurrency 3 --timeout 30
uv run moli-benchmark cdp-session --case static-html --runs 5 --timeout 30
uv run moli-benchmark agent-episode --target moli-cdp --target chrome-cdp --workers 1 --parallelism 1 --runs 1 --step-dwell-ms 14000 --sample-interval-ms 500 --timeout 30 --output-dir ../target/agent-episode-bench
uv run moli-benchmark run --suite synthetic-compare --suite cdp-session --case static-html --runs 5 --timeout 30
uv run moli-benchmark run --profile horizontal --timeout 10
python3 -m moli_benchmark crawler --pages 50 --timeout 30
uv run moli-benchmark amiibo-crawler --target moli --pool 1 --limit 5 --timeout 60
uv run moli-benchmark amiibo-crawler --target moli --pool 2 --limit 5 --amiibo-mode process --timeout 60
uv run moli-benchmark amiibo-crawler --target moli --amiibo-profile formal --timeout 60
python3 -m moli_benchmark wild-web --seed baidu-home --timeout 30
python3 -m moli_benchmark wild-web --seed zhihu-home --capture-replay --timeout 30
python3 -m moli_benchmark top-sites --profile quick --timeout 15
python3 -m moli_benchmark top-sites --profile full --timeout 15 --parallelism 6
uv run moli-benchmark top-sites --target moli --target moli-cdp --target moli-full --target moli-full-cdp --profile full --timeout 30 --parallelism 2
python3 -m moli_benchmark top-sites --source webfetch-mix --profile webfetch --target moli --target moli-cdp --target moli-full --target moli-full-cdp --target obscura --target obscura-cdp --timeout 30 --parallelism 6
uv run moli-benchmark top-sites --source legacy-encoding --limit 6 --target moli --target lightpanda --target chrome --timeout 30 --parallelism 3 --chrome-parallelism 2
./scripts/run-webfetch-mix-benchmark.sh
uv run moli-benchmark render-compare --source webfetch-mix --profile webfetch --limit 50 --target moli --target lightpanda --baseline-target chrome --timeout 30 --parallelism 6
uv run moli-benchmark render-compare --source webfetch-mix --profile webfetch --limit 50 --target moli --target lightpanda --baseline-target chrome --match-threshold 0.65 --key-hit-threshold 0.70 --timeout 30 --parallelism 6
uv run moli-benchmark render-compare --source render-quality --profile quick --limit 12 --target moli --target lightpanda --baseline-target chrome --timeout 30 --parallelism 3
python3 -m moli_benchmark cdp-smoke --group protocol --timeout 30
uv run moli-benchmark cdp-smoke --profile formal --timeout 120
python3 -m moli_benchmark wpt --case abortcontroller-basic --timeout 60
python3 -m moli_benchmark wpt --no-run --baseline ../previous/wpt/moli-wpt-compat-report.json
```
`agent-episode` is one fixed local benchmark, not a family of smoke/stress/live
profiles. Both engines consume the same checked-in manifest, fixture,
`Runtime.evaluate(awaitPromise=true)` expressions, and assertions. The
canonical `14,000 ms` dwell is workload idle between steps; readiness still
comes from CDP responses and lifecycle events. Only correct episodes contribute
to latency summaries. `report-data.json` is the suite-level authority and the
HTML report is a self-contained renderer of that payload.
Compare a new top-level report with a previous report directory or `summary.json`:
```bash
python3 -m moli_benchmark run --report-date 2026-05-07 --baseline-report ../benchmarks/results/2026-05-01
```
Use an exact binary:
```bash
MOLI_BIN=../target/release/moli python3 -m moli_benchmark run --runs 5
LIGHTPANDA_BIN=/usr/local/bin/lightpanda CHROME_BIN=/usr/bin/chromium OBSCURA_BIN=$HOME/.cargo/bin/obscura python3 -m moli_benchmark collect-env
```
For publishable P0 synthetic data, use at least:
```bash
python3 -m moli_benchmark synthetic-matrix --profile formal --timeout 30
```
`--profile formal` uses all synthetic cases, the `1/5/10/25/100` concurrency matrix, `runs=100`, and `matrix-repeats=5` unless those values are explicitly overridden. The report marks formal profile requirement failures separately from workload failures.
## Remaining P0 Gaps
The current harness is a runnable benchmark skeleton with smoke coverage. It is not yet a publishable P0 benchmark report. Each run writes `publish-readiness.json`; a report remains `investigation` until the formal synthetic matrix, formal Amiibo crawler, WPT P0 smoke, CDP, startup/size, wild-web, top-level artifacts, and four-way target matrix checks all pass.
The default execution order is documented in `docs/moli-benchmark-standard.md`: formalize synthetic first, fill startup/size second, then add the real Amiibo crawler.
- `Targets Available` means the binary was detected and recorded in `versions.json`; it does not mean every suite measured that target. Each suite summary must be read separately. For example, `startup --profile formal` is currently Moli-only, while `synthetic-compare` measures fetch-style variants and `cdp-session` measures `*-cdp` variants.
- Synthetic has a formal profile for the required `1/5/10/25/100` concurrency matrix, repeated stability checks, and `RUNS=100` floor; P0 still needs an actual completed formal run artifact and follow-up profiling for slow cases.
- Startup now has a `formal` profile with `runs=10`, CDP first page, 10 warm-process page creations, idle footprint at `1s/5s/30s`, and structured `gate-rows.json`. The 2026-05-07 local formal verification passed with 0 failures; P0 still needs release/report artifact retention rather than committing large raw outputs to git. Container image measurement is intentionally out of scope.
- CDP smoke now has a formal profile that requires raw CDP, Playwright, and Puppeteer coverage from `moli-cdp-smoke`, and CDP session records compact Runtime/Log/Network traces for failed or error-bearing runs. CDP still needs wider Playwright/Puppeteer workflow depth beyond the current smoke gate.
- Crawler has a Python raw-CDP Amiibo crawler for the Lightpanda demo workload. Amiibo rows are bounded by `--timeout`, CDP page-session setup failures are archived as failure artifacts instead of blocking the queue, and fetched pages assert URL/title/body text/link-count plus Amiibo name/series/image fields. The suite supports `session` mode for one browser process with multiple page sessions and `process` mode for one browser process per worker. The default `smoke` profile runs `pool=1`, `limit=5`, and `session`; `formal` expands to the full pool matrix, both modes, and all 933 pages. P0 still needs an actual completed formal artifact.
- Wild-web has first-pass title/body keyword extraction assertions, failure taxonomy, failure snapshots, and explicit opt-in replay capture via `--capture-replay`; it still needs deeper per-site business-field extraction and curated replay usage.
- Obscura is now a runnable target adapter for binary discovery, CLI fetch, and CDP serve startup. Local benchmark fixtures use an external-looking global IPv6 URL for Obscura when the host has one, so the target binary stays unmodified.
moli-benchmark/browser-spider-local/.gitignore
+2
View File
@@ -0,0 +1,2 @@
/node_modules/
/results/
moli-benchmark/browser-spider-local/bench.mjs
+1725
View File
File diff suppressed because it is too large Load Diff
moli-benchmark/browser-spider-local/compare.mjs
+744
View File
@@ -0,0 +1,744 @@
import fs from 'node:fs';
import path from 'node:path';
import { spawn } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import {
buildResourceTimeline,
renderResourceTimelineSection
} from './lib/comparison/resource-timeline.mjs';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const BENCHMARK_ENTRYPOINT = path.join(__dirname, 'bench.mjs');
export const SPIDER_COMPARISON_SCHEMA = 'moli.browser-spider.comparison.v1';
export const SPIDER_CI_COMMENT_MARKER = '<!-- moli-spider-bench -->';
const SUITE_CONFIGS = {
fixture: {
classification: 'diagnostic',
siteSource: 'fixture',
commandTimeout: '12m',
gotoTimeoutMs: 1500,
sampleIntervalMs: 100,
extraArgs: ['--assert-no-stale-page-leakage', '--server-hang-ms', '3000']
},
real48: {
classification: 'informational',
siteSource: 'real',
commandTimeout: '30m',
gotoTimeoutMs: 60_000,
sampleIntervalMs: 500,
extraArgs: []
}
};
const CASE_ORDER = ['news', 'stocks', 'tech', 'sports', 'games', 'life'];
const SITE_OUTCOME_LABELS = [
['extracted', 'extracted'],
['http_error_with_items', 'HTTP error + rows'],
['navigation_error_with_items', 'navigation error + rows'],
['timeout_with_items', 'timeout + rows'],
['empty', 'empty'],
['http_error_empty', 'HTTP error + empty'],
['navigation_error_empty', 'navigation error + empty'],
['timeout_empty', 'timeout + empty'],
['snapshot_error_empty', 'snapshot error + empty'],
['mismatched', 'stale/mismatched']
];
const ASSESSMENT_REASON_LABELS = {
'head-report-unavailable': 'HEAD fixture report was unavailable',
'head-benchmark-failed': 'HEAD fixture command failed',
'head-service-run-failed': 'HEAD fixture browser run failed',
'head-stale-page-mismatch': 'HEAD fixture observed stale-page item leakage',
'head-timeout-returned-stale-items': 'HEAD fixture returned items after a timeout',
'head-fixture-row-contract-mismatch': 'HEAD fixture did not satisfy its explicit row contract',
'fixture-row-count-regressed': 'fixture row count regressed against base',
'fixture-fill-rate-regressed': 'fixture fill rate regressed against base'
};
function readJson(filePath) {
return JSON.parse(fs.readFileSync(filePath, 'utf8'));
}
function writeJson(filePath, value) {
fs.mkdirSync(path.dirname(filePath), { recursive: true });
fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, 'utf8');
}
function finite(value) {
return Number.isFinite(value) ? value : null;
}
function sum(values) {
return values.reduce((total, value) => total + (finite(value) ?? 0), 0);
}
function maximum(values) {
const present = values.map(finite).filter((value) => value !== null);
return present.length > 0 ? Math.max(...present) : null;
}
function round(value, digits = 3) {
if (!Number.isFinite(value)) {
return null;
}
const scale = 10 ** digits;
return Math.round(value * scale) / scale;
}
function directRunDirectories(inputDir) {
if (!fs.existsSync(inputDir)) {
return [];
}
if (fs.existsSync(path.join(inputDir, 'summary.json'))) {
return [inputDir];
}
return fs.readdirSync(inputDir, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => path.join(inputDir, entry.name))
.filter((entryPath) => fs.existsSync(path.join(entryPath, 'summary.json')));
}
function unavailableRun(exitCode, error) {
return {
availability: 'unavailable',
exit_code: exitCode,
error,
metrics: null,
resource_timeline: null
};
}
function countSiteOutcomes(reportData) {
const outcomes = {};
for (const service of reportData.services ?? []) {
for (const site of service.sites ?? []) {
const outcome = typeof site.outcome === 'string' ? site.outcome : 'unknown';
outcomes[outcome] = (outcomes[outcome] ?? 0) + 1;
}
}
return outcomes;
}
function reportSites(reportData) {
return (reportData.services ?? []).flatMap((service) => service.sites ?? []);
}
function aggregateSiteMetrics(reportData) {
const sites = reportSites(reportData);
const knownContracts = sites.filter((site) => Number.isInteger(site.expected_item_count));
const contractIsComplete = sites.length > 0 && knownContracts.length === sites.length;
const itemCount = (site) => Math.max(0, finite(site.item_count) ?? 0);
return {
total_sites: sites.length,
sites_with_rows: sites.filter((site) => itemCount(site) > 0).length,
sites_without_rows: sites.filter((site) => itemCount(site) === 0).length,
expected_row_sites: contractIsComplete
? sites.filter((site) => site.expected_item_count > 0).length
: null,
expected_empty_sites: contractIsComplete
? sites.filter((site) => site.expected_item_count === 0).length
: null,
sites_meeting_row_contract: contractIsComplete
? sites.filter((site) => itemCount(site) === site.expected_item_count).length
: null,
unexpected_empty_sites: contractIsComplete
? sites.filter((site) => site.expected_item_count > 0 && itemCount(site) === 0).length
: null,
partial_sites: contractIsComplete
? sites.filter((site) => itemCount(site) > 0 && itemCount(site) < site.expected_item_count).length
: null,
site_outcomes: countSiteOutcomes(reportData)
};
}
function aggregateCaseRows(reportData) {
const rows = Object.fromEntries(CASE_ORDER.map((caseName) => [caseName, {
actual_rows: 0,
expected_rows: 0
}]));
for (const service of reportData.services ?? []) {
for (const file of service.evaluation?.files ?? []) {
if (!CASE_ORDER.includes(file.caseName)) {
continue;
}
rows[file.caseName].actual_rows += finite(file.actualRows) ?? 0;
rows[file.caseName].expected_rows += finite(file.expectedRows) ?? 0;
}
}
return rows;
}
function aggregateRun(summary, reportData) {
const results = Array.isArray(summary.results) ? summary.results : [];
const evaluations = results
.map((result) => result.evaluation)
.filter((evaluation) => evaluation && typeof evaluation === 'object');
const resources = results
.map((result) => result.resources)
.filter((resource) => resource && typeof resource === 'object');
const leakages = results
.map((result) => result.leakage)
.filter((leakage) => leakage && typeof leakage === 'object');
const actualRows = sum(evaluations.map((evaluation) => evaluation.totalActualRows));
const expectedRows = sum(evaluations.map((evaluation) => evaluation.totalExpectedRows));
const durationMs = sum(resources.map((resource) => resource.duration_ms));
const estimatedCpuSeconds = sum(resources.map((resource) => {
const averageCpu = finite(resource.average_cpu_percent);
const duration = finite(resource.duration_ms);
return averageCpu === null || duration === null
? 0
: (averageCpu / 100) * (duration / 1000);
}));
const siteMetrics = aggregateSiteMetrics(reportData);
return {
service_runs: results.length,
successful_runs: results.filter((result) => result.success === true).length,
failed_runs: results.filter((result) => result.success !== true).length,
actual_rows: actualRows,
expected_rows: expectedRows,
fill_rate_percent: expectedRows > 0 ? round((actualRows / expectedRows) * 100, 2) : null,
suspicious_sites: sum(leakages.map((leakage) => leakage.suspiciousCount)),
mismatched_item_sites: sum(leakages.map((leakage) => leakage.mismatchedItemSites)),
timeout_with_items: sum(leakages.map((leakage) => leakage.timeoutWithItems)),
http_error_with_items: sum(leakages.map((leakage) => leakage.httpErrorWithItems)),
...siteMetrics,
case_rows: aggregateCaseRows(reportData),
duration_ms: round(durationMs),
estimated_cpu_seconds: round(estimatedCpuSeconds),
peak_rss_bytes: maximum(resources.map((resource) => resource.peak_rss_bytes)),
peak_pss_bytes: maximum(resources.map((resource) => resource.peak_pss_bytes)),
peak_process_count: maximum(resources.map((resource) => resource.peak_process_count)),
peak_thread_count: maximum(resources.map((resource) => resource.peak_thread_count))
};
}
export function readSpiderBenchmark(inputDir, exitCode) {
const runDirectories = directRunDirectories(path.resolve(inputDir));
if (runDirectories.length !== 1) {
return unavailableRun(
exitCode,
`expected one benchmark run directory, found ${runDirectories.length}`
);
}
const runDirectory = runDirectories[0];
const reportPath = path.join(runDirectory, 'report-data.json');
if (!fs.existsSync(reportPath)) {
return unavailableRun(exitCode, 'report-data.json is missing');
}
try {
const reportData = readJson(reportPath);
return {
availability: 'available',
exit_code: exitCode,
error: null,
metrics: aggregateRun(
readJson(path.join(runDirectory, 'summary.json')),
reportData
),
resource_timeline: buildResourceTimeline(reportData)
};
} catch (error) {
return unavailableRun(exitCode, `failed to read benchmark report: ${error.message}`);
}
}
function numericDelta(base, head) {
if (!Number.isFinite(base) || !Number.isFinite(head)) {
return null;
}
return {
absolute: round(head - base),
percent: base === 0 ? null : round(((head - base) / Math.abs(base)) * 100, 2)
};
}
function metricDeltas(base, head) {
if (!base.metrics || !head.metrics) {
return null;
}
return Object.fromEntries([
'actual_rows',
'fill_rate_percent',
'duration_ms',
'estimated_cpu_seconds',
'peak_rss_bytes',
'peak_pss_bytes',
'peak_thread_count',
'sites_with_rows',
'sites_meeting_row_contract',
'unexpected_empty_sites',
'partial_sites',
'mismatched_item_sites',
'timeout_with_items'
].map((name) => [name, numericDelta(base.metrics[name], head.metrics[name])]));
}
function fixtureAssessment(base, head) {
const reasons = [];
if (head.availability !== 'available') {
reasons.push('head-report-unavailable');
} else {
const metrics = head.metrics;
if (head.exit_code !== 0) {
reasons.push('head-benchmark-failed');
}
if (metrics.failed_runs > 0) {
reasons.push('head-service-run-failed');
}
if (metrics.mismatched_item_sites > 0) {
reasons.push('head-stale-page-mismatch');
}
if (metrics.timeout_with_items > 0) {
reasons.push('head-timeout-returned-stale-items');
}
const rowTotalMissed = Number.isFinite(metrics.expected_rows)
&& metrics.actual_rows !== metrics.expected_rows;
const siteContractMissed = Number.isFinite(metrics.total_sites)
&& Number.isFinite(metrics.sites_meeting_row_contract)
&& metrics.sites_meeting_row_contract !== metrics.total_sites;
if (rowTotalMissed || siteContractMissed) {
reasons.push('head-fixture-row-contract-mismatch');
}
}
if (base.availability === 'available' && head.availability === 'available') {
if (head.metrics.actual_rows < base.metrics.actual_rows) {
reasons.push('fixture-row-count-regressed');
}
if (
head.metrics.expected_rows === base.metrics.expected_rows
&& head.metrics.fill_rate_percent < base.metrics.fill_rate_percent
) {
reasons.push('fixture-fill-rate-regressed');
}
}
return { clean: reasons.length === 0, reasons };
}
function suiteComparison(classification, base, head) {
return {
classification,
base,
head,
delta: metricDeltas(base, head),
assessment: classification === 'diagnostic'
? fixtureAssessment(base, head)
: null
};
}
export function buildSpiderComparison({ baseSha, headSha, executionOrder, runs }) {
return {
schema: SPIDER_COMPARISON_SCHEMA,
generated_at: new Date().toISOString(),
base: { sha: baseSha },
head: { sha: headSha },
execution_order: executionOrder,
suites: Object.fromEntries(Object.entries(SUITE_CONFIGS).map(([name, config]) => [
name,
suiteComparison(config.classification, runs[name].base, runs[name].head)
]))
};
}
function safeSha(value) {
return /^[0-9a-f]{7,40}$/i.test(value ?? '') ? value.toLowerCase() : 'unknown';
}
function shortSha(value) {
const safe = safeSha(value);
return safe === 'unknown' ? safe : safe.slice(0, 12);
}
function formatNumber(value, digits = 2) {
const number = finite(value);
return number === null ? '—' : number.toFixed(digits);
}
function formatInteger(value) {
const number = finite(value);
return number === null ? '—' : Math.round(number).toLocaleString('en-US');
}
function formatBytes(value) {
const number = finite(value);
return number === null ? '—' : `${(number / 1024 / 1024).toFixed(2)} MiB`;
}
function formatDuration(value) {
const number = finite(value);
return number === null ? '—' : `${(number / 1000).toFixed(2)} s`;
}
function signed(value, suffix = '', digits = 2) {
const number = finite(value);
if (number === null) {
return '—';
}
return `${number > 0 ? '+' : ''}${number.toFixed(digits)}${suffix}`;
}
function deltaCell(delta, formatter, unit = '') {
if (!delta) {
return '—';
}
const percent = finite(delta.percent) === null ? '' : ` (${signed(delta.percent, '%')})`;
return `${formatter(delta.absolute)}${unit}${percent}`;
}
function metrics(run) {
return run?.availability === 'available' ? run.metrics : null;
}
function status(run) {
if (run?.availability !== 'available') {
return 'unavailable';
}
return run.exit_code === 0 && run.metrics.failed_runs === 0 ? 'success' : 'failed';
}
function rows(run) {
const value = metrics(run);
return value ? `${formatInteger(value.actual_rows)} / ${formatInteger(value.expected_rows)}` : '—';
}
function ratio(value, total) {
return Number.isFinite(value) && Number.isFinite(total)
? `${formatInteger(value)} / ${formatInteger(total)}`
: '—';
}
function siteOutcomeSummary(value) {
if (!value) {
return '—';
}
const parts = SITE_OUTCOME_LABELS
.map(([name, label]) => [finite(value.site_outcomes?.[name]), label])
.filter(([count]) => count !== null && count > 0)
.map(([count, label]) => `${label}: ${formatInteger(count)}`);
return parts.length > 0 ? parts.join('; ') : '—';
}
function caseRows(value, caseName) {
const row = value?.case_rows?.[caseName];
return row && Number.isFinite(row.expected_rows) && row.expected_rows > 0
? row
: null;
}
function caseTableForSuite(suite) {
const base = metrics(suite.base);
const head = metrics(suite.head);
const lines = [
'| Category rows | Base | HEAD | Δ |',
'| --- | ---: | ---: | ---: |'
];
for (const caseName of CASE_ORDER) {
const baseRow = caseRows(base, caseName);
const headRow = caseRows(head, caseName);
if (!baseRow && !headRow) {
continue;
}
const baseActual = baseRow?.actual_rows;
const headActual = headRow?.actual_rows;
lines.push(
`| ${caseName} | ${ratio(baseActual, baseRow?.expected_rows)} | ${ratio(headActual, headRow?.expected_rows)} | ${Number.isFinite(baseActual) && Number.isFinite(headActual) ? signed(headActual - baseActual, '', 0) : '—'} |`
);
}
return lines.length > 2 ? lines.join('\n') : '';
}
function tableForSuite(suite) {
const base = metrics(suite.base);
const head = metrics(suite.head);
const delta = suite.delta ?? {};
return [
'| Metric | Base | HEAD | Δ |',
'| --- | ---: | ---: | ---: |',
`| Run status | ${status(suite.base)} | ${status(suite.head)} | — |`,
`| Extracted rows / contract | ${rows(suite.base)} | ${rows(suite.head)} | ${deltaCell(delta.actual_rows, (value) => signed(value, '', 0))} |`,
`| Contract fill | ${base ? `${formatNumber(base.fill_rate_percent)}%` : '—'} | ${head ? `${formatNumber(head.fill_rate_percent)}%` : '—'} | ${deltaCell(delta.fill_rate_percent, (value) => signed(value), ' pp')} |`,
`| Sites with rows | ${base ? ratio(base.sites_with_rows, base.total_sites) : '—'} | ${head ? ratio(head.sites_with_rows, head.total_sites) : '—'} | ${deltaCell(delta.sites_with_rows, (value) => signed(value, '', 0))} |`,
`| Sites meeting row contract | ${base ? ratio(base.sites_meeting_row_contract, base.total_sites) : '—'} | ${head ? ratio(head.sites_meeting_row_contract, head.total_sites) : '—'} | ${deltaCell(delta.sites_meeting_row_contract, (value) => signed(value, '', 0))} |`,
`| Unexpected empty sites | ${base ? formatInteger(base.unexpected_empty_sites) : '—'} | ${head ? formatInteger(head.unexpected_empty_sites) : '—'} | ${deltaCell(delta.unexpected_empty_sites, (value) => signed(value, '', 0))} |`,
`| Partial-row sites | ${base ? formatInteger(base.partial_sites) : '—'} | ${head ? formatInteger(head.partial_sites) : '—'} | ${deltaCell(delta.partial_sites, (value) => signed(value, '', 0))} |`,
`| Site outcomes | ${base ? siteOutcomeSummary(base) : '—'} | ${head ? siteOutcomeSummary(head) : '—'} | — |`,
`| Observed duration | ${base ? formatDuration(base.duration_ms) : '—'} | ${head ? formatDuration(head.duration_ms) : '—'} | ${deltaCell(delta.duration_ms, (value) => signed(value / 1000), ' s')} |`,
`| Estimated CPU | ${base ? `${formatNumber(base.estimated_cpu_seconds)} s` : '—'} | ${head ? `${formatNumber(head.estimated_cpu_seconds)} s` : '—'} | ${deltaCell(delta.estimated_cpu_seconds, (value) => signed(value), ' s')} |`,
`| Peak PSS | ${base ? formatBytes(base.peak_pss_bytes) : '—'} | ${head ? formatBytes(head.peak_pss_bytes) : '—'} | ${deltaCell(delta.peak_pss_bytes, (value) => signed(value / 1024 / 1024), ' MiB')} |`,
`| Peak RSS | ${base ? formatBytes(base.peak_rss_bytes) : '—'} | ${head ? formatBytes(head.peak_rss_bytes) : '—'} | ${deltaCell(delta.peak_rss_bytes, (value) => signed(value / 1024 / 1024), ' MiB')} |`,
`| Peak threads | ${base ? formatInteger(base.peak_thread_count) : '—'} | ${head ? formatInteger(head.peak_thread_count) : '—'} | ${deltaCell(delta.peak_thread_count, (value) => signed(value, '', 0))} |`,
`| Mismatched sites | ${base ? formatInteger(base.mismatched_item_sites) : '—'} | ${head ? formatInteger(head.mismatched_item_sites) : '—'} | ${deltaCell(delta.mismatched_item_sites, (value) => signed(value, '', 0))} |`,
`| Timeout with items | ${base ? formatInteger(base.timeout_with_items) : '—'} | ${head ? formatInteger(head.timeout_with_items) : '—'} | ${deltaCell(delta.timeout_with_items, (value) => signed(value, '', 0))} |`
].join('\n');
}
function publicHeadline(suite) {
const head = metrics(suite.head);
if (!head) {
return '**Public HEAD:** unavailable.';
}
return `**Public HEAD:** ${rows(suite.head)} rows; ${ratio(head.sites_with_rows, head.total_sites)} sites produced rows; ${formatInteger(head.unexpected_empty_sites)} unexpected empty sites.`;
}
function fixtureContractSummary(suite) {
const head = metrics(suite.head);
if (!head) {
return 'Fixture contract details are unavailable.';
}
return `The fixture exercises ${formatInteger(head.total_sites)} routes: ${formatInteger(head.expected_row_sites)} are expected to emit rows and ${formatInteger(head.expected_empty_sites)} intentionally exercise empty, loading, or timeout behavior.`;
}
function assessmentSummary(assessment) {
if (assessment.clean) {
return '✅ Deterministic fixture diagnostic is clean.';
}
const reasons = assessment.reasons
.slice(0, 8)
.map((reason) => ASSESSMENT_REASON_LABELS[reason] ?? 'unknown fixture diagnostic')
.map((reason) => ` - ${reason}`)
.join('\n');
return `⚠️ Deterministic fixture recorded diagnostic issues (non-blocking):\n${reasons}`;
}
function safeRunUrl(value) {
try {
const url = new URL(value);
return url.protocol === 'https:' && url.hostname === 'github.com' ? url.href : null;
} catch {
return null;
}
}
function safeConclusion(value) {
return ['success', 'failure', 'cancelled', 'timed_out', 'local'].includes(value)
? value
: 'unknown';
}
export function renderSpiderInfrastructureFailure({ runUrl, conclusion }) {
const artifactUrl = safeRunUrl(runUrl);
const workflowLink = artifactUrl
? `[workflow run](${artifactUrl})`
: 'workflow run';
return [
SPIDER_CI_COMMENT_MARKER,
'## Spider Bench A/B',
'',
'⚠️ Spider Bench infrastructure failed before a comparison could be produced.',
'',
`Workflow: \`${safeConclusion(conclusion)}\`. Inspect the ${workflowLink} for the failing build, harness, or artifact step.`,
'',
'_No benchmark result or performance conclusion is available for this run._'
].join('\n');
}
export function renderSpiderComparison(comparison, { runUrl, conclusion }) {
if (comparison?.schema !== SPIDER_COMPARISON_SCHEMA) {
throw new Error(`unsupported Spider comparison schema: ${comparison?.schema}`);
}
const fixture = comparison.suites?.fixture;
const real48 = comparison.suites?.real48;
if (!fixture || !real48) {
throw new Error('Spider comparison is missing expected suites');
}
const artifactUrl = safeRunUrl(runUrl);
const workflowLink = artifactUrl
? `[workflow run and full \`spider-bench-results\` artifact](${artifactUrl})`
: 'local output or `spider-bench-results` artifact';
const renderedConclusion = safeConclusion(conclusion);
const resourceTimelines = renderResourceTimelineSection(real48);
return [
SPIDER_CI_COMMENT_MARKER,
'## Spider Bench A/B',
'',
publicHeadline(real48),
'',
assessmentSummary(fixture.assessment),
'',
`Common ancestor \`${shortSha(comparison.base?.sha)}\` → HEAD \`${shortSha(comparison.head?.sha)}\`; benchmark order: \`${comparison.execution_order === 'head-first' ? 'head-first' : 'base-first'}\`; workflow: \`${renderedConclusion}\`.`,
'',
'### Public 48-site run · informational',
'',
tableForSuite(real48),
'',
caseTableForSuite(real48),
'',
'Public-site content, timing, and memory are noisy. This single A/B run reports evidence only; site outcome counts explain missing rows without treating them as a deterministic regression.',
...(resourceTimelines ? ['', resourceTimelines] : []),
'',
'### Deterministic fixture · diagnostic',
'',
fixtureContractSummary(fixture),
'',
tableForSuite(fixture),
'',
`Full HTML, JSON, CSV, logs, and page snapshots: ${workflowLink}.`,
'',
'_Spider Bench is observational: fixture diagnostics and performance deltas never fail the PR. Investigate changes using the complete artifact and controlled repeated runs._'
].join('\n');
}
function processExit(child) {
return new Promise((resolve) => {
let settled = false;
const finish = (status) => {
if (!settled) {
settled = true;
resolve(status);
}
};
child.once('error', () => finish(125));
child.once('exit', (code) => finish(Number.isInteger(code) ? code : 125));
});
}
async function runBenchmark({ suite, side, binary, outputDir }) {
const config = SUITE_CONFIGS[suite];
const args = [
'--kill-after=30s',
config.commandTimeout,
process.execPath,
BENCHMARK_ENTRYPOINT,
'--target', 'moli',
'--moli-bin', path.resolve(binary),
'--workers', '1',
'--parallelism', '1',
'--runs', '1',
'--site-source', config.siteSource,
'--goto-timeout-ms', String(config.gotoTimeoutMs),
'--sample-interval-ms', String(config.sampleIntervalMs),
'--output-dir', outputDir,
...config.extraArgs
];
console.log(`\n[spider-ab] ${suite} ${side}: timeout ${args.join(' ')}`);
const status = await processExit(spawn('timeout', args, { stdio: 'inherit' }));
fs.writeFileSync(path.join(path.dirname(path.dirname(outputDir)), 'status', `${suite}-${side}.exit-code`), `${status}\n`);
return status;
}
function parseArgs(argv) {
const args = {};
for (let index = 0; index < argv.length; index += 2) {
const name = argv[index];
const value = argv[index + 1];
if (!name?.startsWith('--') || value === undefined) {
throw new Error(`invalid argument: ${name ?? '<missing>'}`);
}
args[name.slice(2)] = value;
}
return args;
}
function required(args, name) {
if (!args[name]) {
throw new Error(`missing --${name}`);
}
return args[name];
}
async function runComparisonCommand(args) {
const outputDir = path.resolve(required(args, 'output-dir'));
const executionOrder = args['execution-order'] === 'head-first' ? 'head-first' : 'base-first';
const binaries = {
base: required(args, 'base-bin'),
head: required(args, 'head-bin')
};
const statuses = {};
fs.mkdirSync(path.join(outputDir, 'status'), { recursive: true });
for (const suite of Object.keys(SUITE_CONFIGS)) {
statuses[suite] = {};
for (const side of executionOrder === 'head-first' ? ['head', 'base'] : ['base', 'head']) {
const suiteOutput = path.join(outputDir, suite, side);
fs.mkdirSync(suiteOutput, { recursive: true });
statuses[suite][side] = await runBenchmark({
suite,
side,
binary: binaries[side],
outputDir: suiteOutput
});
}
}
const runs = Object.fromEntries(Object.keys(SUITE_CONFIGS).map((suite) => [suite, {
base: readSpiderBenchmark(path.join(outputDir, suite, 'base'), statuses[suite].base),
head: readSpiderBenchmark(path.join(outputDir, suite, 'head'), statuses[suite].head)
}]));
const comparison = buildSpiderComparison({
baseSha: required(args, 'base-sha'),
headSha: required(args, 'head-sha'),
executionOrder,
runs
});
writeJson(path.join(outputDir, 'comparison.json'), comparison);
fs.writeFileSync(path.join(outputDir, 'comment.md'), `${renderSpiderComparison(comparison, {
runUrl: args['run-url'],
conclusion: args['run-url'] ? 'success' : 'local'
})}\n`, 'utf8');
if (!comparison.suites.fixture.assessment.clean) {
console.warn(`deterministic fixture recorded diagnostic issues: ${comparison.suites.fixture.assessment.reasons.join(', ')}`);
}
}
function renderCommand(args) {
const inputPath = required(args, 'input');
if (fs.statSync(inputPath).size > 1024 * 1024) {
throw new Error('comparison.json exceeds the 1 MiB trusted-renderer limit');
}
const comparison = readJson(inputPath);
const comment = renderSpiderComparison(comparison, {
runUrl: required(args, 'run-url'),
conclusion: required(args, 'conclusion')
});
if (Buffer.byteLength(comment, 'utf8') > 32 * 1024) {
throw new Error('rendered Spider comparison comment exceeds 32 KiB');
}
const outputPath = path.resolve(required(args, 'output'));
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
fs.writeFileSync(outputPath, `${comment}\n`, 'utf8');
}
function renderInfrastructureFailureCommand(args) {
const comment = renderSpiderInfrastructureFailure({
runUrl: required(args, 'run-url'),
conclusion: required(args, 'conclusion')
});
const outputPath = path.resolve(required(args, 'output'));
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
fs.writeFileSync(outputPath, `${comment}\n`, 'utf8');
}
async function main(argv) {
const [command, ...rest] = argv;
const args = parseArgs(rest);
if (command === 'run') {
await runComparisonCommand(args);
return;
}
if (command === 'comment') {
renderCommand(args);
return;
}
if (command === 'infrastructure-comment') {
renderInfrastructureFailureCommand(args);
return;
}
throw new Error('usage: compare.mjs run|comment|infrastructure-comment [options]');
}
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
main(process.argv.slice(2)).catch((error) => {
console.error(error.stack || error.message || String(error));
process.exitCode = 1;
});
}
moli-benchmark/browser-spider-local/lib/comparison/resource-timeline.mjs
+268
View File
@@ -0,0 +1,268 @@
// Resource timelines cross an untrusted artifact boundary before the
// default-branch renderer turns them into GitHub Markdown. Keep extraction,
// downsampling, validation, and fixed Mermaid generation in one module so the
// local comment and trusted PR comment cannot drift.
const MAX_TIMELINE_POINTS = 40;
const MAX_TIMELINE_SECONDS = 2 * 60 * 60;
const MAX_CPU_PERCENT = 100_000;
const MAX_MEMORY_MIB = 1024 * 1024;
const BYTES_PER_MIB = 1024 * 1024;
function finite(value) {
return Number.isFinite(value) ? value : null;
}
function round(value, digits = 3) {
if (!Number.isFinite(value)) {
return null;
}
const scale = 10 ** digits;
return Math.round(value * scale) / scale;
}
function maximum(values) {
const present = values.map(finite).filter((value) => value !== null);
return present.length > 0 ? Math.max(...present) : null;
}
function boundedMetric(value, maximum) {
const number = finite(value);
return number !== null && number >= 0 && number <= maximum ? number : null;
}
function normalizeResourceSample(sample) {
const elapsedMs = boundedMetric(sample?.elapsed_ms, MAX_TIMELINE_SECONDS * 1000);
if (elapsedMs === null) {
return null;
}
const total = sample?.total;
const cpuPercent = boundedMetric(total?.cpu_percent, MAX_CPU_PERCENT);
const rssBytes = boundedMetric(total?.rss_bytes, MAX_MEMORY_MIB * BYTES_PER_MIB);
const pssBytes = boundedMetric(total?.pss_bytes, MAX_MEMORY_MIB * BYTES_PER_MIB);
if (cpuPercent === null && rssBytes === null && pssBytes === null) {
return null;
}
return {
elapsed_seconds: round(elapsedMs / 1000, 3),
cpu_percent: round(cpuPercent, 2),
rss_mib: round(rssBytes === null ? null : rssBytes / BYTES_PER_MIB, 2),
pss_mib: round(pssBytes === null ? null : pssBytes / BYTES_PER_MIB, 2)
};
}
function mergeSameElapsedSamples(samples) {
const merged = [];
for (const sample of samples) {
const previous = merged.at(-1);
if (previous?.elapsed_seconds === sample.elapsed_seconds) {
for (const field of ['cpu_percent', 'rss_mib', 'pss_mib']) {
if (sample[field] !== null) {
previous[field] = sample[field];
}
}
} else {
merged.push(sample);
}
}
return merged;
}
function extremeIndices(samples) {
const indices = new Set([0, samples.length - 1]);
for (const field of ['cpu_percent', 'rss_mib', 'pss_mib']) {
let minimum = null;
let maximum = null;
for (let index = 0; index < samples.length; index += 1) {
const value = finite(samples[index][field]);
if (value === null) {
continue;
}
if (minimum === null || value < samples[minimum][field]) {
minimum = index;
}
if (maximum === null || value > samples[maximum][field]) {
maximum = index;
}
}
if (minimum !== null) {
indices.add(minimum);
}
if (maximum !== null) {
indices.add(maximum);
}
}
return indices;
}
function downsampleResourceTimeline(samples) {
if (samples.length <= MAX_TIMELINE_POINTS) {
return samples;
}
const selected = extremeIndices(samples);
const remaining = MAX_TIMELINE_POINTS - selected.size;
for (let index = 1; index <= remaining; index += 1) {
selected.add(Math.round((index * (samples.length - 1)) / (remaining + 1)));
}
// An evenly spaced point can collide with an extremum. Fill any resulting
// holes without removing the extrema that expose short-lived spikes.
if (selected.size < MAX_TIMELINE_POINTS) {
const stride = (samples.length - 1) / (MAX_TIMELINE_POINTS - 1);
for (let index = 0; index < MAX_TIMELINE_POINTS && selected.size < MAX_TIMELINE_POINTS; index += 1) {
selected.add(Math.round(index * stride));
}
}
return [...selected]
.sort((left, right) => left - right)
.map((index) => samples[index]);
}
export function buildResourceTimeline(reportData) {
const services = (reportData.services ?? [])
.filter((service) => Array.isArray(service.resources?.samples));
if (services.length !== 1) {
return null;
}
const samples = services[0].resources.samples
.map(normalizeResourceSample)
.filter((sample) => sample !== null)
.sort((left, right) => left.elapsed_seconds - right.elapsed_seconds);
const merged = mergeSameElapsedSamples(samples);
return merged.length >= 2 ? downsampleResourceTimeline(merged) : null;
}
function trustedResourceTimeline(run) {
const timeline = run?.resource_timeline;
if (
run?.availability !== 'available'
|| !Array.isArray(timeline)
|| timeline.length < 2
|| timeline.length > MAX_TIMELINE_POINTS
) {
return null;
}
const points = [];
let previousElapsed = -1;
for (const point of timeline) {
const elapsedSeconds = boundedMetric(point?.elapsed_seconds, MAX_TIMELINE_SECONDS);
if (elapsedSeconds === null || elapsedSeconds <= previousElapsed) {
return null;
}
const normalized = {
elapsed_seconds: round(elapsedSeconds, 3),
cpu_percent: round(boundedMetric(point?.cpu_percent, MAX_CPU_PERCENT), 2),
rss_mib: round(boundedMetric(point?.rss_mib, MAX_MEMORY_MIB), 2),
pss_mib: round(boundedMetric(point?.pss_mib, MAX_MEMORY_MIB), 2)
};
if (
normalized.cpu_percent === null
&& normalized.rss_mib === null
&& normalized.pss_mib === null
) {
return null;
}
points.push(normalized);
previousElapsed = elapsedSeconds;
}
return points;
}
function niceAxisMaximum(values) {
const highest = maximum(values);
if (highest === null || highest <= 0) {
return 1;
}
const padded = highest * 1.05;
const magnitude = 10 ** Math.floor(Math.log10(padded));
const normalized = padded / magnitude;
const step = normalized <= 1 ? 1 : normalized <= 2 ? 2 : normalized <= 5 ? 5 : 10;
return round(step * magnitude, 3);
}
function mermaidNumber(value) {
const number = finite(value);
return String(number === null || Object.is(number, -0) ? 0 : number);
}
function mermaidLineChart({ title, yAxis, points, fields }) {
if (points.length < 2 || fields.length === 0) {
return '';
}
const values = fields.flatMap((field) => points.map((point) => point[field]));
return [
'```mermaid',
'xychart-beta',
` title "${title}"`,
` x-axis "Elapsed seconds" [${points.map((point) => mermaidNumber(point.elapsed_seconds)).join(', ')}]`,
` y-axis "${yAxis}" 0 --> ${mermaidNumber(niceAxisMaximum(values))}`,
...fields.map((field) => ` line [${points.map((point) => mermaidNumber(point[field])).join(', ')}]`),
'```'
].join('\n');
}
function resourceChartsForRun(label, run) {
const timeline = trustedResourceTimeline(run);
if (!timeline) {
return '';
}
const cpuPoints = timeline.filter((point) => point.cpu_percent !== null);
const pairedMemoryPoints = timeline.filter(
(point) => point.rss_mib !== null && point.pss_mib !== null
);
const rssPoints = timeline.filter((point) => point.rss_mib !== null);
const pssPoints = timeline.filter((point) => point.pss_mib !== null);
const hasCompletePss = rssPoints.length >= 2
&& pairedMemoryPoints.length === rssPoints.length;
const memoryPoints = hasCompletePss
? rssPoints
: rssPoints.length >= 2
? rssPoints
: pssPoints;
const memoryFields = hasCompletePss
? ['rss_mib', 'pss_mib']
: rssPoints.length >= 2
? ['rss_mib']
: pssPoints.length >= 2
? ['pss_mib']
: [];
const memoryTitle = memoryFields.length === 2
? `${label} memory: RSS then PSS`
: `${label} memory: ${memoryFields[0] === 'pss_mib' ? 'PSS' : 'RSS'}`;
const charts = [
mermaidLineChart({
title: `${label} CPU`,
yAxis: 'CPU percent',
points: cpuPoints,
fields: ['cpu_percent']
}),
mermaidLineChart({
title: memoryTitle,
yAxis: 'Memory MiB',
points: memoryPoints,
fields: memoryFields
})
].filter(Boolean);
return charts.length > 0 ? [`#### ${label}`, ...charts].join('\n\n') : '';
}
export function renderResourceTimelineSection(suite) {
const charts = [
resourceChartsForRun('Base', suite.base),
resourceChartsForRun('HEAD', suite.head)
].filter(Boolean);
if (charts.length === 0) {
return '';
}
return [
'### CPU and memory timelines',
'',
`These are bounded ${MAX_TIMELINE_POINTS}-point views of the same complete process-tree samples used by the HTML report. CPU uses 100% per occupied logical core; memory lines are RSS first and PSS second when complete PSS samples are available.`,
'',
charts.join('\n\n')
].join('\n');
}
moli-benchmark/browser-spider-local/lib/observability/dashboard.mjs
+1026
View File
File diff suppressed because it is too large Load Diff
moli-benchmark/browser-spider-local/lib/observability/index.mjs
+117
View File
@@ -0,0 +1,117 @@
import { writeSpiderReport } from './dashboard.mjs';
import { writeResourceArtifacts } from './report-data.mjs';
import { ProcessTreeResourceSampler } from './sampler.mjs';
// One service observer owns one browser pool and its complete artifact
// lifecycle. Keeping this type private prevents benchmark code from coupling
// itself to the worker-thread sampler or report serialization.
class SpiderServiceObserver {
#finished;
#finishPromise;
#outputDir;
#sampler;
#service;
#target;
constructor({ outputDir, service, target, enabled, intervalMs }) {
this.#outputDir = outputDir;
this.#service = service;
this.#target = target;
this.#finished = false;
this.#finishPromise = null;
this.#sampler = new ProcessTreeResourceSampler({
enabled,
intervalMs
});
this.#sampler.start();
this.mark('service-start');
}
get finished() {
return this.#finished;
}
#assertOpen() {
if (this.#finishPromise) {
throw new Error(`observability service ${this.#service} is already finishing`);
}
}
mark(event, details = {}) {
this.#assertOpen();
const payload = typeof event === 'string'
? { ...details, type: event }
: { ...event };
this.#sampler.mark({
...payload,
service: this.#service,
target: this.#target
});
}
registerWorker(label, pid) {
this.#assertOpen();
this.mark('worker-spawn', { worker: label, pid });
this.#sampler.addRoot(label, pid);
}
finish() {
if (this.#finishPromise) {
return this.#finishPromise;
}
this.mark('service-stopped');
this.#finishPromise = this.#sampler.stop().then((resourceData) => {
writeResourceArtifacts(this.#outputDir, resourceData);
this.#finished = true;
return resourceData;
});
return this.#finishPromise;
}
}
export class SpiderRunObserver {
#args;
#runDir;
#services;
constructor({ runDir, args }) {
this.#runDir = runDir;
this.#args = {
...args,
targets: [...args.targets],
cases: [...args.cases]
};
this.#services = new Map();
}
beginService({ outputDir, service, target }) {
if (this.#services.has(service)) {
throw new Error(`observability service ${service} was already registered`);
}
const observer = new SpiderServiceObserver({
outputDir,
service,
target,
enabled: this.#args.sampleResources,
intervalMs: this.#args.sampleIntervalMs
});
this.#services.set(service, observer);
return observer;
}
writeReport(results) {
const unfinished = [...this.#services]
.filter(([, observer]) => !observer.finished)
.map(([service]) => service);
if (unfinished.length > 0) {
throw new Error(
`cannot render spider report before observability finishes: ${unfinished.join(', ')}`
);
}
return writeSpiderReport({
runDir: this.#runDir,
args: this.#args,
results
});
}
}
moli-benchmark/browser-spider-local/lib/observability/linux-procfs.mjs
+386
View File
@@ -0,0 +1,386 @@
import fs from 'node:fs';
import os from 'node:os';
import { execFileSync } from 'node:child_process';
import { performance } from 'node:perf_hooks';
// Linux-only collection primitives. The browser-spider runner reaches this
// module through the observability facade rather than depending on procfs.
const KIB = 1024;
function numericValue(value, fallback) {
const parsed = Number(value);
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
}
function getconf(name, fallback) {
try {
return numericValue(
execFileSync('getconf', [name], {
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'ignore']
}).trim(),
fallback
);
} catch (_error) {
return fallback;
}
}
export function parseProcStat(text) {
const open = text.indexOf('(');
const close = text.lastIndexOf(')');
if (open <= 0 || close <= open) {
throw new Error('invalid /proc stat record');
}
const pid = Number(text.slice(0, open).trim());
const fields = text.slice(close + 1).trim().split(/\s+/);
if (!Number.isInteger(pid) || fields.length < 22) {
throw new Error('truncated /proc stat record');
}
const ppid = Number(fields[1]);
const userTicks = Number(fields[11]);
const systemTicks = Number(fields[12]);
const threadCount = Number(fields[17]);
const startTimeTicks = Number(fields[19]);
const rssPages = Number(fields[21]);
if (
!Number.isInteger(ppid)
|| !Number.isFinite(userTicks)
|| !Number.isFinite(systemTicks)
|| !Number.isFinite(startTimeTicks)
) {
throw new Error('non-numeric /proc stat field');
}
return {
pid,
ppid,
state: fields[0],
cpu_ticks: userTicks + systemTicks,
start_time_ticks: startTimeTicks,
thread_count: Number.isFinite(threadCount) ? threadCount : null,
rss_pages: Number.isFinite(rssPages) ? rssPages : null
};
}
function statusKib(text, key) {
const match = new RegExp(`^${key}:\\s+(\\d+)\\s+kB$`, 'm').exec(text);
return match ? Number(match[1]) : null;
}
function statusInteger(text, key) {
const match = new RegExp(`^${key}:\\s+(\\d+)$`, 'm').exec(text);
return match ? Number(match[1]) : null;
}
export function parseProcStatus(text) {
const rssKib = statusKib(text, 'VmRSS');
return {
rss_bytes: rssKib === null ? null : rssKib * KIB,
thread_count: statusInteger(text, 'Threads')
};
}
export function parseSmapsRollup(text) {
const pssKib = statusKib(text, 'Pss');
return pssKib === null ? null : pssKib * KIB;
}
function processIdentity(process) {
return `${process.pid}:${process.start_time_ticks}`;
}
export class LinuxProcfs {
constructor({
procRoot = '/proc',
pageSize = getconf('PAGESIZE', 4096)
} = {}) {
this.procRoot = procRoot;
this.pageSize = pageSize;
}
readStat(pid) {
return parseProcStat(fs.readFileSync(`${this.procRoot}/${pid}/stat`, 'utf8'));
}
scanProcesses() {
const processes = new Map();
for (const entry of fs.readdirSync(this.procRoot, { withFileTypes: true })) {
if (!entry.isDirectory() || !/^\d+$/.test(entry.name)) {
continue;
}
try {
const process = this.readStat(Number(entry.name));
processes.set(process.pid, process);
} catch (_error) {
// Processes routinely disappear between readdir() and read(). They were
// not part of a stable snapshot and must simply be omitted.
}
}
return processes;
}
readMemory(process) {
let status = null;
try {
status = parseProcStatus(
fs.readFileSync(`${this.procRoot}/${process.pid}/status`, 'utf8')
);
} catch (_error) {
status = null;
}
let pssBytes = null;
try {
pssBytes = parseSmapsRollup(
fs.readFileSync(`${this.procRoot}/${process.pid}/smaps_rollup`, 'utf8')
);
} catch (_error) {
pssBytes = null;
}
const statRss = process.rss_pages === null
? null
: Math.max(0, process.rss_pages) * this.pageSize;
return {
rss_bytes: status?.rss_bytes ?? statRss,
pss_bytes: pssBytes,
thread_count: status?.thread_count ?? process.thread_count
};
}
}
export function processTrees(processes, roots) {
const children = new Map();
for (const process of processes.values()) {
const siblings = children.get(process.ppid) ?? [];
siblings.push(process.pid);
children.set(process.ppid, siblings);
}
const trees = new Map();
for (const [label, root] of roots) {
const currentRoot = processes.get(root.pid);
if (
!currentRoot
|| (
root.start_time_ticks !== null
&& currentRoot.start_time_ticks !== root.start_time_ticks
)
) {
trees.set(label, new Set());
continue;
}
if (root.start_time_ticks === null) {
root.start_time_ticks = currentRoot.start_time_ticks;
}
const members = new Set();
const pending = [root.pid];
while (pending.length > 0) {
const pid = pending.pop();
if (members.has(pid)) {
continue;
}
members.add(pid);
for (const child of children.get(pid) ?? []) {
pending.push(child);
}
}
trees.set(label, members);
}
return trees;
}
function aggregateTree({
members,
processes,
memoryByPid,
previousTicks,
elapsedSeconds,
ticksPerSecond
}) {
let rssBytes = 0;
let rssProcessCount = 0;
let pssBytes = 0;
let pssProcessCount = 0;
let threadCount = 0;
let cpuDeltaTicks = 0;
let cpuProcessCount = 0;
for (const pid of members) {
const process = processes.get(pid);
if (!process) {
continue;
}
const memory = memoryByPid.get(pid);
if (memory?.rss_bytes !== null && memory?.rss_bytes !== undefined) {
rssBytes += memory.rss_bytes;
rssProcessCount += 1;
}
if (memory?.pss_bytes !== null && memory?.pss_bytes !== undefined) {
pssBytes += memory.pss_bytes;
pssProcessCount += 1;
}
threadCount += memory?.thread_count ?? process.thread_count ?? 0;
const previous = previousTicks.get(processIdentity(process));
if (previous !== undefined && process.cpu_ticks >= previous) {
cpuDeltaTicks += process.cpu_ticks - previous;
cpuProcessCount += 1;
}
}
const processCount = members.size;
const cpuPercent = elapsedSeconds > 0 && cpuProcessCount > 0
? cpuDeltaTicks / ticksPerSecond / elapsedSeconds * 100
: null;
return {
cpu_percent: cpuPercent,
rss_bytes: rssProcessCount === processCount ? rssBytes : null,
rss_process_count: rssProcessCount,
pss_bytes: pssProcessCount === processCount ? pssBytes : null,
pss_process_count: pssProcessCount,
process_count: processCount,
thread_count: threadCount
};
}
export class LinuxProcessTreeCollector {
constructor({
procfs = new LinuxProcfs(),
ticksPerSecond = getconf('CLK_TCK', 100),
intervalMs = 500,
clock = () => performance.now()
} = {}) {
this.procfs = procfs;
this.ticksPerSecond = ticksPerSecond;
this.intervalMs = intervalMs;
this.clock = clock;
this.roots = new Map();
this.previousTicks = new Map();
this.previousElapsedMs = null;
this.samples = [];
this.errors = {};
}
addRoot(label, pid) {
let startTimeTicks = null;
try {
startTimeTicks = this.procfs.readStat(pid).start_time_ticks;
} catch (_error) {
// The process may have been registered in the short window before procfs
// exposes it. The first successful scan binds the immutable start time.
}
this.roots.set(label, {
pid,
start_time_ticks: startTimeTicks
});
}
recordError(kind) {
this.errors[kind] = (this.errors[kind] ?? 0) + 1;
}
sample({
elapsedMs,
wallTime = new Date().toISOString(),
kind = 'periodic'
}) {
const captureStarted = this.clock();
let processes;
try {
processes = this.procfs.scanProcesses();
} catch (_error) {
this.recordError('proc_scan_failed');
const sample = {
elapsed_ms: elapsedMs,
wall_time: wallTime,
kind,
capture_duration_ms: this.clock() - captureStarted,
total: null,
workers: {}
};
this.samples.push(sample);
return sample;
}
const trees = processTrees(processes, this.roots);
const union = new Set();
for (const members of trees.values()) {
for (const pid of members) {
union.add(pid);
}
}
const memoryByPid = new Map();
for (const pid of union) {
const process = processes.get(pid);
if (!process) {
continue;
}
const memory = this.procfs.readMemory(process);
if (memory.rss_bytes === null) {
this.recordError('rss_unavailable');
}
if (memory.pss_bytes === null) {
this.recordError('pss_unavailable');
}
memoryByPid.set(pid, memory);
}
const elapsedDeltaMs = this.previousElapsedMs === null
? 0
: Math.max(0, elapsedMs - this.previousElapsedMs);
// A forced final memory snapshot can happen immediately after a periodic
// sample. Do not turn a single scheduler tick in that tiny window into an
// artificial CPU spike.
const elapsedSeconds = elapsedDeltaMs >= this.intervalMs * 0.8
? elapsedDeltaMs / 1000
: 0;
const aggregate = (members) => aggregateTree({
members,
processes,
memoryByPid,
previousTicks: this.previousTicks,
elapsedSeconds,
ticksPerSecond: this.ticksPerSecond
});
const workers = {};
for (const [label, members] of trees) {
workers[label] = aggregate(members);
}
const total = aggregate(union);
const sample = {
elapsed_ms: elapsedMs,
wall_time: wallTime,
kind,
capture_duration_ms: 0,
total,
workers
};
sample.capture_duration_ms = this.clock() - captureStarted;
this.samples.push(sample);
this.previousElapsedMs = elapsedMs;
this.previousTicks = new Map(
[...processes.values()].map((process) => [
processIdentity(process),
process.cpu_ticks
])
);
return sample;
}
result() {
return {
platform: process.platform,
method: 'linux_procfs_process_tree',
interval_ms: this.intervalMs,
cpu_ticks_per_second: this.ticksPerSecond,
host_logical_cpu_count: os.cpus().length,
roots: Object.fromEntries(this.roots),
errors: this.errors,
samples: this.samples
};
}
}
moli-benchmark/browser-spider-local/lib/observability/report-data.mjs
+272
View File
@@ -0,0 +1,272 @@
import fs from 'node:fs';
import path from 'node:path';
export function writeJson(filePath, value) {
fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, 'utf8');
}
function csvCell(value) {
if (value === null || value === undefined) {
return '';
}
const text = String(value);
return /[",\n\r]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text;
}
export function resourceSamplesCsv(resourceData) {
const rows = [[
'elapsed_ms',
'wall_time',
'kind',
'cpu_percent',
'rss_bytes',
'rss_process_count',
'pss_bytes',
'pss_process_count',
'process_count',
'thread_count',
'capture_duration_ms'
]];
for (const sample of resourceData.samples ?? []) {
rows.push([
sample.elapsed_ms,
sample.wall_time,
sample.kind,
sample.total?.cpu_percent,
sample.total?.rss_bytes,
sample.total?.rss_process_count,
sample.total?.pss_bytes,
sample.total?.pss_process_count,
sample.total?.process_count,
sample.total?.thread_count,
sample.capture_duration_ms
]);
}
return `${rows.map((row) => row.map(csvCell).join(',')).join('\n')}\n`;
}
export function writeResourceArtifacts(outputDir, resourceData) {
const jsonPath = path.join(outputDir, 'resource-samples.json');
const csvPath = path.join(outputDir, 'resource-samples.csv');
writeJson(jsonPath, resourceData);
fs.writeFileSync(csvPath, resourceSamplesCsv(resourceData), 'utf8');
return { jsonPath, csvPath };
}
function reportConfig(args) {
return {
targets: args.targets,
workers: args.workers,
parallelism: args.parallelism,
runs: args.runs,
cases: args.cases,
site_limit: args.siteLimit,
site_source: args.siteSource,
goto_timeout_ms: args.gotoTimeoutMs,
resource_sampling: args.sampleResources,
sample_interval_ms: args.sampleIntervalMs
};
}
function siteKey(caseName, site) {
return `${caseName ?? ''}\u0000${site ?? ''}`;
}
function completedSiteIntervals(markers) {
const pending = new Map();
const completed = new Map();
for (const marker of markers ?? []) {
const key = siteKey(marker.case_name, marker.site);
if (marker.type === 'site-start') {
const queue = pending.get(key) ?? [];
queue.push(marker);
pending.set(key, queue);
continue;
}
if (marker.type !== 'site-done') {
continue;
}
const start = pending.get(key)?.shift();
if (!start) {
continue;
}
const queue = completed.get(key) ?? [];
queue.push({
worker: marker.worker ?? start.worker ?? null,
start_ms: start.elapsed_ms,
end_ms: marker.elapsed_ms,
duration_ms: Math.max(0, marker.elapsed_ms - start.elapsed_ms),
marker_success: marker.success,
marker_item_count: marker.item_count
});
completed.set(key, queue);
}
return completed;
}
function conciseError(value) {
if (!value) {
return null;
}
return String(value)
.replace(/\u001b\[[0-9;]*m/g, '')
.split(' | at ')[0]
.slice(0, 500);
}
function siteOutcome(site) {
const itemCount = Number(site.item_count) || 0;
if (site.item_classification === 'mismatched') {
return 'mismatched';
}
if (itemCount > 0 && site.timed_out) {
return 'timeout_with_items';
}
if (itemCount > 0 && site.navigation_failed) {
return 'navigation_error_with_items';
}
if (itemCount > 0 && site.http_error) {
return 'http_error_with_items';
}
if (itemCount > 0) {
return 'extracted';
}
if (site.timed_out) {
return 'timeout_empty';
}
if (site.navigation_failed) {
return 'navigation_error_empty';
}
if (site.http_error) {
return 'http_error_empty';
}
if (site.snapshot_error_summary) {
return 'snapshot_error_empty';
}
return 'empty';
}
function normalizeSite(meta, interval) {
const gotoError = conciseError(meta.gotoError);
const snapshotError = conciseError(meta.htmlSaveError);
const responseStatus = Number.isFinite(meta.responseStatus)
? meta.responseStatus
: null;
const site = {
case_name: meta.caseName ?? null,
site: meta.site ?? null,
url: meta.url ?? null,
worker: interval?.worker ?? null,
start_ms: interval?.start_ms ?? null,
end_ms: interval?.end_ms ?? null,
duration_ms: interval?.duration_ms ?? null,
goto_ok: typeof meta.gotoOk === 'boolean' ? meta.gotoOk : null,
error_summary: gotoError,
timed_out: Boolean(gotoError && /timeout/i.test(gotoError)),
navigation_failed: meta.gotoOk === false,
response_status: responseStatus,
http_error: responseStatus !== null && responseStatus >= 400,
final_url: meta.finalUrlAfterExtract ?? meta.finalUrlAfterGoto ?? null,
title: meta.title ?? null,
snapshot_saved: Boolean(meta.htmlSha256),
snapshot_characters: Number.isFinite(meta.htmlLength) ? meta.htmlLength : null,
snapshot_sha256: meta.htmlSha256 ?? null,
snapshot_error_summary: snapshotError,
expected_item_count: Number.isInteger(meta.expectedItemCount)
? meta.expectedItemCount
: null,
item_count: Number.isFinite(meta.itemCount)
? meta.itemCount
: interval?.marker_item_count ?? 0,
item_classification: meta.itemClassification ?? null,
first_item_title: meta.items?.[0]?.title ?? null,
first_item_link: meta.items?.[0]?.link ?? null
};
site.outcome = siteOutcome(site);
return site;
}
function fallbackSites(intervals) {
const rows = [];
for (const [key, queue] of intervals) {
const [caseName, site] = key.split('\u0000');
for (const interval of queue) {
const itemCount = Number(interval.marker_item_count) || 0;
rows.push({
case_name: caseName,
site,
url: null,
worker: interval.worker,
start_ms: interval.start_ms,
end_ms: interval.end_ms,
duration_ms: interval.duration_ms,
goto_ok: null,
error_summary: null,
timed_out: false,
navigation_failed: false,
response_status: null,
http_error: false,
final_url: null,
title: null,
snapshot_saved: null,
snapshot_characters: null,
snapshot_sha256: null,
snapshot_error_summary: null,
expected_item_count: null,
item_count: itemCount,
item_classification: null,
first_item_title: null,
first_item_link: null,
outcome: itemCount > 0 ? 'extracted' : 'empty'
});
}
}
return rows;
}
export function buildSiteDiagnostics(result) {
const intervals = completedSiteIntervals(result.resourceData?.markers);
const rows = [];
for (const caseResult of result.metadata?.cases ?? []) {
for (const meta of caseResult.siteMeta ?? []) {
const interval = intervals.get(siteKey(meta.caseName, meta.site))?.shift();
rows.push(normalizeSite(meta, interval));
}
}
return rows.length > 0 ? rows : fallbackSites(intervals);
}
export function buildSpiderReportData({ runDir, args, results }) {
return {
schema: 'moli.browser-spider.report.v3',
generated_at: new Date().toISOString(),
run_dir: runDir,
config: reportConfig(args),
services: results.map((result) => ({
target: result.target,
service: result.service,
success: result.success,
error: result.error ?? null,
evaluation: result.report ?? null,
leakage: result.leakage ?? null,
resources: result.resourceData ?? null,
sites: buildSiteDiagnostics(result),
artifacts: {
directory: path.relative(runDir, result.outputDir),
samples_json: path.relative(
runDir,
path.join(result.outputDir, 'resource-samples.json')
),
samples_csv: path.relative(
runDir,
path.join(result.outputDir, 'resource-samples.csv')
),
events_log: path.relative(runDir, path.join(result.outputDir, 'events.log')),
evaluation_json: path.relative(
runDir,
path.join(result.outputDir, 'service-evaluation.json')
)
}
}))
};
}
moli-benchmark/browser-spider-local/lib/observability/sampler-worker.mjs
+60
View File
@@ -0,0 +1,60 @@
import { parentPort, workerData } from 'node:worker_threads';
import { performance } from 'node:perf_hooks';
import { LinuxProcessTreeCollector } from './linux-procfs.mjs';
if (!parentPort) {
throw new Error('resource sampler worker requires a parent port');
}
const intervalMs = Math.max(100, Number(workerData?.intervalMs) || 500);
// Node worker threads share the process performance timeline. Using the
// runner-provided origin keeps samples and main-thread phase markers aligned.
const startedAt = Number.isFinite(workerData?.startedAt)
? workerData.startedAt
: performance.now();
const collector = new LinuxProcessTreeCollector({ intervalMs });
let stopped = false;
let lastCaptureAt = null;
function capture(force = false, kind = 'periodic') {
if (stopped) {
return;
}
const now = performance.now();
if (
lastCaptureAt !== null
&& now - lastCaptureAt < intervalMs * 0.8
&& !force
) {
return;
}
collector.sample({
elapsedMs: now - startedAt,
wallTime: new Date().toISOString(),
kind
});
lastCaptureAt = now;
}
const timer = setInterval(capture, intervalMs);
parentPort.on('message', (message) => {
if (message?.type === 'add-root') {
collector.addRoot(message.label, message.pid);
capture(true, 'root-registered');
return;
}
if (message?.type !== 'stop' || stopped) {
return;
}
capture(true, 'final');
stopped = true;
clearInterval(timer);
parentPort.postMessage({
type: 'result',
collector: collector.result()
});
parentPort.close();
});
moli-benchmark/browser-spider-local/lib/observability/sampler.mjs
+324
View File
@@ -0,0 +1,324 @@
import os from 'node:os';
import { performance } from 'node:perf_hooks';
import { Worker } from 'node:worker_threads';
const MINIMUM_INTERVAL_MS = 100;
const STOP_TIMEOUT_MS = 5000;
function finiteValues(samples, getter) {
return samples
.map(getter)
.filter((value) => Number.isFinite(value));
}
function maximum(values) {
return values.length > 0 ? Math.max(...values) : null;
}
function timeWeightedAverage(samples, getter) {
let weightedTotal = 0;
let durationTotal = 0;
for (let index = 1; index < samples.length; index += 1) {
const value = getter(samples[index]);
const duration = samples[index].elapsed_ms - samples[index - 1].elapsed_ms;
if (!Number.isFinite(value) || !Number.isFinite(duration) || duration <= 0) {
continue;
}
weightedTotal += value * duration;
durationTotal += duration;
}
return durationTotal > 0 ? weightedTotal / durationTotal : null;
}
export function summarizeSamples(samples, getter = (sample) => sample.total) {
const scoped = samples.filter((sample) => getter(sample));
const cpu = finiteValues(scoped, (sample) => getter(sample)?.cpu_percent);
const rss = finiteValues(scoped, (sample) => getter(sample)?.rss_bytes);
const pss = finiteValues(scoped, (sample) => getter(sample)?.pss_bytes);
const processCounts = finiteValues(scoped, (sample) => getter(sample)?.process_count);
const threadCounts = finiteValues(scoped, (sample) => getter(sample)?.thread_count);
return {
sample_count: scoped.length,
peak_cpu_percent: maximum(cpu),
average_cpu_percent: timeWeightedAverage(scoped, (sample) => getter(sample)?.cpu_percent),
peak_rss_bytes: maximum(rss),
peak_pss_bytes: maximum(pss),
peak_process_count: maximum(processCounts),
peak_thread_count: maximum(threadCounts)
};
}
function caseIntervals(markers) {
const pending = new Map();
const intervals = [];
for (const marker of markers) {
if (marker.type === 'case-start') {
pending.set(marker.case_name, marker);
} else if (marker.type === 'case-done') {
const start = pending.get(marker.case_name);
if (start) {
intervals.push({
case_name: marker.case_name,
start_ms: start.elapsed_ms,
end_ms: marker.elapsed_ms
});
pending.delete(marker.case_name);
}
}
}
return intervals;
}
export function buildResourceArtifact({
enabled,
status,
intervalMs,
markers,
collector = null,
error = null
}) {
const samples = collector?.samples ?? [];
const captureDurations = finiteValues(samples, (sample) => sample.capture_duration_ms);
// Root registration and shutdown deliberately force snapshots outside the
// periodic cadence. Only adjacent timer-driven samples describe scheduler
// health; including forced samples would make multi-worker runs appear to
// sample faster than configured.
const periodicSamples = samples.filter((sample) => sample.kind === 'periodic');
const observedIntervals = [];
for (let index = 1; index < periodicSamples.length; index += 1) {
const duration =
periodicSamples[index].elapsed_ms - periodicSamples[index - 1].elapsed_ms;
if (Number.isFinite(duration) && duration > 0) {
observedIntervals.push(duration);
}
}
const workerLabels = new Set();
for (const sample of samples) {
for (const label of Object.keys(sample.workers ?? {})) {
workerLabels.add(label);
}
}
const workers = Object.fromEntries(
[...workerLabels].sort().map((label) => [
label,
summarizeSamples(samples, (sample) => sample.workers?.[label])
])
);
const cases = caseIntervals(markers).map((interval) => ({
...interval,
...summarizeSamples(
samples.filter(
(sample) => sample.elapsed_ms >= interval.start_ms && sample.elapsed_ms <= interval.end_ms
)
)
}));
const summary = {
...summarizeSamples(samples),
duration_ms: samples.length > 0 ? samples.at(-1).elapsed_ms : 0,
average_capture_duration_ms: captureDurations.length > 0
? captureDurations.reduce((sum, value) => sum + value, 0) / captureDurations.length
: null,
max_capture_duration_ms: maximum(captureDurations),
sampling_overrun_count: captureDurations.filter((value) => value > intervalMs).length,
average_observed_interval_ms: observedIntervals.length > 0
? observedIntervals.reduce((sum, value) => sum + value, 0) / observedIntervals.length
: null,
max_observed_interval_ms: maximum(observedIntervals),
late_sample_count: observedIntervals.filter((value) => value > intervalMs * 1.5).length,
workers,
cases
};
return {
schema: 'moli.browser-spider.resources.v1',
enabled,
status,
error,
sampling: {
interval_ms: intervalMs,
platform: collector?.platform ?? process.platform,
method: collector?.method ?? null,
cpu_ticks_per_second: collector?.cpu_ticks_per_second ?? null,
host_logical_cpu_count: collector?.host_logical_cpu_count ?? os.cpus().length,
roots: collector?.roots ?? {},
errors: collector?.errors ?? {}
},
markers,
samples,
summary
};
}
function normalizedMarker(event, elapsedMs) {
return {
elapsed_ms: elapsedMs,
type: String(event?.type ?? 'event'),
case_name: event?.caseName ?? event?.case_name ?? null,
site: event?.site ?? null,
worker: event?.worker ?? null,
target: event?.target ?? null,
service: event?.service ?? null,
pid: Number.isInteger(event?.pid) ? event.pid : null,
success: typeof event?.success === 'boolean' ? event.success : null,
item_count: Number.isFinite(event?.itemCount) ? event.itemCount : null
};
}
export class ProcessTreeResourceSampler {
constructor({
enabled = true,
intervalMs = 500,
platform = process.platform,
now = () => performance.now(),
workerFactory = (options) => new Worker(
new URL('./sampler-worker.mjs', import.meta.url),
options
)
} = {}) {
this.enabled = enabled;
this.intervalMs = Math.max(
MINIMUM_INTERVAL_MS,
Number.isFinite(intervalMs) ? Math.trunc(intervalMs) : 500
);
this.platform = platform;
this.now = now;
this.workerFactory = workerFactory;
this.startedAt = null;
this.worker = null;
this.markers = [];
this.workerResult = null;
this.workerError = null;
this.resultResolver = null;
this.resultPromise = null;
this.stoppedArtifact = null;
}
start() {
if (this.startedAt !== null) {
throw new Error('resource sampler has already started');
}
this.startedAt = this.now();
if (!this.enabled || this.platform !== 'linux') {
return;
}
try {
this.worker = this.workerFactory({
workerData: {
intervalMs: this.intervalMs,
startedAt: this.startedAt
}
});
this.resultPromise = new Promise((resolve) => {
this.resultResolver = resolve;
});
this.worker.on('message', (message) => {
if (message?.type === 'result' && !this.workerResult) {
this.workerResult = message.collector;
this.resultResolver?.(message.collector);
}
});
this.worker.on('error', (error) => {
this.workerError = error;
this.resultResolver?.(null);
});
this.worker.on('exit', (code) => {
if (!this.workerResult && !this.workerError) {
this.workerError = new Error(
`resource sampler worker exited before returning data (code ${code})`
);
}
this.resultResolver?.(this.workerResult);
});
} catch (error) {
this.workerError = error;
this.worker = null;
}
}
elapsedMs() {
return this.startedAt === null ? 0 : Math.max(0, this.now() - this.startedAt);
}
mark(event) {
this.markers.push(normalizedMarker(event, this.elapsedMs()));
}
addRoot(label, pid) {
if (!Number.isInteger(pid) || pid <= 0) {
return;
}
this.worker?.postMessage({
type: 'add-root',
label,
pid
});
}
async stop() {
if (this.stoppedArtifact) {
return this.stoppedArtifact;
}
if (this.startedAt === null) {
throw new Error('resource sampler must be started before it is stopped');
}
if (!this.enabled) {
this.stoppedArtifact = buildResourceArtifact({
enabled: false,
status: 'disabled',
intervalMs: this.intervalMs,
markers: this.markers
});
return this.stoppedArtifact;
}
if (this.platform !== 'linux') {
this.stoppedArtifact = buildResourceArtifact({
enabled: true,
status: 'unsupported',
intervalMs: this.intervalMs,
markers: this.markers,
error: `resource sampling requires Linux procfs; current platform is ${this.platform}`
});
return this.stoppedArtifact;
}
if (!this.worker) {
this.stoppedArtifact = buildResourceArtifact({
enabled: true,
status: 'error',
intervalMs: this.intervalMs,
markers: this.markers,
error: this.workerError?.message ?? 'resource sampler worker failed to start'
});
return this.stoppedArtifact;
}
try {
this.worker.postMessage({ type: 'stop' });
} catch (error) {
this.workerError = error;
this.resultResolver?.(null);
}
let timeout;
const collector = await Promise.race([
this.resultPromise,
new Promise((resolve) => {
timeout = setTimeout(() => resolve(null), STOP_TIMEOUT_MS);
})
]);
clearTimeout(timeout);
if (!collector && !this.workerError) {
this.workerError = new Error('resource sampler worker did not stop within 5 seconds');
}
await this.worker.terminate().catch(() => undefined);
this.stoppedArtifact = buildResourceArtifact({
enabled: true,
status: collector ? 'available' : 'error',
intervalMs: this.intervalMs,
markers: this.markers,
collector,
error: collector ? null : this.workerError?.message ?? 'resource sampler failed'
});
return this.stoppedArtifact;
}
}
moli-benchmark/browser-spider-local/package-lock.json
Generated
+31
View File
@@ -0,0 +1,31 @@
{
"name": "browser-spider-local-bench",
"version": "0.1.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "browser-spider-local-bench",
"version": "0.1.0",
"license": "MIT OR Apache-2.0",
"dependencies": {
"playwright-core": "^1.52.0"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/playwright-core": {
"version": "1.60.0",
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.60.0.tgz",
"integrity": "sha512-9bW6zvX/m0lEbgTKJ6YppOKx8H3VOPBMOCFh2irXFOT4BbHgrx5hPjwJYLT40Lu+4qtD36qKc/Hn56StUW57IA==",
"license": "Apache-2.0",
"bin": {
"playwright-core": "cli.js"
},
"engines": {
"node": ">=18"
}
}
}
}
moli-benchmark/browser-spider-local/package.json
+18
View File
@@ -0,0 +1,18 @@
{
"name": "browser-spider-local-bench",
"version": "0.1.0",
"private": true,
"license": "MIT OR Apache-2.0",
"type": "module",
"scripts": {
"bench": "node bench.mjs",
"bench:assert": "node bench.mjs --assert-no-stale-page-leakage",
"test": "node --test tests/*.test.mjs"
},
"dependencies": {
"playwright-core": "^1.52.0"
},
"engines": {
"node": ">=18.0.0"
}
}
moli-benchmark/browser-spider-local/per-page-isolate-memory.mjs
+568
View File
@@ -0,0 +1,568 @@
import { execFileSync, spawn } from 'node:child_process';
import fs from 'node:fs';
import http from 'node:http';
import path from 'node:path';
import process from 'node:process';
import { fileURLToPath } from 'node:url';
import { chromium } from 'playwright-core';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = path.resolve(__dirname, '..', '..');
const CLOCK_TICKS_PER_SECOND = Number.parseInt(
execFileSync('getconf', ['CLK_TCK'], { encoding: 'utf8' }).trim(),
10
);
if (!Number.isFinite(CLOCK_TICKS_PER_SECOND) || CLOCK_TICKS_PER_SECOND <= 0) {
throw new Error(`invalid CLK_TCK: ${CLOCK_TICKS_PER_SECOND}`);
}
function parseArgs(argv) {
const args = {
binary: path.join(REPO_ROOT, 'target', 'release', 'moli'),
outputDir: null,
navigations: 120,
temporaryPageEvery: 12,
payloadObjects: 24000,
domNodes: 4000,
domSource: 'script',
promises: 1000,
externalScriptBytes: 0,
extraction: 'none',
navigationDelayMs: 0,
settleMs: 5000,
expectModel: 'page-vm'
};
for (let index = 0; index < argv.length; index += 1) {
const value = argv[index];
const next = () => {
index += 1;
if (index >= argv.length) {
throw new Error(`missing value after ${value}`);
}
return argv[index];
};
if (value === '--binary') {
args.binary = path.resolve(next());
} else if (value === '--output-dir') {
args.outputDir = path.resolve(next());
} else if (value === '--navigations') {
args.navigations = Number.parseInt(next(), 10);
} else if (value === '--temporary-page-every') {
args.temporaryPageEvery = Number.parseInt(next(), 10);
} else if (value === '--payload-objects') {
args.payloadObjects = Number.parseInt(next(), 10);
} else if (value === '--dom-nodes') {
args.domNodes = Number.parseInt(next(), 10);
} else if (value === '--dom-source') {
args.domSource = next();
} else if (value === '--promises') {
args.promises = Number.parseInt(next(), 10);
} else if (value === '--external-script-bytes') {
args.externalScriptBytes = Number.parseInt(next(), 10);
} else if (value === '--extraction') {
args.extraction = next();
} else if (value === '--navigation-delay-ms') {
args.navigationDelayMs = Number.parseInt(next(), 10);
} else if (value === '--settle-ms') {
args.settleMs = Number.parseInt(next(), 10);
} else if (value === '--expect-model') {
args.expectModel = next();
} else {
throw new Error(`unknown argument: ${value}`);
}
}
if (!args.outputDir) {
throw new Error('--output-dir is required');
}
if (!Number.isInteger(args.navigations) || args.navigations < 1) {
throw new Error('--navigations must be a positive integer');
}
if (!Number.isInteger(args.temporaryPageEvery) || args.temporaryPageEvery < 0) {
throw new Error('--temporary-page-every must be a non-negative integer');
}
for (const key of [
'payloadObjects',
'domNodes',
'promises',
'externalScriptBytes',
'navigationDelayMs',
'settleMs'
]) {
if (!Number.isInteger(args[key]) || args[key] < 0) {
throw new Error(`--${key.replace(/[A-Z]/g, (letter) => `-${letter.toLowerCase()}`)} must be a non-negative integer`);
}
}
if (!['script', 'markup'].includes(args.domSource)) {
throw new Error('--dom-source must be script or markup');
}
if (!['none', 'content', 'selectors', 'full'].includes(args.extraction)) {
throw new Error('--extraction must be none, content, selectors, or full');
}
return args;
}
function writeJson(filePath, value) {
fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, 'utf8');
}
function readProcessSample(pid) {
const rawStat = fs.readFileSync(`/proc/${pid}/stat`, 'utf8');
const close = rawStat.lastIndexOf(')');
const fields = rawStat.slice(close + 2).trim().split(/\s+/);
const ticks = Number(fields[11]) + Number(fields[12]);
const status = fs.readFileSync(`/proc/${pid}/status`, 'utf8');
const statusMiB = (key) => {
const match = new RegExp(`^${key}:\\s+(\\d+)\\s+kB$`, 'm').exec(status);
return match ? Number(match[1]) / 1024 : null;
};
return {
ticks,
rssMiB: statusMiB('VmRSS'),
rssAnonMiB: statusMiB('RssAnon'),
rssFileMiB: statusMiB('RssFile'),
vmDataMiB: statusMiB('VmData'),
vmSwapMiB: statusMiB('VmSwap')
};
}
function linearSlope(values) {
if (values.length < 2) {
return null;
}
const meanX = (values.length - 1) / 2;
const meanY = values.reduce((sum, value) => sum + value, 0) / values.length;
let numerator = 0;
let denominator = 0;
for (let index = 0; index < values.length; index += 1) {
numerator += (index - meanX) * (values[index] - meanY);
denominator += (index - meanX) ** 2;
}
return denominator === 0 ? null : numerator / denominator;
}
function average(values) {
return values.length === 0
? null
: values.reduce((sum, value) => sum + value, 0) / values.length;
}
function isolateCounters(diagnostics) {
const isolateScope = diagnostics.isolateScope;
const accounting = isolateScope?.documentIsolateAccounting;
const activeBrowserContext = diagnostics.activeBrowserContext;
const runtimeSession = activeBrowserContext?.runtimeSession;
if (!isolateScope) {
throw new Error(`missing isolate diagnostics: ${JSON.stringify(diagnostics)}`);
}
return {
model: isolateScope.documentIsolateModel,
created: accounting?.created ?? null,
destroyed: accounting?.destroyed ?? null,
live: accounting?.live ?? null,
reserved: accounting?.reserved ?? null,
loadedDocumentPageCount: isolateScope.loadedDocumentPageCount,
estimatedDocumentIsolateCount: isolateScope.estimatedDocumentIsolateCount,
documentContextCount: isolateScope.documentContextCount ?? null,
isolatedWorldContextCount: isolateScope.isolatedWorldContextCount ?? null,
childDefaultContextCount: isolateScope.childDefaultContextCount ?? null,
dedicatedWorkerLoadingCount: isolateScope.dedicatedWorkerLoadingCount ?? null,
dedicatedWorkerRunningWorkerIsolateCount:
isolateScope.dedicatedWorkerRunningWorkerIsolateCount ?? null,
estimatedWorkerIsolateCount: isolateScope.estimatedWorkerIsolateCount,
estimatedLiveV8IsolateCount: isolateScope.estimatedLiveV8IsolateCount ?? null,
pendingInspectorAwaitCount: isolateScope.pendingInspectorAwaitCount ?? null,
rendererInspectorSessionRetained:
runtimeSession?.rendererInspectorSessionRetained ?? null,
activeLoadedPage: activeBrowserContext?.activeLoadedPage ?? null,
domRemoteObjectNodeCacheCount:
activeBrowserContext?.domRemoteObjectNodeCacheCount ?? null
};
}
function pageHtml(index, payload) {
const markupRows = payload.domSource === 'markup'
? Array.from(
{ length: payload.domNodes },
(_, nodeIndex) => `<a class="row" data-bucket="${nodeIndex % 10}" href="/item/${index}/${nodeIndex}">page-${index}-node-${nodeIndex}</a>`
).join('')
: '';
const scriptedDomNodes = payload.domSource === 'script' ? payload.domNodes : 0;
const externalScript = payload.externalScriptBytes > 0
? `<script src="/script.js?index=${index}"></script>`
: '';
return `<!doctype html>
<meta charset="utf-8">
<title>isolate-sequence-${index}</title>
<style>
.row:nth-child(3n) { color: rgb(20, 40, 60); }
.row[data-bucket="5"] { contain: layout style; }
</style>
<main id="root">${markupRows}</main>
<script>
const marker = ${JSON.stringify(`page-${index}-`)};
globalThis.__retainedPayload = Array.from({ length: ${payload.payloadObjects} }, (_, itemIndex) => ({
itemIndex,
text: (marker + itemIndex).padEnd(112, "x"),
values: [itemIndex, itemIndex + 1, itemIndex + 2, itemIndex + 3]
}));
const fragment = document.createDocumentFragment();
for (let nodeIndex = 0; nodeIndex < ${scriptedDomNodes}; nodeIndex += 1) {
const row = document.createElement("a");
row.className = "row";
row.dataset.bucket = String(nodeIndex % 10);
row.href = "/item/" + ${index} + "/" + nodeIndex;
row.textContent = marker + "node-" + nodeIndex;
fragment.appendChild(row);
}
document.getElementById("root").appendChild(fragment);
globalThis.__resolvedPromises = Array.from({ length: ${payload.promises} }, (_, promiseIndex) =>
Promise.resolve(marker + promiseIndex)
);
</script>
${externalScript}`;
}
function externalScriptBody(index, requestedBytes) {
const prefix = `globalThis.__externalScriptMarker=${JSON.stringify(`external-${index}`)};/*`;
const suffix = '*/';
return `${prefix}${'x'.repeat(Math.max(0, requestedBytes - prefix.length - suffix.length))}${suffix}`;
}
async function extractSpiderSurface(page, mode) {
const result = {
htmlLength: null,
locatorCount: null,
sampledItemCount: 0
};
if (mode === 'content' || mode === 'full') {
result.htmlLength = (await page.content()).length;
}
if (mode === 'selectors' || mode === 'full') {
const elements = await page.locator('a.row').all();
result.locatorCount = elements.length;
for (const element of elements.slice(0, 5)) {
const text = await element.innerText();
const href = await element.getAttribute('href');
if (text && href) {
result.sampledItemCount += 1;
}
}
}
return result;
}
function startFixture(payload) {
const server = http.createServer((request, response) => {
const url = new URL(request.url || '/', 'http://127.0.0.1');
const index = Number.parseInt(url.searchParams.get('index') || '0', 10);
if (url.pathname === '/script.js') {
const body = externalScriptBody(
Number.isFinite(index) ? index : 0,
payload.externalScriptBytes
);
response.writeHead(200, {
'cache-control': 'public, max-age=3600',
'content-length': Buffer.byteLength(body),
'content-type': 'text/javascript; charset=utf-8'
});
response.end(body);
return;
}
const body = pageHtml(Number.isFinite(index) ? index : 0, payload);
response.writeHead(200, {
'cache-control': 'no-store',
'content-length': Buffer.byteLength(body),
'content-type': 'text/html; charset=utf-8'
});
response.end(body);
});
return new Promise((resolve, reject) => {
server.once('error', reject);
server.listen(0, '127.0.0.1', () => {
const address = server.address();
resolve({
baseUrl: `http://127.0.0.1:${address.port}`,
close: () => new Promise((closeResolve) => server.close(closeResolve))
});
});
});
}
function startMoli(binary, logs) {
const child = spawn(binary, ['serve', '--host', '127.0.0.1', '--port', '0'], {
cwd: REPO_ROOT,
detached: true,
env: { ...process.env, NO_PROXY: '*', no_proxy: '*' },
stdio: ['ignore', 'pipe', 'pipe']
});
const address = new Promise((resolve, reject) => {
const timeout = setTimeout(() => reject(new Error('timed out waiting for Moli address')), 15000);
const onChunk = (chunk) => {
const text = chunk.toString('utf8');
logs.push(text);
const match = /protocol server listening addr=(127\.0\.0\.1:\d+)/.exec(text);
if (match) {
clearTimeout(timeout);
resolve(match[1]);
}
};
child.stdout.on('data', onChunk);
child.stderr.on('data', onChunk);
child.once('error', (error) => {
clearTimeout(timeout);
reject(error);
});
child.once('exit', (code, signal) => {
clearTimeout(timeout);
reject(new Error(`Moli exited before startup: code=${code} signal=${signal}`));
});
});
return { child, address };
}
async function stopMoli(child) {
if (!child || child.exitCode !== null || child.signalCode !== null) {
return;
}
try {
process.kill(-child.pid, 'SIGTERM');
} catch (_error) {
child.kill('SIGTERM');
}
await Promise.race([
new Promise((resolve) => child.once('exit', resolve)),
new Promise((resolve) => setTimeout(resolve, 2000))
]);
if (child.exitCode === null && child.signalCode === null) {
try {
process.kill(-child.pid, 'SIGKILL');
} catch (_error) {
child.kill('SIGKILL');
}
}
}
async function browserWebSocketUrl(address) {
const response = await fetch(`http://${address}/json/version`);
if (!response.ok) {
throw new Error(`/json/version returned ${response.status}`);
}
const payload = await response.json();
return payload.webSocketDebuggerUrl;
}
async function main() {
const args = parseArgs(process.argv.slice(2));
if (fs.existsSync(args.outputDir)) {
throw new Error(`refusing to reuse output directory: ${args.outputDir}`);
}
fs.mkdirSync(args.outputDir, { recursive: true });
if (!fs.existsSync(args.binary)) {
throw new Error(`binary does not exist: ${args.binary}`);
}
const fixture = await startFixture(args);
const logs = [];
const moli = startMoli(args.binary, logs);
let browser;
const records = [];
const startedAt = process.hrtime.bigint();
const processStart = readProcessSample(moli.child.pid);
try {
const address = await moli.address;
browser = await chromium.connectOverCDP(await browserWebSocketUrl(address), { timeout: 15000 });
const browserSession = await browser.newBrowserCDPSession();
const context = browser.contexts()[0] || await browser.newContext();
const page = context.pages()[0] || await context.newPage();
const pageSession = await context.newCDPSession(page);
const baselineDiagnostics = await browserSession.send('HeapProfiler.moliDiagnostics');
const baselineCounters = isolateCounters(baselineDiagnostics);
for (let index = 1; index <= args.navigations; index += 1) {
await page.goto(`${fixture.baseUrl}/page?index=${index}`, {
waitUntil: 'domcontentloaded',
timeout: 30000
});
if (args.externalScriptBytes > 0) {
const marker = await page.evaluate(() => globalThis.__externalScriptMarker);
if (marker !== `external-${index}`) {
throw new Error(`external script did not execute for navigation ${index}: ${marker}`);
}
}
const extraction = await extractSpiderSurface(page, args.extraction);
const [heap, diagnostics] = await Promise.all([
pageSession.send('Runtime.getHeapUsage'),
browserSession.send('HeapProfiler.moliDiagnostics')
]);
records.push({
kind: 'navigation',
index,
...readProcessSample(moli.child.pid),
usedHeapMiB: heap.usedSize / 1024 / 1024,
totalHeapMiB: heap.totalSize / 1024 / 1024,
embedderHeapUsedMiB: (heap.embedderHeapUsedSize ?? 0) / 1024 / 1024,
backingStorageMiB: heap.backingStorageSize / 1024 / 1024,
networkMemoryCache:
diagnostics.connection?.activeNavigationEngine?.networkMemoryCache ?? null,
extraction,
counters: isolateCounters(diagnostics)
});
if (args.temporaryPageEvery > 0 && index % args.temporaryPageEvery === 0) {
const temporaryPage = await context.newPage();
await temporaryPage.goto(`${fixture.baseUrl}/temporary?index=${index}`, {
waitUntil: 'domcontentloaded',
timeout: 30000
});
const beforeClose = isolateCounters(
await browserSession.send('HeapProfiler.moliDiagnostics')
);
await temporaryPage.close();
const afterClose = isolateCounters(
await browserSession.send('HeapProfiler.moliDiagnostics')
);
records.push({
kind: 'temporary-page-close',
index,
...readProcessSample(moli.child.pid),
beforeClose,
afterClose
});
}
if (args.navigationDelayMs > 0) {
await new Promise((resolve) => setTimeout(resolve, args.navigationDelayMs));
}
}
const beforeFinalCloseDiagnostics = await browserSession.send(
'HeapProfiler.moliDiagnostics'
);
const beforeFinalClose = isolateCounters(beforeFinalCloseDiagnostics);
await page.close();
let afterFinalClose;
let afterFinalCloseDiagnostics;
for (let attempt = 0; attempt < 40; attempt += 1) {
afterFinalCloseDiagnostics = await browserSession.send(
'HeapProfiler.moliDiagnostics'
);
afterFinalClose = isolateCounters(afterFinalCloseDiagnostics);
if (
afterFinalClose.live === null ||
(afterFinalClose.live === 0 && afterFinalClose.reserved === 0)
) {
break;
}
await new Promise((resolve) => setTimeout(resolve, 25));
}
const postCloseSamples = [];
const settleStartedAt = Date.now();
do {
postCloseSamples.push({
elapsedMs: Date.now() - settleStartedAt,
...readProcessSample(moli.child.pid)
});
if (Date.now() - settleStartedAt >= args.settleMs) {
break;
}
await new Promise((resolve) => setTimeout(resolve, 100));
} while (true);
const navigationRecords = records.filter((record) => record.kind === 'navigation');
const rssValues = navigationRecords.map((record) => record.rssMiB);
const heapValues = navigationRecords.map((record) => record.usedHeapMiB);
const warmStart = Math.floor(navigationRecords.length / 2);
const warmRssValues = rssValues.slice(warmStart);
const warmHeapValues = heapValues.slice(warmStart);
const networkMemoryCacheRecords = navigationRecords
.map((record) => record.networkMemoryCache)
.filter((value) => value !== null);
const firstWindow = rssValues.slice(0, Math.min(10, rssValues.length));
const lastWindow = rssValues.slice(-Math.min(10, rssValues.length));
const processEnd = readProcessSample(moli.child.pid);
const elapsedSeconds = Number(process.hrtime.bigint() - startedAt) / 1e9;
const summary = {
binary: args.binary,
binarySha256: execFileSync('sha256sum', [args.binary], { encoding: 'utf8' })
.trim()
.split(/\s+/)[0],
navigations: args.navigations,
temporaryPageEvery: args.temporaryPageEvery,
payloadObjects: args.payloadObjects,
domNodes: args.domNodes,
domSource: args.domSource,
promises: args.promises,
externalScriptBytes: args.externalScriptBytes,
extraction: args.extraction,
navigationDelayMs: args.navigationDelayMs,
settleMs: args.settleMs,
elapsedSeconds,
childUserPlusSystemCpuSeconds:
(processEnd.ticks - processStart.ticks) / CLOCK_TICKS_PER_SECOND,
peakRssMiB: Math.max(...rssValues),
firstTenAverageRssMiB: average(firstWindow),
lastTenAverageRssMiB: average(lastWindow),
firstToLastWindowRssMiB: average(lastWindow) - average(firstWindow),
warmHalfRssSlopeMiBPerNavigation: linearSlope(warmRssValues),
peakCurrentIsolateUsedHeapMiB: Math.max(...heapValues),
warmHalfUsedHeapSlopeMiBPerNavigation: linearSlope(warmHeapValues),
peakNetworkMemoryCacheRetainedMiB: networkMemoryCacheRecords.length === 0
? null
: Math.max(...networkMemoryCacheRecords.map((value) => value.retainedBytes)) / 1024 / 1024,
finalNetworkMemoryCache: networkMemoryCacheRecords.at(-1) ?? null,
postCloseInitialRssMiB: postCloseSamples.at(0)?.rssMiB ?? null,
postCloseFinalRssMiB: postCloseSamples.at(-1)?.rssMiB ?? null,
postCloseMinimumRssMiB: Math.min(...postCloseSamples.map((sample) => sample.rssMiB)),
postCloseFinalAnonRssMiB: postCloseSamples.at(-1)?.rssAnonMiB ?? null,
postCloseFinalVmDataMiB: postCloseSamples.at(-1)?.vmDataMiB ?? null,
postCloseFinalVmSwapMiB: postCloseSamples.at(-1)?.vmSwapMiB ?? null,
baselineCounters,
beforeFinalClose,
afterFinalClose,
createdDelta: afterFinalClose.created === null
? null
: afterFinalClose.created - baselineCounters.created,
destroyedDelta: afterFinalClose.destroyed === null
? null
: afterFinalClose.destroyed - baselineCounters.destroyed,
baselineOutstandingIsolates: baselineCounters.created === null
? null
: baselineCounters.created - baselineCounters.destroyed,
finalOutstandingIsolates: afterFinalClose.created === null
? null
: afterFinalClose.created - afterFinalClose.destroyed
};
if (baselineCounters.model !== args.expectModel) {
throw new Error(`unexpected document isolate model: ${baselineCounters.model}`);
}
if (args.expectModel === 'page-vm') {
if (afterFinalClose.live !== 0 || afterFinalClose.reserved !== 0) {
throw new Error(`document isolate ownership did not close: ${JSON.stringify(afterFinalClose)}`);
}
if (summary.baselineOutstandingIsolates !== baselineCounters.live) {
throw new Error(`baseline isolate accounting mismatch: ${JSON.stringify(summary)}`);
}
if (summary.finalOutstandingIsolates !== afterFinalClose.live) {
throw new Error(`final isolate accounting mismatch: ${JSON.stringify(summary)}`);
}
}
writeJson(path.join(args.outputDir, 'records.json'), records);
writeJson(path.join(args.outputDir, 'post-close-samples.json'), postCloseSamples);
writeJson(path.join(args.outputDir, 'diagnostics.json'), {
baseline: baselineDiagnostics,
beforeFinalClose: beforeFinalCloseDiagnostics,
afterFinalClose: afterFinalCloseDiagnostics
});
writeJson(path.join(args.outputDir, 'summary.json'), summary);
fs.writeFileSync(path.join(args.outputDir, 'moli.log'), logs.join(''), 'utf8');
console.log(JSON.stringify(summary, null, 2));
} finally {
await browser?.close().catch(() => undefined);
await stopMoli(moli.child);
await fixture.close();
}
}
main().catch((error) => {
console.error(error.stack || error.message || String(error));
process.exit(1);
});
moli-benchmark/browser-spider-local/tests/bench.test.mjs
+91
View File
@@ -0,0 +1,91 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { extractItems } from '../bench.mjs';
function candidate({ text, href = null, closestHref = null, childHref = null }) {
const anchor = (value) => ({
getAttribute(name) {
return name === 'href' ? value : null;
}
});
return {
innerText: text,
getAttribute(name) {
return name === 'href' ? href : null;
},
closest(selector) {
assert.equal(selector, 'a');
return closestHref === null ? null : anchor(closestHref);
},
querySelector(selector) {
assert.equal(selector, 'a');
return childHref === null ? null : anchor(childHref);
}
};
}
test('item extraction snapshots the complete selector set in one page-realm job', async () => {
const generations = {
old: new Map([
['news', [
candidate({ text: ' First story ', href: '/first' }),
candidate({ text: 'First story', href: '/duplicate' }),
candidate({ text: 'Nested story', closestHref: '/nested' })
]],
['sports', [
candidate({ text: 'Sports story', childHref: '/sports' })
]]
]),
replacement: new Map([
['news', []],
['sports', []]
])
};
let generation = 'old';
let replacementScheduled = false;
const selectorGenerations = [];
const snapshots = [];
const page = {
locator() {
assert.fail('extractItems must not create Locators');
},
async evaluate(project, selectors) {
snapshots.push([...selectors]);
const previousDocument = globalThis.document;
globalThis.document = {
baseURI: 'https://example.test/root/',
querySelectorAll(selector) {
selectorGenerations.push(`${generation}:${selector}`);
if (!replacementScheduled) {
replacementScheduled = true;
queueMicrotask(() => {
generation = 'replacement';
});
}
return generations[generation].get(selector) ?? [];
}
};
try {
return project(selectors);
} finally {
if (previousDocument === undefined) {
delete globalThis.document;
} else {
globalThis.document = previousDocument;
}
}
}
};
const items = await extractItems(page, ['news', 'sports']);
assert.deepEqual(snapshots, [['news', 'sports']]);
assert.deepEqual(selectorGenerations, ['old:news', 'old:sports']);
assert.equal(generation, 'replacement');
assert.deepEqual(items, [
{ title: 'First story', link: 'https://example.test/first' },
{ title: 'Nested story', link: 'https://example.test/nested' },
{ title: 'Sports story', link: 'https://example.test/sports' }
]);
});
moli-benchmark/browser-spider-local/tests/compare.test.mjs
+570
View File
@@ -0,0 +1,570 @@
import assert from 'node:assert/strict';
import { spawnSync } from 'node:child_process';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { fileURLToPath } from 'node:url';
import {
buildSpiderComparison,
readSpiderBenchmark,
renderSpiderComparison,
renderSpiderInfrastructureFailure,
SPIDER_CI_COMMENT_MARKER,
SPIDER_COMPARISON_SCHEMA
} from '../compare.mjs';
import {
endpointFromMoliLogs,
expectedRowsForCase,
makeCaseSet,
validateServiceOutput
} from '../bench.mjs';
import { renderResourceTimelineSection } from '../lib/comparison/resource-timeline.mjs';
const MIB = 1024 * 1024;
function resourceSample(elapsedMs, cpuPercent, rssMib, pssMib = rssMib - 5) {
return {
elapsed_ms: elapsedMs,
kind: 'periodic',
total: {
cpu_percent: cpuPercent,
rss_bytes: rssMib * MIB,
pss_bytes: pssMib === null ? null : pssMib * MIB
}
};
}
function defaultResourceSamples() {
return [
resourceSample(0, null, 100, 95),
resourceSample(500, 50, 160, 150),
resourceSample(1000, 125, 220, 210),
resourceSample(1500, 25, 180, 170)
];
}
function defaultComparisonTimeline() {
return defaultResourceSamples().map((sample) => ({
elapsed_seconds: sample.elapsed_ms / 1000,
cpu_percent: sample.total.cpu_percent,
rss_mib: sample.total.rss_bytes / MIB,
pss_mib: sample.total.pss_bytes / MIB
}));
}
function writeRun(root, name, {
actualRows = 174,
expectedRows = 240,
success = true,
mismatched = 0,
timeoutWithItems = 0,
durationMs = 80_000,
averageCpuPercent = 75,
peakRssBytes = 450 * 1024 * 1024,
peakPssBytes = 445 * 1024 * 1024,
peakThreads = 26,
resourceSamples = defaultResourceSamples()
} = {}) {
const runDir = path.join(root, name);
fs.mkdirSync(runDir, { recursive: true });
const evaluation = {
totalActualRows: actualRows,
totalExpectedRows: expectedRows,
averageFillRate: (actualRows / expectedRows) * 100
};
const resources = {
duration_ms: durationMs,
average_cpu_percent: averageCpuPercent,
peak_rss_bytes: peakRssBytes,
peak_pss_bytes: peakPssBytes,
peak_process_count: 1,
peak_thread_count: peakThreads
};
const leakage = {
suspiciousCount: mismatched + timeoutWithItems,
mismatchedItemSites: mismatched,
timeoutWithItems,
httpErrorWithItems: 0
};
fs.writeFileSync(path.join(runDir, 'summary.json'), JSON.stringify({
results: [{ success, evaluation, resources, leakage }]
}));
fs.writeFileSync(path.join(runDir, 'report-data.json'), JSON.stringify({
services: [{
resources: { samples: resourceSamples },
sites: [
{ outcome: 'extracted', item_count: 5, expected_item_count: 5 },
{
outcome: timeoutWithItems > 0 ? 'timeout_with_items' : 'empty',
item_count: timeoutWithItems > 0 ? 1 : 0,
expected_item_count: 0
}
]
}]
}));
}
function makeRun(metrics, exitCode = 0) {
const {
resource_timeline: resourceTimeline = defaultComparisonTimeline(),
...metricOverrides
} = metrics;
return {
availability: 'available',
exit_code: exitCode,
error: null,
metrics: {
service_runs: 1,
successful_runs: exitCode === 0 ? 1 : 0,
failed_runs: exitCode === 0 ? 0 : 1,
actual_rows: 174,
expected_rows: 240,
fill_rate_percent: 72.5,
suspicious_sites: 0,
mismatched_item_sites: 0,
timeout_with_items: 0,
http_error_with_items: 0,
total_sites: 48,
sites_with_rows: 36,
sites_without_rows: 12,
expected_row_sites: 48,
expected_empty_sites: 0,
sites_meeting_row_contract: 34,
unexpected_empty_sites: 12,
partial_sites: 2,
site_outcomes: {
extracted: 34,
http_error_with_items: 2,
empty: 10,
http_error_empty: 1,
navigation_error_empty: 1
},
case_rows: {
news: { actual_rows: 35, expected_rows: 40 },
stocks: { actual_rows: 20, expected_rows: 40 },
tech: { actual_rows: 35, expected_rows: 40 },
sports: { actual_rows: 27, expected_rows: 40 },
games: { actual_rows: 37, expected_rows: 40 },
life: { actual_rows: 20, expected_rows: 40 }
},
duration_ms: 80_000,
estimated_cpu_seconds: 60,
peak_rss_bytes: 450 * 1024 * 1024,
peak_pss_bytes: 445 * 1024 * 1024,
peak_process_count: 1,
peak_thread_count: 26,
...metricOverrides
},
resource_timeline: resourceTimeline
};
}
function makeFixtureRun(metrics = {}, exitCode = 0) {
return makeRun({
actual_rows: 40,
expected_rows: 40,
fill_rate_percent: 100,
total_sites: 15,
sites_with_rows: 8,
sites_without_rows: 7,
expected_row_sites: 8,
expected_empty_sites: 7,
sites_meeting_row_contract: 15,
unexpected_empty_sites: 0,
partial_sites: 0,
site_outcomes: {
extracted: 7,
http_error_with_items: 1,
empty: 3,
timeout_empty: 4
},
case_rows: {},
...metrics
}, exitCode);
}
function buildComparison(fixtureBase, fixtureHead, realBase, realHead) {
return buildSpiderComparison({
baseSha: '1'.repeat(40),
headSha: '2'.repeat(40),
executionOrder: 'base-first',
runs: {
fixture: { base: fixtureBase, head: fixtureHead },
real48: { base: realBase, head: realHead }
}
});
}
test('comparison aggregates resources and keeps the public run informational', () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'spider-comparison-'));
try {
const fixtureBase = path.join(tempDir, 'fixture-base');
const fixtureHead = path.join(tempDir, 'fixture-head');
const realBase = path.join(tempDir, 'real-base');
const realHead = path.join(tempDir, 'real-head');
writeRun(fixtureBase, 'run', { actualRows: 174, expectedRows: 174, durationMs: 10_000 });
writeRun(fixtureHead, 'run', { actualRows: 174, expectedRows: 174, durationMs: 9_000 });
writeRun(realBase, 'run', { actualRows: 170, peakPssBytes: 440 * 1024 * 1024 });
writeRun(realHead, 'run', {
actualRows: 165,
mismatched: 2,
peakPssBytes: 450 * 1024 * 1024
});
const comparison = buildComparison(
readSpiderBenchmark(fixtureBase, 0),
readSpiderBenchmark(fixtureHead, 0),
readSpiderBenchmark(realBase, 0),
readSpiderBenchmark(realHead, 0)
);
assert.equal(comparison.schema, SPIDER_COMPARISON_SCHEMA);
assert.equal(comparison.suites.fixture.assessment.clean, true);
assert.equal(comparison.suites.fixture.delta.duration_ms.absolute, -1000);
assert.equal(comparison.suites.fixture.head.metrics.estimated_cpu_seconds, 6.75);
assert.equal(comparison.suites.real48.delta.peak_pss_bytes.absolute, 10 * 1024 * 1024);
assert.equal(comparison.suites.real48.assessment, null);
assert.equal(comparison.suites.real48.head.metrics.mismatched_item_sites, 2);
assert.deepEqual(comparison.suites.real48.head.resource_timeline, [
{ elapsed_seconds: 0, cpu_percent: null, rss_mib: 100, pss_mib: 95 },
{ elapsed_seconds: 0.5, cpu_percent: 50, rss_mib: 160, pss_mib: 150 },
{ elapsed_seconds: 1, cpu_percent: 125, rss_mib: 220, pss_mib: 210 },
{ elapsed_seconds: 1.5, cpu_percent: 25, rss_mib: 180, pss_mib: 170 }
]);
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
test('deterministic fixture assessment records stale output and row regressions', () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'spider-assessment-'));
try {
const fixtureBase = path.join(tempDir, 'fixture-base');
const fixtureHead = path.join(tempDir, 'fixture-head');
const realBase = path.join(tempDir, 'real-base');
const realHead = path.join(tempDir, 'real-head');
writeRun(fixtureBase, 'run', { actualRows: 174, expectedRows: 174 });
writeRun(fixtureHead, 'run', { actualRows: 170, expectedRows: 174, timeoutWithItems: 1 });
writeRun(realBase, 'run');
writeRun(realHead, 'run');
const comparison = buildComparison(
readSpiderBenchmark(fixtureBase, 0),
readSpiderBenchmark(fixtureHead, 1),
readSpiderBenchmark(realBase, 0),
readSpiderBenchmark(realHead, 0)
);
assert.equal(comparison.suites.fixture.assessment.clean, false);
assert.deepEqual(comparison.suites.fixture.assessment.reasons, [
'head-benchmark-failed',
'head-timeout-returned-stale-items',
'head-fixture-row-contract-mismatch',
'fixture-row-count-regressed',
'fixture-fill-rate-regressed'
]);
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
test('fixture assessment rejects per-site contract cancellation at the same row total', () => {
const comparison = buildComparison(
makeFixtureRun({}),
makeFixtureRun({ sites_meeting_row_contract: 13 }),
makeRun({}),
makeRun({})
);
assert.deepEqual(comparison.suites.fixture.assessment.reasons, [
'head-fixture-row-contract-mismatch'
]);
});
test('missing output becomes unavailable instead of throwing', () => {
const missing = readSpiderBenchmark('/definitely/missing/spider-output', 125);
assert.equal(missing.availability, 'unavailable');
assert.equal(missing.exit_code, 125);
assert.match(missing.error, /found 0/);
});
test('comment labels every result as non-blocking evidence', () => {
const comparison = buildComparison(
makeFixtureRun({}),
makeFixtureRun({ duration_ms: 82_000, peak_pss_bytes: 448 * 1024 * 1024 }),
makeRun({}),
makeRun({})
);
const comment = renderSpiderComparison(comparison, {
runUrl: 'https://github.com/lexmount/moli-dark/actions/runs/123',
conclusion: 'success'
});
assert.ok(comment.startsWith(SPIDER_CI_COMMENT_MARKER));
assert.match(comment, /Deterministic fixture diagnostic is clean/);
assert.match(comment, /Deterministic fixture · diagnostic/);
assert.match(comment, /Public 48-site run · informational/);
assert.ok(
comment.indexOf('Public 48-site run · informational')
< comment.indexOf('Deterministic fixture · diagnostic')
);
assert.match(comment, /### Public 48-site run · informational/);
assert.match(comment, /### Deterministic fixture · diagnostic/);
assert.doesNotMatch(comment, /<details/);
assert.match(comment, /Public HEAD:\*\* 174 \/ 240 rows; 36 \/ 48 sites produced rows/);
assert.match(comment, /fixture exercises 15 routes: 8 are expected to emit rows and 7 intentionally/);
assert.match(comment, /Sites meeting row contract \| 15 \/ 15 \| 15 \/ 15/);
assert.match(comment, /Unexpected empty sites \| 12 \| 12/);
assert.match(comment, /HTTP error \+ rows: 2/);
assert.match(comment, /\| sports \| 27 \/ 40 \| 27 \/ 40 \| 0 \|/);
assert.match(comment, /\+3\.00 MiB \(\+0\.67%\)/);
assert.match(comment, /actions\/runs\/123/);
assert.match(comment, /never fail the PR/);
assert.match(comment, /Common ancestor `111111111111` → HEAD `222222222222`/);
assert.match(comment, /### CPU and memory timelines/);
assert.match(comment, /bounded 40-point views of the same complete process-tree samples/);
assert.match(comment, /xychart-beta/);
assert.match(comment, /title "Base CPU"/);
assert.match(comment, /title "HEAD memory: RSS then PSS"/);
assert.match(comment, /x-axis "Elapsed seconds" \[0, 0\.5, 1, 1\.5\]/);
assert.match(comment, /line \[100, 160, 220, 180\]/);
assert.ok(comment.indexOf('#### Base') < comment.indexOf('#### HEAD'));
});
test('resource timeline is bounded while retaining endpoints and metric extrema', () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'spider-timeline-'));
try {
const samples = Array.from({ length: 101 }, (_, index) => resourceSample(
index * 500,
index === 57 ? 900 : 20 + (index % 7),
index === 68 ? 900 : 100 + index,
index === 73 ? 850 : 90 + index
));
writeRun(tempDir, 'run', { resourceSamples: samples });
const run = readSpiderBenchmark(tempDir, 0);
assert.equal(run.resource_timeline.length, 40);
assert.equal(run.resource_timeline[0].elapsed_seconds, 0);
assert.equal(run.resource_timeline.at(-1).elapsed_seconds, 50);
assert.ok(run.resource_timeline.some((point) => point.cpu_percent === 900));
assert.ok(run.resource_timeline.some((point) => point.rss_mib === 900));
assert.ok(run.resource_timeline.some((point) => point.pss_mib === 850));
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
test('trusted comment renderer omits malformed or oversized resource timelines', () => {
const invalidTimeline = Array.from({ length: 41 }, (_, index) => ({
elapsed_seconds: index,
cpu_percent: index,
rss_mib: 100,
pss_mib: 90
}));
const comparison = buildComparison(
makeFixtureRun({}),
makeFixtureRun({}),
makeRun({ resource_timeline: invalidTimeline }),
makeRun({ resource_timeline: [{
elapsed_seconds: 0,
cpu_percent: '```@everyone',
rss_mib: 100,
pss_mib: 90
}] })
);
const comment = renderSpiderComparison(comparison, {
runUrl: 'https://github.com/lexmount/moli-dark/actions/runs/123',
conclusion: 'success'
});
assert.doesNotMatch(comment, /CPU and memory timelines/);
assert.doesNotMatch(comment, /@everyone/);
});
test('memory chart keeps the complete RSS series when PSS has a sampling gap', () => {
const run = makeRun({
resource_timeline: [
{ elapsed_seconds: 0, cpu_percent: 10, rss_mib: 100, pss_mib: 90 },
{ elapsed_seconds: 1, cpu_percent: 20, rss_mib: 150, pss_mib: null },
{ elapsed_seconds: 2, cpu_percent: 30, rss_mib: 140, pss_mib: 130 }
]
});
const section = renderResourceTimelineSection({ base: run, head: run });
assert.match(section, /title "Base memory: RSS"/);
assert.match(section, /x-axis "Elapsed seconds" \[0, 1, 2\]/);
assert.match(section, /line \[100, 150, 140\]/);
assert.doesNotMatch(section, /memory: RSS then PSS/);
});
test('infrastructure failures render a bounded comment without artifact data', () => {
const comment = renderSpiderInfrastructureFailure({
runUrl: 'https://github.com/lexmount/moli-dark/actions/runs/123',
conclusion: 'failure'
});
assert.ok(comment.startsWith(SPIDER_CI_COMMENT_MARKER));
assert.match(comment, /infrastructure failed before a comparison/);
assert.match(comment, /Workflow: `failure`/);
assert.match(comment, /actions\/runs\/123/);
assert.match(comment, /No benchmark result or performance conclusion/);
});
test('infrastructure-comment command writes the trusted fallback comment', () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'spider-infrastructure-comment-'));
try {
const output = path.join(tempDir, 'comment.md');
const result = spawnSync(process.execPath, [
fileURLToPath(new URL('../compare.mjs', import.meta.url)),
'infrastructure-comment',
'--run-url', 'https://github.com/lexmount/moli-dark/actions/runs/123',
'--conclusion', 'failure',
'--output', output
], { encoding: 'utf8' });
assert.equal(result.status, 0, result.stderr);
assert.match(fs.readFileSync(output, 'utf8'), /infrastructure failed before a comparison/);
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
test('CI resolves a common-ancestor baseline and restores the target-base harness', () => {
const benchmarkWorkflow = fs.readFileSync(
fileURLToPath(new URL('../../../.github/workflows/spider-bench.yml', import.meta.url)),
'utf8'
);
const commentWorkflow = fs.readFileSync(
fileURLToPath(new URL('../../../.github/workflows/spider-bench-comment.yml', import.meta.url)),
'utf8'
);
assert.match(
benchmarkWorkflow,
/base_sha=\$\(git merge-base "\$target_base_sha" "\$head_sha"\)/
);
assert.match(benchmarkWorkflow, /restore_harness\(\)[\s\S]*TARGET_BASE_SHA/);
assert.match(commentWorkflow, /compareCommitsWithBasehead/);
assert.match(commentWorkflow, /comparison\.base\?\.sha !== expectedCommonAncestor/);
});
test('Moli endpoint discovery accepts ANSI-formatted tracing output', () => {
const log = '\u001b[2m2026-08-03T09:21:36Z\u001b[0m \u001b[32m INFO\u001b[0m '
+ 'protocol server listening \u001b[3maddr\u001b[0m\u001b[2m=\u001b[0m127.0.0.1:45445';
assert.equal(endpointFromMoliLogs([`stderr: ${log}`]), 'http://127.0.0.1:45445');
});
test('run command records failed browser runs without failing the observer', () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'spider-observer-'));
try {
const result = spawnSync(process.execPath, [
fileURLToPath(new URL('../compare.mjs', import.meta.url)),
'run',
'--base-bin', process.execPath,
'--head-bin', process.execPath,
'--base-sha', '1'.repeat(40),
'--head-sha', '2'.repeat(40),
'--execution-order', 'base-first',
'--output-dir', tempDir
], { encoding: 'utf8' });
assert.equal(result.status, 0, result.stderr);
const comparison = JSON.parse(fs.readFileSync(path.join(tempDir, 'comparison.json'), 'utf8'));
assert.equal(comparison.suites.fixture.assessment.clean, false);
assert.ok(comparison.suites.fixture.assessment.reasons.includes('head-benchmark-failed'));
assert.match(result.stderr, /recorded diagnostic issues/);
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
test('comment rejects artifact-controlled URLs and SHA markup', () => {
const comparison = buildComparison(
makeFixtureRun({}),
makeFixtureRun({}),
makeRun({}),
makeRun({})
);
comparison.head.sha = '`@everyone`';
const comment = renderSpiderComparison(comparison, {
runUrl: 'javascript:alert(1)',
conclusion: '<script>'
});
assert.doesNotMatch(comment, /@everyone/);
assert.doesNotMatch(comment, /javascript:/);
assert.doesNotMatch(comment, /<script>/);
assert.match(comment, /HEAD `unknown`/);
assert.match(comment, /workflow: `unknown`/);
});
test('workload row contracts distinguish fixture outcomes from public capacity', () => {
const fixtureCases = makeCaseSet({ siteSource: 'fixture', siteLimit: 0 }, 'http://fixture.test');
const realCases = makeCaseSet({ siteSource: 'real', siteLimit: 0 }, null);
const limitedRealCases = makeCaseSet({ siteSource: 'real', siteLimit: 1 }, null);
const totals = (cases) => Object.values(cases)
.reduce((total, caseConfig) => total + expectedRowsForCase(caseConfig), 0);
assert.equal(totals(fixtureCases), 40);
assert.deepEqual(
Object.fromEntries(Object.entries(fixtureCases).map(([name, config]) => [
name,
expectedRowsForCase(config)
])),
{ news: 10, stocks: 5, tech: 5, sports: 5, games: 10, life: 5 }
);
assert.equal(totals(realCases), 240);
assert.equal(totals(limitedRealCases), 30);
assert.deepEqual(
realCases.life.sites.map((site) => site.name),
['美食天下', '什么值得买', '果壳', '马蜂窝', '穷游', '太平洋家居', '汽车之家', '下厨房']
);
assert.ok(fixtureCases.life.sites.some((site) => site.name === '本地家居预响应挂起'));
assert.ok(realCases.sports.rules.PP体育.selectors.includes("a[href*='ppsport']"));
assert.throws(
() => expectedRowsForCase({ sites: [{ name: 'invalid', expectedItemCount: 6 }] }),
/invalid expected item count/
);
});
test('fixture service evaluation reports its explicit 40-row contract', () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'spider-fixture-contract-'));
const csvNames = {
news: 'news_top_headlines.csv',
stocks: 'stocks_top_list.csv',
tech: 'tech_top_list.csv',
sports: 'sports_top_list.csv',
games: 'games_top_list.csv',
life: 'life_top_list.csv'
};
try {
const cases = makeCaseSet({ siteSource: 'fixture', siteLimit: 0 }, 'http://fixture.test');
for (const [caseName, caseConfig] of Object.entries(cases)) {
const caseDir = path.join(tempDir, caseName);
fs.mkdirSync(caseDir, { recursive: true });
const rows = ['site,title,link'];
for (const site of caseConfig.sites) {
for (let index = 0; index < site.expectedItemCount; index += 1) {
rows.push(`${site.name},item-${index},${site.url}/${index}`);
}
}
fs.writeFileSync(path.join(caseDir, csvNames[caseName]), `${rows.join('\n')}\n`, 'utf8');
}
const report = validateServiceOutput(
tempDir,
'moli',
Object.keys(cases),
cases
);
assert.equal(report.summary.totalActualRows, 40);
assert.equal(report.summary.totalExpectedRows, 40);
assert.equal(report.summary.averageFillRate, 100);
assert.equal(report.summary.totalSites, 15);
assert.equal(report.summary.sitesWithRows, 8);
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
moli-benchmark/browser-spider-local/tests/observability.test.mjs
+79
View File
@@ -0,0 +1,79 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { SpiderRunObserver } from '../lib/observability/index.mjs';
function args() {
return {
targets: ['moli'],
workers: 1,
parallelism: 1,
runs: 1,
cases: ['news'],
siteLimit: 1,
siteSource: 'fixture',
gotoTimeoutMs: 1500,
sampleResources: false,
sampleIntervalMs: 500
};
}
test('public observer owns service artifacts and the final dashboard lifecycle', async () => {
const runDir = fs.mkdtempSync(path.join(os.tmpdir(), 'spider-observer-test-'));
try {
const outputDir = path.join(runDir, 'output-moli');
fs.mkdirSync(outputDir);
const runObserver = new SpiderRunObserver({ runDir, args: args() });
const serviceObserver = runObserver.beginService({
outputDir,
service: 'moli',
target: 'moli'
});
serviceObserver.registerWorker('worker-1', process.pid);
serviceObserver.mark({ type: 'case-start', caseName: 'news' });
serviceObserver.mark({ type: 'case-done', caseName: 'news' });
const resourceData = await serviceObserver.finish();
const report = runObserver.writeReport([{
target: 'moli',
service: 'moli',
success: true,
outputDir,
report: { summary: { averageFillRate: 100 }, files: [] },
leakage: { suspiciousCount: 0 },
resourceData
}]);
assert.equal(resourceData.status, 'disabled');
assert.ok(fs.existsSync(path.join(outputDir, 'resource-samples.json')));
assert.ok(fs.existsSync(path.join(outputDir, 'resource-samples.csv')));
assert.ok(fs.existsSync(report.htmlPath));
assert.ok(fs.existsSync(report.dataPath));
} finally {
fs.rmSync(runDir, { recursive: true, force: true });
}
});
test('run observer refuses duplicate services and reports from unfinished services', async () => {
const runDir = fs.mkdtempSync(path.join(os.tmpdir(), 'spider-observer-state-test-'));
try {
const outputDir = path.join(runDir, 'output-moli');
fs.mkdirSync(outputDir);
const runObserver = new SpiderRunObserver({ runDir, args: args() });
const serviceObserver = runObserver.beginService({
outputDir,
service: 'moli',
target: 'moli'
});
assert.throws(
() => runObserver.beginService({ outputDir, service: 'moli', target: 'moli' }),
/already registered/
);
assert.throws(() => runObserver.writeReport([]), /before observability finishes/);
await serviceObserver.finish();
} finally {
fs.rmSync(runDir, { recursive: true, force: true });
}
});
moli-benchmark/browser-spider-local/tests/report.test.mjs
+266
View File
@@ -0,0 +1,266 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import vm from 'node:vm';
import {
spiderReportDocument,
writeSpiderReport
} from '../lib/observability/dashboard.mjs';
import {
buildSpiderReportData,
buildSiteDiagnostics,
resourceSamplesCsv,
writeResourceArtifacts
} from '../lib/observability/report-data.mjs';
function fixtureResourceData() {
return {
schema: 'moli.browser-spider.resources.v1',
enabled: true,
status: 'available',
error: null,
sampling: {
interval_ms: 100,
method: 'test',
host_logical_cpu_count: 8,
errors: {}
},
markers: [
{ type: 'case-start', case_name: 'news', elapsed_ms: 0 },
{
type: 'site-start',
case_name: 'news',
site: 'fixture news',
worker: 'worker-1',
elapsed_ms: 10
},
{
type: 'site-done',
case_name: 'news',
site: 'fixture news',
worker: 'worker-1',
elapsed_ms: 90,
success: true,
item_count: 5
},
{ type: 'case-done', case_name: 'news', elapsed_ms: 100 }
],
samples: [
{
elapsed_ms: 100,
wall_time: '2026-07-26T00:00:00.000Z',
kind: 'periodic',
capture_duration_ms: 1.25,
total: {
cpu_percent: 88,
rss_bytes: 2048,
rss_process_count: 1,
pss_bytes: 1024,
pss_process_count: 1,
process_count: 1,
thread_count: 4
},
workers: {}
}
],
summary: {
sample_count: 1,
peak_cpu_percent: 88,
average_cpu_percent: 88,
peak_rss_bytes: 2048,
peak_pss_bytes: 1024,
peak_process_count: 1,
peak_thread_count: 4,
average_capture_duration_ms: 1.25,
max_capture_duration_ms: 1.25,
sampling_overrun_count: 0,
workers: {},
cases: []
}
};
}
function fixtureArgs() {
return {
targets: ['moli'],
workers: 1,
parallelism: 1,
runs: 1,
cases: ['news'],
siteLimit: 1,
siteSource: 'fixture',
gotoTimeoutMs: 1500,
sampleResources: true,
sampleIntervalMs: 100
};
}
function fixtureMetadata() {
return {
cases: [{
caseName: 'news',
siteMeta: [{
caseName: 'news',
site: 'fixture news',
url: 'https://example.test/news',
gotoOk: false,
gotoError: 'page.goto: Timeout 1500ms exceeded. | at runSite',
responseStatus: null,
finalUrlAfterExtract: 'https://example.test/news',
title: 'Fixture news',
htmlLength: 0,
htmlSha256: null,
htmlSaveError: 'page.content: document changed during snapshot. | at runSite',
expectedItemCount: 5,
itemCount: 5,
itemClassification: 'expected',
items: [{
title: 'Fixture item',
link: 'https://example.test/news/item'
}]
}]
}]
};
}
test('resource CSV preserves explicit process-tree fields', () => {
const csv = resourceSamplesCsv(fixtureResourceData());
assert.match(csv, /^elapsed_ms,wall_time,kind,cpu_percent,rss_bytes,rss_process_count,pss_bytes,/);
assert.match(csv, /100,2026-07-26T00:00:00.000Z,periodic,88,2048,1,1024,1,1,4,1.25/);
});
test('report payload and document retain charts, raw links and safely embedded data', () => {
const runDir = '/tmp/spider-report';
const outputDir = `${runDir}/output-moli`;
const payload = buildSpiderReportData({
runDir,
args: fixtureArgs(),
results: [{
target: 'moli',
service: '</script><script>alert(1)</script>',
success: true,
outputDir,
report: {
summary: {
totalActualRows: 5,
totalExpectedRows: 5,
averageFillRate: 100
},
files: []
},
leakage: { suspiciousCount: 0, mismatchedItemSites: 0, timeoutWithItems: 0 },
metadata: fixtureMetadata(),
resourceData: fixtureResourceData()
}]
});
const document = spiderReportDocument(payload);
assert.equal(payload.config.sample_interval_ms, 100);
assert.match(document, /Peak resource comparison/);
assert.match(document, /Memory timeline · process tree/);
assert.match(document, /Case duration and extraction quality/);
assert.match(document, /Slowest site phases · top 20/);
assert.match(document, /Site result distribution/);
assert.match(document, /HTML snapshot warnings/);
assert.match(document, /resource-samples\.csv/);
assert.match(document, /phaseBands/);
assert.equal(payload.services[0].sites[0].outcome, 'timeout_with_items');
assert.equal(payload.services[0].sites[0].duration_ms, 80);
assert.equal(payload.services[0].sites[0].snapshot_saved, false);
assert.equal(
payload.services[0].sites[0].snapshot_error_summary,
'page.content: document changed during snapshot.'
);
assert.equal(document.includes('</script><script>alert(1)</script>'), false);
assert.ok(document.includes('\\u003c/script>'));
const inlineScripts = [...document.matchAll(/<script(?: [^>]*)?>([\s\S]*?)<\/script>/g)]
.map((match) => match[1])
.filter((script) => script.trim() && !script.trim().startsWith('{'));
assert.equal(inlineScripts.length, 1);
assert.doesNotThrow(() => new vm.Script(inlineScripts[0]));
});
test('report and resource artifacts are generated as standalone files', () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'spider-report-test-'));
try {
const outputDir = path.join(tempDir, 'output-moli');
fs.mkdirSync(outputDir);
const resourceData = fixtureResourceData();
writeResourceArtifacts(outputDir, resourceData);
const result = {
target: 'moli',
service: 'moli',
success: true,
outputDir,
report: { summary: { averageFillRate: 100 }, files: [] },
leakage: { suspiciousCount: 0 },
metadata: fixtureMetadata(),
resourceData
};
const written = writeSpiderReport({
runDir: tempDir,
args: fixtureArgs(),
results: [result]
});
assert.ok(fs.existsSync(path.join(outputDir, 'resource-samples.json')));
assert.ok(fs.existsSync(path.join(outputDir, 'resource-samples.csv')));
assert.ok(fs.existsSync(written.htmlPath));
assert.ok(fs.existsSync(written.dataPath));
assert.equal(
JSON.parse(fs.readFileSync(written.dataPath, 'utf8')).schema,
'moli.browser-spider.report.v3'
);
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
test('site diagnostics retain timing and classify navigation outcomes', () => {
const sites = buildSiteDiagnostics({
metadata: fixtureMetadata(),
resourceData: fixtureResourceData()
});
assert.deepEqual(sites.map((site) => ({
case_name: site.case_name,
site: site.site,
duration_ms: site.duration_ms,
outcome: site.outcome,
expected_item_count: site.expected_item_count,
item_count: site.item_count,
error_summary: site.error_summary,
snapshot_saved: site.snapshot_saved,
snapshot_error_summary: site.snapshot_error_summary
})), [{
case_name: 'news',
site: 'fixture news',
duration_ms: 80,
outcome: 'timeout_with_items',
expected_item_count: 5,
item_count: 5,
error_summary: 'page.goto: Timeout 1500ms exceeded.',
snapshot_saved: false,
snapshot_error_summary: 'page.content: document changed during snapshot.'
}]);
});
test('site diagnostics distinguish snapshot failures from ordinary empty pages', () => {
const metadata = fixtureMetadata();
const site = metadata.cases[0].siteMeta[0];
site.gotoOk = true;
site.gotoError = null;
site.itemCount = 0;
site.items = [];
const [diagnostic] = buildSiteDiagnostics({
metadata,
resourceData: fixtureResourceData()
});
assert.equal(diagnostic.outcome, 'snapshot_error_empty');
assert.equal(diagnostic.expected_item_count, 5);
});
moli-benchmark/browser-spider-local/tests/resource-collector.test.mjs
+231
View File
@@ -0,0 +1,231 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
LinuxProcessTreeCollector,
parseProcStat,
parseProcStatus,
parseSmapsRollup
} from '../lib/observability/linux-procfs.mjs';
import { buildResourceArtifact } from '../lib/observability/sampler.mjs';
function processRecord({
pid,
ppid = 0,
ticks = 0,
start = pid * 10,
threads = 1,
rssPages = 1
}) {
return {
pid,
ppid,
state: 'S',
cpu_ticks: ticks,
start_time_ticks: start,
thread_count: threads,
rss_pages: rssPages
};
}
class FakeProcfs {
constructor(frames) {
this.frames = frames;
this.index = 0;
}
frame() {
return this.frames[this.index];
}
advance() {
this.index = Math.min(this.index + 1, this.frames.length - 1);
}
readStat(pid) {
const process = this.frame().processes.get(pid);
if (!process) {
throw new Error(`missing pid ${pid}`);
}
return process;
}
scanProcesses() {
return this.frame().processes;
}
readMemory(process) {
return this.frame().memory.get(process.pid);
}
}
function frame(processes, memory) {
return {
processes: new Map(processes.map((process) => [process.pid, process])),
memory: new Map(Object.entries(memory).map(([pid, value]) => [Number(pid), value]))
};
}
test('proc parsers preserve CPU, memory and thread semantics', () => {
const fields = [
'S', '7', '0', '0', '0', '0', '0', '0', '0', '0', '0',
'120', '30', '0', '0', '0', '0', '9', '0', '444', '0', '12'
];
const stat = parseProcStat(`42 (renderer worker) ${fields.join(' ')}`);
assert.deepEqual(stat, {
pid: 42,
ppid: 7,
state: 'S',
cpu_ticks: 150,
start_time_ticks: 444,
thread_count: 9,
rss_pages: 12
});
assert.deepEqual(
parseProcStatus('VmRSS:\t321 kB\nThreads:\t7\n'),
{ rss_bytes: 321 * 1024, thread_count: 7 }
);
assert.equal(parseSmapsRollup('Rss: 400 kB\nPss: 123 kB\n'), 123 * 1024);
});
test('collector aggregates a process-tree delta and does not double-count overlapping roots', () => {
const procfs = new FakeProcfs([
frame(
[
processRecord({ pid: 10, ticks: 100, start: 1000, threads: 3 }),
processRecord({ pid: 11, ppid: 10, ticks: 40, start: 1100, threads: 2 })
],
{
10: { rss_bytes: 1000, pss_bytes: 700, thread_count: 3 },
11: { rss_bytes: 500, pss_bytes: 300, thread_count: 2 }
}
),
frame(
[
processRecord({ pid: 10, ticks: 120, start: 1000, threads: 3 }),
processRecord({ pid: 11, ppid: 10, ticks: 50, start: 1100, threads: 2 })
],
{
10: { rss_bytes: 1200, pss_bytes: 800, thread_count: 3 },
11: { rss_bytes: 600, pss_bytes: 350, thread_count: 2 }
}
)
]);
const collector = new LinuxProcessTreeCollector({
procfs,
ticksPerSecond: 100,
intervalMs: 100,
clock: () => 0
});
collector.addRoot('browser', 10);
collector.addRoot('renderer', 11);
collector.sample({ elapsedMs: 0 });
procfs.advance();
const sample = collector.sample({ elapsedMs: 100 });
assert.equal(sample.total.process_count, 2);
assert.equal(sample.total.thread_count, 5);
assert.equal(sample.total.rss_bytes, 1800);
assert.equal(sample.total.pss_bytes, 1150);
assert.ok(Math.abs(sample.total.cpu_percent - 300) < 1e-9);
assert.equal(sample.workers.browser.process_count, 2);
assert.equal(sample.workers.renderer.process_count, 1);
});
test('collector rejects recycled root PIDs and incomplete PSS', () => {
const procfs = new FakeProcfs([
frame(
[
processRecord({ pid: 10, ticks: 10, start: 1000 }),
processRecord({ pid: 11, ppid: 10, ticks: 5, start: 1100 })
],
{
10: { rss_bytes: 1000, pss_bytes: 700, thread_count: 1 },
11: { rss_bytes: 500, pss_bytes: null, thread_count: 1 }
}
),
frame(
[processRecord({ pid: 10, ticks: 1, start: 9999 })],
{ 10: { rss_bytes: 200, pss_bytes: 150, thread_count: 1 } }
)
]);
const collector = new LinuxProcessTreeCollector({
procfs,
ticksPerSecond: 100,
intervalMs: 100,
clock: () => 0
});
collector.addRoot('browser', 10);
const first = collector.sample({ elapsedMs: 0 });
assert.equal(first.total.rss_bytes, 1500);
assert.equal(first.total.pss_bytes, null);
assert.equal(first.total.pss_process_count, 1);
procfs.advance();
const recycled = collector.sample({ elapsedMs: 100 });
assert.equal(recycled.total.process_count, 0);
assert.equal(recycled.workers.browser.process_count, 0);
});
test('resource artifact derives case, worker, weighted CPU and sampler health summaries', () => {
const collector = {
platform: 'linux',
method: 'test',
interval_ms: 100,
cpu_ticks_per_second: 100,
host_logical_cpu_count: 8,
roots: {},
errors: {},
samples: [
{
elapsed_ms: 0,
kind: 'root-registered',
capture_duration_ms: 1,
total: { cpu_percent: null, rss_bytes: 100, pss_bytes: 90, process_count: 1, thread_count: 2 },
workers: { 'worker-1': { cpu_percent: null, rss_bytes: 100, pss_bytes: 90, process_count: 1, thread_count: 2 } }
},
{
elapsed_ms: 100,
kind: 'periodic',
capture_duration_ms: 2,
total: { cpu_percent: 50, rss_bytes: 200, pss_bytes: 180, process_count: 1, thread_count: 2 },
workers: { 'worker-1': { cpu_percent: 50, rss_bytes: 200, pss_bytes: 180, process_count: 1, thread_count: 2 } }
},
{
elapsed_ms: 200,
kind: 'periodic',
capture_duration_ms: 3,
total: { cpu_percent: 75, rss_bytes: 250, pss_bytes: 230, process_count: 1, thread_count: 2 },
workers: { 'worker-1': { cpu_percent: 75, rss_bytes: 250, pss_bytes: 230, process_count: 1, thread_count: 2 } }
},
{
elapsed_ms: 300,
kind: 'final',
capture_duration_ms: 120,
total: { cpu_percent: 100, rss_bytes: 300, pss_bytes: 280, process_count: 1, thread_count: 3 },
workers: { 'worker-1': { cpu_percent: 100, rss_bytes: 300, pss_bytes: 280, process_count: 1, thread_count: 3 } }
}
]
};
const artifact = buildResourceArtifact({
enabled: true,
status: 'available',
intervalMs: 100,
collector,
markers: [
{ type: 'case-start', case_name: 'news', elapsed_ms: 50 },
{ type: 'case-done', case_name: 'news', elapsed_ms: 300 }
]
});
assert.equal(artifact.summary.peak_cpu_percent, 100);
assert.equal(artifact.summary.average_cpu_percent, 75);
assert.equal(artifact.summary.peak_rss_bytes, 300);
assert.equal(artifact.summary.sampling_overrun_count, 1);
assert.equal(artifact.summary.average_observed_interval_ms, 100);
assert.equal(artifact.summary.max_observed_interval_ms, 100);
assert.equal(artifact.summary.late_sample_count, 0);
assert.equal(artifact.summary.workers['worker-1'].peak_pss_bytes, 280);
assert.equal(artifact.summary.cases[0].case_name, 'news');
assert.equal(artifact.summary.cases[0].sample_count, 3);
});
moli-benchmark/browser-spider-local/tests/resource-sampler.test.mjs
+39
View File
@@ -0,0 +1,39 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { ProcessTreeResourceSampler } from '../lib/observability/sampler.mjs';
test('worker-thread sampler captures the current Linux process tree', {
skip: process.platform !== 'linux',
timeout: 5000
}, async () => {
const sampler = new ProcessTreeResourceSampler({ intervalMs: 100 });
sampler.start();
sampler.addRoot('self', process.pid);
sampler.mark({ type: 'case-start', caseName: 'fixture' });
await new Promise((resolve) => setTimeout(resolve, 240));
sampler.mark({ type: 'case-done', caseName: 'fixture' });
const artifact = await sampler.stop();
assert.equal(artifact.status, 'available');
assert.ok(artifact.samples.length >= 3);
assert.ok(artifact.summary.peak_rss_bytes > 0);
assert.ok(artifact.summary.peak_process_count >= 1);
assert.equal(artifact.summary.cases[0].case_name, 'fixture');
assert.ok(
artifact.samples.at(-1).elapsed_ms
>= artifact.markers.find((marker) => marker.type === 'case-done').elapsed_ms
);
});
test('disabled sampler produces an explicit empty artifact', async () => {
const sampler = new ProcessTreeResourceSampler({ enabled: false });
sampler.start();
sampler.mark({ type: 'service-start', service: 'moli' });
const artifact = await sampler.stop();
assert.equal(artifact.status, 'disabled');
assert.equal(artifact.enabled, false);
assert.equal(artifact.samples.length, 0);
assert.equal(artifact.markers[0].service, 'moli');
});
moli-benchmark/fixtures/agent-episode/manifest.json
+184
View File
@@ -0,0 +1,184 @@
{
"schema": "moli.agent-episode.manifest.v1",
"fixture_version": "agent-episode-fixture-v1",
"episodes": [
{
"id": "observe-static",
"start_path": "/agent/observe-static",
"steps": [
{
"operation": "observe",
"expect": {
"url_path": "/agent/observe-static",
"text_contains": ["Agent static observation ready", "Search"],
"min_controls": 3
}
}
]
},
{
"id": "fill-reactive-form",
"start_path": "/agent/fill-reactive-form",
"steps": [
{
"operation": "fill",
"selector": "[data-lex-id=lex-0]",
"value": "moli"
},
{
"operation": "observe",
"expect": {
"url_path": "/agent/fill-reactive-form",
"text_contains": ["value:moli;events:input,change"],
"min_controls": 1
}
}
]
},
{
"id": "click-same-document",
"start_path": "/agent/click-same-document",
"steps": [
{
"operation": "click",
"selector": "[data-lex-id=lex-0]"
},
{
"operation": "observe",
"expect": {
"url_path": "/agent/click-same-document",
"text_contains": ["events:pointerdown,mousedown,pointerup,mouseup,click"],
"min_controls": 1
}
}
]
},
{
"id": "click-cross-document",
"start_path": "/agent/click-cross-document",
"steps": [
{
"operation": "click",
"selector": "[data-lex-id=lex-0]",
"expect_navigation_path": "/agent/cross-document-result"
},
{
"operation": "observe",
"expect": {
"url_path": "/agent/cross-document-result",
"text_contains": ["Cross document result cross-document-result"],
"min_controls": 1,
"old_marker_type": "undefined",
"new_marker": "cross-document-result"
}
}
]
},
{
"id": "dynamic-controls",
"start_path": "/agent/dynamic-controls",
"steps": [
{
"operation": "click",
"selector": "[data-lex-id=lex-0]"
},
{
"operation": "fill",
"selector": "[data-lex-id=lex-0]",
"value": "dynamic-moli"
},
{
"operation": "observe",
"expect": {
"url_path": "/agent/dynamic-controls",
"text_contains": ["phase:replaced;value:dynamic-moli"],
"min_controls": 1
}
}
]
},
{
"id": "failed-navigation",
"start_path": "/agent/failed-navigation",
"steps": [
{
"operation": "navigate",
"path": "/agent/reset-before-response?episode=failed-navigation",
"expect_error_text_contains": "ERR_CONNECTION_RESET",
"expect_error_document": true
},
{
"operation": "observe",
"expect": {
"url": "chrome-error://chromewebdata/",
"unreachable_path": "/agent/reset-before-response",
"old_marker_type": "undefined"
}
}
]
},
{
"id": "idle-resume",
"start_path": "/agent/idle-resume",
"steps": [
{
"operation": "observe",
"expect": {
"url_path": "/agent/idle-resume",
"text_contains": ["Idle episode remains available"],
"min_controls": 1,
"idle_boot_type": "number"
}
},
{
"operation": "observe",
"expect": {
"url_path": "/agent/idle-resume",
"text_contains": ["Idle episode remains available"],
"min_controls": 1,
"idle_boot_type": "number"
}
}
]
},
{
"id": "episode-isolation",
"start_path": "/agent/isolation?token=alpha",
"peer_start_path": "/agent/isolation?token=beta",
"steps": [
{
"operation": "fill",
"participant": "primary",
"selector": "[data-lex-id=lex-0]",
"value": "alpha-value"
},
{
"operation": "fill",
"participant": "peer",
"selector": "[data-lex-id=lex-0]",
"value": "beta-value"
},
{
"operation": "observe",
"participant": "primary",
"expect": {
"url_path": "/agent/isolation",
"text_contains": ["token:alpha;value:alpha-value"],
"text_not_contains": ["beta-value"],
"isolation_token": "alpha"
}
},
{
"operation": "observe",
"participant": "peer",
"expect": {
"url_path": "/agent/isolation",
"text_contains": ["token:beta;value:beta-value"],
"text_not_contains": ["alpha-value"],
"isolation_token": "beta"
}
}
]
}
]
}
moli-benchmark/known-failures/wasm-moli-current.json
+286
View File
@@ -0,0 +1,286 @@
{
"engine": "moli",
"name": "wasm-moli-current",
"categories": {
"wpt-core-harness-table64": {
"tracking_doc": "docs/wasm-wpt-compat-followups-current.md#14",
"scope": "The upstream async core harness does not provide the spectest.table64 import required by table64.wast.js.",
"evidence": [
{
"kind": "doc",
"path": "docs/wasm-wpt-compat-followups-current.md#14",
"note": "Focused Moli and Chromium comparison after declared-limit normalization."
},
{
"kind": "wpt",
"path": "../wpt/wasm/core/memory64/table64.wast.js.html",
"note": "Imports spectest.table64 in its final instantiation case."
},
{
"kind": "wpt",
"path": "../wpt/wasm/core/js/harness/async_index.js",
"note": "Defines spectest.table and spectest.memory but no spectest.table64."
}
]
},
"wasm-global-live-binding": {
"tracking_doc": "docs/wasm-global-live-binding-design-current.md",
"scope": "Mutable wasm global export live bindings require V8/module-binding support, not Rust wrapper refreshes.",
"evidence": [
{
"kind": "doc",
"path": "docs/wasm-global-live-binding-design-current.md",
"note": "V8 cell-load design boundary and ignored acceptance tests."
},
{
"kind": "wpt",
"path": "../wpt/wasm/jsapi/esm-integration/global-exports-live-bindings.tentative.any.js",
"note": "WPT mutable global live-binding expectation."
},
{
"kind": "local-test",
"path": "moli-core/tests/scripts.rs",
"note": "Window ignored acceptance tests for mutable wasm global live bindings and re-exports."
},
{
"kind": "local-test",
"path": "moli-renderer-v8/src/worker/thread/tests/modules.rs",
"note": "Worker ignored acceptance tests for mutable wasm global live bindings and re-exports."
}
]
},
"wpt-idl-source-phase-skew": {
"tracking_doc": "docs/wasm-module-script-current.md",
"scope": "WPT IDL expectations that lag the source-phase AbstractModuleSource prototype chain.",
"evidence": [
{
"kind": "doc",
"path": "docs/wasm-module-script-current.md",
"note": "Source-phase AbstractModuleSource prototype chain tradeoff."
},
{
"kind": "wpt",
"path": "../wpt/wasm/jsapi/idlharness.any.js",
"note": "Current IDL harness expectation that conflicts with the source-phase prototype chain."
},
{
"kind": "wpt",
"path": "../wpt/wasm/jsapi/esm-integration/source-phase.tentative.any.js",
"note": "Source-phase WPT coverage that current runtime keeps passing instead of reverting the prototype chain."
},
{
"kind": "local-test",
"path": "moli-core/tests/scripts.rs",
"note": "Document source-phase wasm import locks AbstractModuleSource inheritance and hidden global shape."
},
{
"kind": "local-test",
"path": "moli-renderer-v8/src/worker/thread/tests/modules.rs",
"note": "Module worker source-phase wasm import locks AbstractModuleSource inheritance and hidden global shape."
}
]
},
"wpt-webapi-global-shape-skew": {
"tracking_doc": "docs/wasm-wpt-compat-followups-current.md",
"scope": "Tentative webapi ESM cases expecting raw WebAssembly.Global where jsapi ESM expects unwrapped values.",
"evidence": [
{
"kind": "doc",
"path": "docs/wasm-wpt-compat-followups-current.md#7",
"note": "Comparison of jsapi unwrapped global expectations with webapi tentative raw-global assertions."
},
{
"kind": "wpt",
"path": "../wpt/wasm/webapi/esm-integration/exported-names.tentative.html",
"note": "WebAPI tentative raw-global expectation for exported names."
},
{
"kind": "wpt",
"path": "../wpt/wasm/webapi/esm-integration/js-wasm-cycle.tentative.html",
"note": "WebAPI tentative raw-global assertion encountered before the final cycle assertion."
},
{
"kind": "local-test",
"path": "moli-core/tests/scripts.rs",
"note": "Window static/dynamic wasm imports expose global exports as unwrapped numbers, not WebAssembly.Global wrappers."
},
{
"kind": "local-test",
"path": "moli-renderer-v8/src/worker/thread/tests/modules.rs",
"note": "Module worker wasm imports expose global exports as unwrapped numbers, not WebAssembly.Global wrappers."
}
]
},
"wpt-webapi-cycle-no-stable-baseline": {
"tracking_doc": "docs/wasm-js-cycle-design-current.md",
"scope": "Tentative webapi JS/WASM cycle shape without a stable Chromium/Blink passing baseline; recursive SyntheticModule evaluation is not acceptable.",
"evidence": [
{
"kind": "doc",
"path": "docs/wasm-js-cycle-design-current.md",
"note": "Why this tentative cycle is outside the current Chromium-aligned runtime target."
},
{
"kind": "wpt",
"path": "../wpt/wasm/webapi/esm-integration/wasm-js-cycle.tentative.html",
"note": "WPT wasm-higher JS/WASM cycle shape that Chromium/Blink does not provide as a stable passing baseline."
},
{
"kind": "local-test",
"path": "moli-core/tests/scripts.rs",
"note": "Window ignored cycle acceptance and current guard regression."
},
{
"kind": "local-test",
"path": "moli-renderer-v8/src/worker/thread/tests/modules.rs",
"note": "Worker ignored cycle acceptance and current guard regression."
}
]
},
"wpt-worker-message-shape-skew": {
"tracking_doc": "docs/wasm-wpt-compat-followups-current.md",
"scope": "Tentative worker message assertions that conflict with browser MessageEvent/postMessage semantics.",
"evidence": [
{
"kind": "wpt",
"path": "../wpt/wasm/webapi/esm-integration/worker.tentative.html",
"note": "Compares Worker.onmessage's MessageEvent parameter with 42."
},
{
"kind": "wpt",
"path": "../wpt/wasm/webapi/esm-integration/worker-import.tentative.html",
"note": "Compares worker-import helper object payload with 42."
},
{
"kind": "wpt",
"path": "../wpt/wasm/webapi/esm-integration/resources/worker-helper.js",
"note": "Posts { value, checks } for worker-import.tentative.html."
},
{
"kind": "doc",
"path": "docs/wasm-wpt-compat-followups-current.md#8",
"note": "Chromium comparison and local MessageEvent regression."
},
{
"kind": "local-test",
"path": "moli-renderer-v8/src/runtime/page_vm/tests/worker.rs",
"note": "Parent Worker.onmessage receives a MessageEvent with data, toStringTag, and message type."
},
{
"kind": "local-test",
"path": "moli-renderer-v8/src/worker/thread/tests/postmessage.rs",
"note": "WorkerGlobalScope onmessage receives a MessageEvent with data, toStringTag, and message type."
}
]
}
},
"rules": [
{
"case_path": "wasm/core/memory64/table64.wast.js.html",
"category": "wpt-core-harness-table64",
"expected_status": "fail",
"message_contains": [
"table import requires a WebAssembly.Table"
],
"reason": "The upstream async core harness omits spectest.table64; Chromium fails the same import with the same LinkError."
},
{
"case_path": "wasm/jsapi/esm-integration/global-exports-live-bindings.tentative.any.js?moli-wpt-any=dedicatedworker",
"category": "wasm-global-live-binding",
"expected_status": "fail",
"message_contains": [
"Local mutable global exports should be live bindings",
"expected 555 but got 100",
"Dep module mutable global exports should be live bindings",
"expected 3001 but got 2001"
],
"reason": "Mutable wasm global exports are still SyntheticModule snapshot values instead of V8-aware live bindings."
},
{
"case_path": "wasm/jsapi/esm-integration/global-exports-live-bindings.tentative.any.js?moli-wpt-any=window",
"category": "wasm-global-live-binding",
"expected_status": "fail",
"message_contains": [
"Local mutable global exports should be live bindings",
"expected 555 but got 100",
"Dep module mutable global exports should be live bindings",
"expected 3001 but got 2001"
],
"reason": "Mutable wasm global exports are still SyntheticModule snapshot values instead of V8-aware live bindings."
},
{
"case_path": "wasm/jsapi/idlharness.any.js?moli-wpt-any=dedicatedworker",
"category": "wpt-idl-source-phase-skew",
"expected_status": "fail",
"message_contains": [
"prototype of self's property \"Module\" is not Function.prototype",
"AbstractModuleSource",
"prototype of Module.prototype is not Object.prototype"
],
"reason": "Current source-phase runtime prototype chain intentionally extends the hidden AbstractModuleSource constructor while this IDL harness still expects the old Module chain."
},
{
"case_path": "wasm/jsapi/idlharness.any.js?moli-wpt-any=window",
"category": "wpt-idl-source-phase-skew",
"expected_status": "fail",
"message_contains": [
"prototype of self's property \"Module\" is not Function.prototype",
"AbstractModuleSource",
"prototype of Module.prototype is not Object.prototype"
],
"reason": "Current source-phase runtime prototype chain intentionally extends the hidden AbstractModuleSource constructor while this IDL harness still expects the old Module chain."
},
{
"case_path": "wasm/webapi/esm-integration/exported-names.tentative.html",
"category": "wpt-webapi-global-shape-skew",
"expected_status": "fail",
"message_contains": [
"Exported names from a WebAssembly module",
"assert_true: expected true got false"
],
"reason": "This webapi tentative case still expects raw WebAssembly.Global where jsapi ESM integration expects an unwrapped JS value."
},
{
"case_path": "wasm/webapi/esm-integration/js-wasm-cycle.tentative.html",
"category": "wpt-webapi-global-shape-skew",
"expected_status": "fail",
"message_contains": [
"testharness completed without reporting any subtests",
"Unhandled rejection: assert_true: expected true got false"
],
"reason": "The current failure is in the webapi tentative raw-global/cycle expectation area, not a reason to roll back jsapi global unwrapping."
},
{
"case_path": "wasm/webapi/esm-integration/wasm-js-cycle.tentative.html",
"category": "wpt-webapi-cycle-no-stable-baseline",
"expected_status": "fail",
"message_contains": [
"testharness completed without reporting any subtests",
"Unhandled rejection: cyclic WebAssembly module evaluation through JavaScript dependencies is not supported yet"
],
"reason": "This tentative webapi cycle has no stable Chromium/Blink passing baseline; recursive SyntheticModule evaluation is not acceptable."
},
{
"case_path": "wasm/webapi/esm-integration/worker-import.tentative.html",
"category": "wpt-worker-message-shape-skew",
"expected_status": "fail",
"message_contains": [
"Testing import of WebAssembly from JavaScript worker",
"expected (number) 42 but got (object)",
"[object Object]"
],
"reason": "The local WPT worker helper posts an object payload while the page asserts number 42; do not change Worker message semantics to satisfy this case."
},
{
"case_path": "wasm/webapi/esm-integration/worker.tentative.html",
"category": "wpt-worker-message-shape-skew",
"expected_status": "fail",
"message_contains": [
"Testing WebAssembly worker",
"expected (number) 42 but got (object)",
"[object MessageEvent]"
],
"reason": "Worker.onmessage receives a MessageEvent by browser semantics; do not pass bare data to satisfy this tentative case."
}
]
}
moli-benchmark/moli_benchmark/__init__.py
+3
View File
@@ -0,0 +1,3 @@
from __future__ import annotations
__version__ = "0.1.0"
moli-benchmark/moli_benchmark/__main__.py
+7
View File
@@ -0,0 +1,7 @@
from __future__ import annotations
from .cli import main
if __name__ == "__main__":
raise SystemExit(main())
moli-benchmark/moli_benchmark/agent_episode.py
+2105
View File
File diff suppressed because it is too large Load Diff
moli-benchmark/moli_benchmark/agent_episode_fixture.py
+255
View File
@@ -0,0 +1,255 @@
from __future__ import annotations
import html
import json
import socket
import struct
import threading
import time
import urllib.parse
from http import HTTPStatus
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from typing import Any
FIXTURE_VERSION = "agent-episode-fixture-v1"
def _document(title: str, body: str, script: str = "") -> bytes:
script_tag = f"<script>{script}</script>" if script else ""
return (
"<!doctype html><html><head><meta charset='utf-8'>"
f"<title>{html.escape(title)}</title>"
"<style>"
"body{font-family:sans-serif}"
"input,button,a{display:block;width:220px;min-height:32px;margin:8px}"
"output{display:block;white-space:pre-wrap}"
"</style></head><body>"
f"{body}{script_tag}</body></html>"
).encode("utf-8")
def response_for_agent_path(path: str, query: dict[str, list[str]]) -> bytes | None:
if path == "/agent/observe-static":
return _document(
"observe static",
"<main>Agent static observation ready</main>"
"<label>Search <input id='search' placeholder='Search'></label>"
"<button id='noop'>No operation</button>"
"<a href='#details'>Details</a>",
)
if path == "/agent/fill-reactive-form":
return _document(
"reactive form",
"<main>Reactive form ready</main>"
"<label>Search <input id='query' placeholder='Search'></label>"
"<output id='state'>value:;events:</output>",
"""
globalThis.__agentEpisodeState = { events: [] };
const input = document.querySelector('#query');
const output = document.querySelector('#state');
const render = () => {
output.textContent = `value:${input.value};events:${__agentEpisodeState.events.join(',')}`;
};
input.addEventListener('input', () => {
__agentEpisodeState.events.push('input');
queueMicrotask(render);
});
input.addEventListener('change', () => {
__agentEpisodeState.events.push('change');
render();
});
""",
)
if path == "/agent/click-same-document":
return _document(
"same document click",
"<main>Same document click ready</main>"
"<button id='activate'>Activate</button>"
"<output id='events'>events:</output>",
"""
globalThis.__agentEpisodeState = { events: [] };
const button = document.querySelector('#activate');
const output = document.querySelector('#events');
for (const type of ['pointerdown', 'mousedown', 'pointerup', 'mouseup', 'click']) {
button.addEventListener(type, () => {
__agentEpisodeState.events.push(type);
output.textContent = `events:${__agentEpisodeState.events.join(',')}`;
});
}
""",
)
if path == "/agent/click-cross-document":
return _document(
"cross document click",
"<main>Cross document source</main>"
"<button id='navigate'>Open result</button>",
"""
globalThis.__agentOldRealmMarker = 'cross-document-source';
document.querySelector('#navigate').addEventListener('click', () => {
location.href = '/agent/cross-document-result?token=cross-document-result';
});
""",
)
if path == "/agent/cross-document-result":
token = html.escape(query.get("token", ["missing"])[0])
return _document(
"cross document result",
f"<main>Cross document result {token}</main>"
"<button id='result-button'>Result control</button>",
"globalThis.__agentNewRealmMarker = 'cross-document-result';",
)
if path == "/agent/dynamic-controls":
return _document(
"dynamic controls",
"<main>Dynamic controls ready</main>"
"<section id='controls'><button id='replace'>Replace controls</button></section>"
"<output id='dynamic-state'>phase:initial</output>",
"""
const controls = document.querySelector('#controls');
const output = document.querySelector('#dynamic-state');
document.querySelector('#replace').addEventListener('click', () => {
controls.replaceChildren();
const input = document.createElement('input');
input.id = 'dynamic-input';
input.placeholder = 'Dynamic value';
controls.appendChild(input);
output.textContent = 'phase:replaced;value:';
input.addEventListener('input', () => {
queueMicrotask(() => {
output.textContent = `phase:replaced;value:${input.value}`;
});
});
});
""",
)
if path == "/agent/idle-resume":
return _document(
"idle resume",
"<main>Idle episode remains available</main>"
"<button id='idle-control'>Idle control</button>",
"""
globalThis.__agentIdleBoot = Date.now();
globalThis.__agentIdleProbe = () => ({
boot: globalThis.__agentIdleBoot,
now: Date.now(),
href: location.href,
});
""",
)
if path == "/agent/failed-navigation":
return _document(
"failed navigation source",
"<main>Failure source document</main>"
"<button id='old-control'>Old document control</button>",
"globalThis.__agentOldRealmMarker = 'must-retire';",
)
if path == "/agent/isolation":
token = query.get("token", ["missing"])[0]
escaped = html.escape(token)
token_json = json.dumps(token).replace("</", "<\\/")
return _document(
f"isolation {token}",
f"<main>Isolation token {escaped}</main>"
"<label>Value <input id='isolation-input' placeholder='Isolation value'></label>"
f"<output id='isolation-output'>token:{escaped};value:</output>",
f"""
globalThis.__agentIsolationToken = {token_json};
const input = document.querySelector('#isolation-input');
const output = document.querySelector('#isolation-output');
input.addEventListener('input', () => {{
queueMicrotask(() => {{
output.textContent = `token:${{__agentIsolationToken}};value:${{input.value}}`;
}});
}});
""",
)
return None
class AgentEpisodeFixtureServer:
def __init__(self) -> None:
self._lock = threading.Lock()
self._requests: list[dict[str, Any]] = []
handler = self._handler_class()
self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), handler)
self.httpd.daemon_threads = True
self.thread = threading.Thread(
target=self.httpd.serve_forever,
name="moli-agent-episode-fixture",
daemon=True,
)
@property
def base_url(self) -> str:
host, port = self.httpd.server_address
return f"http://{host}:{port}"
@property
def requests(self) -> list[dict[str, Any]]:
with self._lock:
return list(self._requests)
def url(self, path: str) -> str:
return f"{self.base_url}/{path.lstrip('/')}"
def start(self) -> None:
self.thread.start()
def stop(self) -> None:
self.httpd.shutdown()
self.httpd.server_close()
self.thread.join(timeout=5)
def __enter__(self) -> "AgentEpisodeFixtureServer":
self.start()
return self
def __exit__(self, exc_type: object, exc: object, tb: object) -> None:
self.stop()
def _record(self, method: str, path: str) -> None:
with self._lock:
self._requests.append(
{"timestamp": time.time(), "method": method, "path": path}
)
def _handler_class(self) -> type[BaseHTTPRequestHandler]:
outer = self
class Handler(BaseHTTPRequestHandler):
server_version = "MoliAgentEpisodeFixture/1"
def log_message(self, _format: str, *_args: object) -> None:
return
def do_GET(self) -> None:
parsed = urllib.parse.urlparse(self.path)
outer._record("GET", self.path)
if parsed.path == "/agent/reset-before-response":
self.connection.setsockopt(
socket.SOL_SOCKET,
socket.SO_LINGER,
struct.pack("ii", 1, 0),
)
self.close_connection = True
self.connection.close()
return
body = response_for_agent_path(
parsed.path,
urllib.parse.parse_qs(parsed.query),
)
if body is None:
self.send_error(HTTPStatus.NOT_FOUND)
return
self.send_response(HTTPStatus.OK)
self.send_header("Content-Type", "text/html; charset=utf-8")
self.send_header("Cache-Control", "no-store")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
try:
self.wfile.write(body)
except (BrokenPipeError, ConnectionResetError):
return
return Handler
moli-benchmark/moli_benchmark/agent_episode_report.py
+191
View File
@@ -0,0 +1,191 @@
from __future__ import annotations
import datetime as dt
import json
from pathlib import Path
from typing import Any
from .artifacts import write_json, write_text
REPORT_DATA_SCHEMA = "moli.agent-episode.report.v1"
def _embedded_json(payload: dict[str, Any]) -> str:
return (
json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
.replace("</", "<\\/")
.replace("\u2028", "\\u2028")
.replace("\u2029", "\\u2029")
)
def write_agent_episode_report(
*,
suite_dir: Path,
summary: dict[str, Any],
episode_rows: list[dict[str, Any]],
step_rows: list[dict[str, Any]],
resources: dict[str, Any],
markers: list[dict[str, Any]],
config: dict[str, Any],
) -> dict[str, Any]:
payload = {
"schema": REPORT_DATA_SCHEMA,
"generated_at": dt.datetime.now(dt.UTC).isoformat(),
"summary": summary,
"config": config,
"episodes": episode_rows,
"steps": step_rows,
"resources": resources,
"phase_markers": markers,
}
write_json(suite_dir / "report-data.json", payload)
write_text(suite_dir / "index.html", _render_html(payload))
return payload
def _render_html(payload: dict[str, Any]) -> str:
embedded = _embedded_json(payload)
return f"""<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>Agent Episode CDP Benchmark</title>
<style>
:root {{ color-scheme: dark; --bg:#091019; --panel:#111c28; --line:#26384a;
--text:#e7edf4; --muted:#91a3b5; --good:#4dd4a0; --bad:#ff6b7a;
--blue:#61a8ff; --violet:#b28cff; --amber:#f4c15d; }}
* {{ box-sizing:border-box; }}
body {{ margin:0; background:var(--bg); color:var(--text); font:14px/1.45 ui-sans-serif,system-ui,sans-serif; }}
main {{ width:min(1500px,96vw); margin:0 auto; padding:32px 0 60px; }}
h1,h2,h3 {{ margin:0 0 12px; letter-spacing:-.02em; }} h1 {{ font-size:30px; }} h2 {{ margin-top:30px; }}
.subtitle,.muted {{ color:var(--muted); }} .subtitle {{ margin:4px 0 22px; }}
.grid {{ display:grid; grid-template-columns:repeat(auto-fit,minmax(190px,1fr)); gap:12px; }}
.card,.panel {{ background:var(--panel); border:1px solid var(--line); border-radius:10px; }}
.card {{ padding:16px; }} .card .value {{ display:block; font-size:25px; font-weight:700; margin-top:5px; }}
.panel {{ padding:18px; margin-top:12px; overflow:auto; }}
.good {{ color:var(--good); }} .bad {{ color:var(--bad); }}
table {{ width:100%; border-collapse:collapse; font-variant-numeric:tabular-nums; }}
th,td {{ text-align:left; padding:8px 10px; border-bottom:1px solid var(--line); vertical-align:top; }}
th {{ color:var(--muted); font-size:12px; text-transform:uppercase; letter-spacing:.04em; }}
code {{ color:#c6ddf4; }} a {{ color:var(--blue); }}
.charts {{ display:grid; grid-template-columns:repeat(auto-fit,minmax(430px,1fr)); gap:12px; }}
.chart {{ min-height:275px; }} svg {{ width:100%; height:225px; overflow:visible; }}
.axis {{ stroke:#43566a; stroke-width:1; }} .gridline {{ stroke:#203243; stroke-width:1; }}
.rss {{ stroke:var(--blue); }} .pss {{ stroke:var(--violet); }} .cpu {{ stroke:var(--amber); }}
.series {{ fill:none; stroke-width:2; vector-effect:non-scaling-stroke; }}
.marker {{ stroke:#718399; stroke-width:1; stroke-dasharray:2 4; opacity:.5; }}
.legend {{ display:flex; gap:18px; color:var(--muted); font-size:12px; }}
.swatch {{ display:inline-block; width:16px; border-top:2px solid; margin-right:5px; vertical-align:middle; }}
.nowrap {{ white-space:nowrap; }} .error {{ max-width:540px; white-space:pre-wrap; overflow-wrap:anywhere; }}
.empty {{ color:var(--muted); padding:16px 0; }}
</style>
</head>
<body><main>
<h1>Agent Episode CDP Benchmark</h1>
<p class="subtitle">Deterministic local RL-shaped CDP episodes. Correctness gates latency and resource interpretation.</p>
<section id="kpis" class="grid"></section>
<h2>Target summary</h2><section id="targets" class="panel"></section>
<h2>Correct operation latency</h2><section id="operations" class="panel"></section>
<h2>Resource timelines</h2><section id="charts" class="charts"></section>
<h2>Case correctness</h2><section id="cases" class="panel"></section>
<h2>Failure drilldown</h2><section id="failures" class="panel"></section>
<h2>Episode rows</h2><section id="episodes" class="panel"></section>
<h2>Artifact contract</h2>
<section class="panel"><code>report-data.json</code> is the renderer-independent authority.
Raw rows: <a href="runs.csv">runs.csv</a>, <a href="steps.csv">steps.csv</a>;
resources: <a href="resource-samples.json">resource-samples.json</a>;
phase log: <a href="events.log">events.log</a>.</section>
<script id="report-data" type="application/json">{embedded}</script>
<script>
const data = JSON.parse(document.getElementById('report-data').textContent);
const summary = data.summary;
const esc = value => String(value ?? '').replace(/[&<>"']/g, c => ({{'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}}[c]));
const num = (value, digits=1) => value == null || !Number.isFinite(Number(value)) ? '—' : Number(value).toFixed(digits);
const ms = value => value == null ? '—' : `${{num(value)}} ms`;
const mib = value => value == null ? '—' : `${{num(Number(value)/(1024*1024))}} MiB`;
const pct = (ok,total) => total ? `${{num(100*ok/total)}}%` : '—';
const median = stat => stat && stat.p50 != null ? stat.p50 : null;
function table(headers, rows) {{
if (!rows.length) return '<div class="empty">No rows.</div>';
return `<table><thead><tr>${{headers.map(h=>`<th>${{esc(h)}}</th>`).join('')}}</tr></thead><tbody>${{
rows.map(row=>`<tr>${{row.map(cell=>`<td>${{cell}}</td>`).join('')}}</tr>`).join('')
}}</tbody></table>`;
}}
const kpis = [
['Episodes', summary.episodes_total], ['Passed', summary.episodes_passed],
['Failed', summary.total_failures], ['Steps observed', summary.steps_total],
['Assertions', `${{summary.assertions_passed}} / ${{summary.assertions_total}}`],
['Workers / parallelism', `${{summary.workers}} / ${{summary.parallelism}}`],
['Step dwell', `${{summary.step_dwell_ms}} ms`],
];
document.getElementById('kpis').innerHTML = kpis.map(([label,value],i)=>
`<div class="card"><span class="muted">${{esc(label)}}</span><span class="value ${{i===1?'good':i===2&&Number(value)?'bad':''}}">${{esc(value)}}</span></div>`).join('');
const targetRows = Object.entries(summary.targets).map(([name,t]) => [
`<strong>${{esc(t.label || name)}}</strong>`, `${{t.passed}} / ${{t.episodes}}`,
pct(t.passed,t.episodes), `<span class="${{t.failures?'bad':'good'}}">${{t.failures}}</span>`,
ms(median(t.active_elapsed_ms)), num(t.resources?.average_cpu_percent),
mib(t.resources?.peak_rss_bytes), mib(t.resources?.peak_pss_bytes),
esc(JSON.stringify(t.status_counts || {{}})),
]);
document.getElementById('targets').innerHTML = table(
['Target','Passed','Pass rate','Failures','Median active','Avg CPU %','Peak RSS','Peak PSS','Statuses'], targetRows);
const operationRows = [];
for (const [name,t] of Object.entries(summary.targets)) for (const [operation,stat] of Object.entries(t.operations || {{}}))
operationRows.push([esc(t.label || name),esc(operation),esc(stat.count ?? 0),ms(stat.p50),ms(stat.p90),ms(stat.max)]);
document.getElementById('operations').innerHTML = table(['Target','Operation','Correct n','p50','p90','max'],operationRows);
const caseRows = [];
for (const [name,t] of Object.entries(summary.targets)) for (const [caseName,c] of Object.entries(t.cases || {{}}))
caseRows.push([esc(t.label || name),esc(caseName),`<span class="${{c.failures?'bad':'good'}}">${{c.failures}}</span>`,ms(median(c.elapsed_ms)),ms(median(c.active_elapsed_ms))]);
document.getElementById('cases').innerHTML = table(['Target','Episode','Failures','Median wall','Median active'],caseRows);
function linePath(samples,key,x0,y0,width,height,maxValue) {{
const usable = samples.filter(s => s[key] != null && Number.isFinite(Number(s[key])));
if (!usable.length || !maxValue) return '';
const maxTime = Math.max(...samples.map(s=>Number(s.elapsed_ms)||0),1);
return usable.map((s,i) => `${{i?'L':'M'}}${{x0 + width*(Number(s.elapsed_ms)||0)/maxTime}},${{y0 + height-height*Number(s[key])/maxValue}}`).join(' ');
}}
function timeline(target,payload) {{
const samples = payload.samples || []; const width=680,height=155,x0=48,y0=18;
if (!samples.length) return `<section class="panel chart"><h3>${{esc(target)}}</h3><div class="empty">No complete resource samples.</div></section>`;
const memoryMax = Math.max(...samples.flatMap(s=>[Number(s.rss_bytes)||0,Number(s.pss_bytes)||0]),1);
const cpuMax = Math.max(...samples.map(s=>Number(s.cpu_percent)||0),100);
const memory = linePath(samples,'rss_bytes',x0,y0,width,height,memoryMax);
const pss = linePath(samples,'pss_bytes',x0,y0,width,height,memoryMax);
const cpu = linePath(samples,'cpu_percent',x0,y0,width,height,cpuMax);
const start = Number(samples[0].timestamp); const span = Math.max(Number(samples.at(-1).timestamp)-start,.001);
const markerLines = (data.phase_markers||[]).filter(m=>m.target===target && ['episode-start','episode-done','episode-failed'].includes(m.event))
.map(m=>`<line class="marker" x1="${{x0+width*Math.max(0,Math.min(1,(Number(m.timestamp)-start)/span))}}" y1="${{y0}}" x2="${{x0+width*Math.max(0,Math.min(1,(Number(m.timestamp)-start)/span))}}" y2="${{y0+height}}"/>`).join('');
return `<section class="panel chart"><h3>${{esc(summary.targets[target]?.label || target)}}</h3>
<div class="legend"><span><i class="swatch rss"></i>RSS</span><span><i class="swatch pss"></i>PSS</span><span><i class="swatch cpu"></i>CPU (independent scale)</span></div>
<svg viewBox="0 0 760 205" role="img" aria-label="resource timeline">
<line class="axis" x1="${{x0}}" y1="${{y0+height}}" x2="${{x0+width}}" y2="${{y0+height}}"/>
<line class="gridline" x1="${{x0}}" y1="${{y0+height/2}}" x2="${{x0+width}}" y2="${{y0+height/2}}"/>
${{markerLines}}<path class="series rss" d="${{memory}}"/><path class="series pss" d="${{pss}}"/><path class="series cpu" d="${{cpu}}"/>
<text x="4" y="${{y0+5}}" fill="#91a3b5" font-size="11">${{esc(mib(memoryMax))}}</text>
<text x="4" y="${{y0+height}}" fill="#91a3b5" font-size="11">0</text>
<text x="${{x0+width-68}}" y="${{y0+height+18}}" fill="#91a3b5" font-size="11">${{num(span,1)}} s</text>
</svg></section>`;
}}
document.getElementById('charts').innerHTML = Object.entries(data.resources || {{}}).map(([target,p])=>timeline(target,p)).join('');
const failures = data.episodes.filter(row=>!row.ok).map(row=>[
esc(row.target),esc(row.episode),esc(row.run),esc(row.status),esc(row.failure_step),
`<div class="error">${{esc(row.error)}}</div>`,row.failure_artifact?`<a href="${{esc(row.failure_artifact)}}">artifact</a>`:'—'
]);
document.getElementById('failures').innerHTML = table(['Target','Episode','Run','Status','Step','Error','Details'],failures);
const episodeRows = data.episodes.map(row=>[
esc(row.target),esc(row.worker),esc(row.run),esc(row.episode),
`<span class="${{row.ok?'good':'bad'}}">${{esc(row.status)}}</span>`,
ms(row.elapsed_ms),ms(row.active_elapsed_ms),esc(row.step_count),esc(row.final_url)
]);
document.getElementById('episodes').innerHTML = table(['Target','Worker','Run','Episode','Status','Wall','Active','Steps','Final URL'],episodeRows);
</script>
</main></body></html>
"""
moli-benchmark/moli_benchmark/amiibo_crawler.py
+771
View File
@@ -0,0 +1,771 @@
from __future__ import annotations
import asyncio
import contextlib
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.parse import urldefrag, urlparse
from .artifacts import write_csv, write_json
from .stats import summarize
from .synthetic_compare import TARGETS, normalize_cdp_target, target_metadata
from .target_serve import start_target_serve, stop_target_serve
AMIIBO_URL = "https://demo-browser.lightpanda.io/amiibo/"
AMIIBO_EXPECTED_PAGES = 933
AMIIBO_CONCURRENCY_MATRIX = (1, 2, 5, 10, 25, 100)
AMIIBO_SMOKE_CONCURRENCY_MATRIX = (1,)
AMIIBO_SMOKE_LIMIT = 5
DEFAULT_AMIIBO_RUNS = 1
AMIIBO_MODES = ("session", "process")
AMIIBO_PROFILES = ("smoke", "formal")
@dataclass
class PageSession:
client: Any
session_id: str
def _expected_pages(limit: int) -> int:
return limit if limit > 0 else AMIIBO_EXPECTED_PAGES
def _normalize_url(url: str) -> str:
return urldefrag(url)[0]
def _same_origin(seed: str, candidate: str) -> bool:
seed_url = urlparse(seed)
candidate_url = urlparse(candidate)
return candidate_url.scheme in ("http", "https") and candidate_url.netloc == seed_url.netloc
def _classify_amiibo_error(error: str | None) -> str | None:
if error is None:
return None
lower = error.lower()
if "target binary unavailable" in lower:
return "target-unavailable"
if "timeout" in lower or "timed out" in lower:
return "timeout"
if "navigate" in lower or "navigation" in lower:
return "navigation-error"
if "evaluate" in lower or "script" in lower or "javascript" in lower:
return "script-error"
if "cdp" in lower or "protocol" in lower or "target." in lower or "runtime." in lower or "page." in lower:
return "protocol-error"
if "process" in lower or "crash" in lower or "exited" in lower:
return "process-error"
return "error"
def _page_assertion_failures(page: dict[str, Any]) -> list[str]:
failures: list[str] = []
if not isinstance(page.get("url"), str) or not page["url"]:
failures.append("missing-url")
title = page.get("title")
if not isinstance(title, str) or not title.strip():
failures.append("missing-title")
ready_state = page.get("ready_state")
if ready_state != "complete":
failures.append("document-not-complete")
text_length = page.get("text_length")
if not isinstance(text_length, int | float) or text_length <= 0:
failures.append("missing-body-text")
link_count = page.get("link_count")
if not isinstance(link_count, int) or link_count < 0:
failures.append("missing-link-count")
fields = page.get("fields")
if not isinstance(fields, dict):
failures.append("missing-fields")
return failures
name = fields.get("name")
if not isinstance(name, str) or not name.strip() or name == "Amiibo Character":
failures.append("missing-amiibo-name")
if isinstance(title, str) and isinstance(name, str) and title.strip() and name.strip() and title.strip() != name.strip():
failures.append("title-name-mismatch")
game = fields.get("game")
if not isinstance(game, str) or not game.strip() or game == "Amiibo Game":
failures.append("missing-amiibo-series")
serie = fields.get("serie")
if not isinstance(serie, str) or not serie.strip() or serie == "Amiibo Serie":
failures.append("missing-game-series")
image_src = fields.get("imageSrc")
if not isinstance(image_src, str) or not image_src.strip():
failures.append("missing-image-src")
alt_count = fields.get("altCount")
if not isinstance(alt_count, int) or alt_count < 0:
failures.append("missing-alt-count")
return failures
def _collect_page_assertion_failures(pages: list[dict[str, Any]]) -> list[dict[str, Any]]:
failures: list[dict[str, Any]] = []
for page in pages:
page_failures = _page_assertion_failures(page)
if page_failures:
failures.append(
{
"url": page.get("url"),
"worker": page.get("worker"),
"failures": page_failures,
}
)
return failures
def _row_failure_kind(
*,
ok: bool,
pages: int | None,
expected_pages: int,
crawler_errors: int,
assertion_failures: int,
error: str | None = None,
) -> str | None:
if ok:
return None
classified = _classify_amiibo_error(error)
if classified is not None:
return classified
if crawler_errors:
return "crawler-error"
if assertion_failures:
return "assertion-failure"
if pages is not None and pages != expected_pages:
return "page-count-mismatch"
return "error"
def _row_error_message(
*,
ok: bool,
pages: int,
expected_pages: int,
crawler_errors: int,
assertion_failures: int,
) -> str | None:
if ok:
return None
if crawler_errors:
return "crawler recorded navigation/protocol errors"
if assertion_failures:
return "crawler page assertions failed"
if pages != expected_pages:
return f"crawler discovered {pages} of {expected_pages} expected pages"
return "crawler did not complete expected page count without errors"
async def _create_page_session(endpoint: str, timeout_seconds: float) -> PageSession:
from .raw_cdp import connect_raw_cdp
client = await connect_raw_cdp(endpoint)
try:
target_command_id = await client.send("Target.createTarget", {"url": "about:blank"})
target_response, _ = await client.recv_until_id(target_command_id, timeout=timeout_seconds)
target_id = target_response.get("result", {}).get("targetId")
if not isinstance(target_id, str) or not target_id:
raise RuntimeError(f"missing targetId in {target_response}")
attach_id = await client.send("Target.attachToTarget", {"targetId": target_id, "flatten": True})
attach_response, _ = await client.recv_until_id(attach_id, timeout=timeout_seconds)
session_id = attach_response.get("result", {}).get("sessionId")
if not isinstance(session_id, str) or not session_id:
raise RuntimeError(f"missing sessionId in {attach_response}")
for method in ("Page.enable", "Runtime.enable"):
command_id = await client.send(method, session_id=session_id)
await client.recv_until_id(command_id, timeout=timeout_seconds)
return PageSession(client=client, session_id=session_id)
except Exception:
await client.websocket.close()
raise
async def _fetch_links(page: PageSession, url: str, timeout_seconds: float) -> dict[str, Any]:
started = time.perf_counter()
navigate_id = await page.client.send("Page.navigate", {"url": url}, session_id=page.session_id)
await page.client.recv_until_id(navigate_id, timeout=timeout_seconds)
expression = f"""
new Promise(resolve => {{
const deadline = Date.now() + {int(timeout_seconds * 1000)};
const text = selector => {{
const element = document.querySelector(selector);
return element ? element.textContent.trim() : "";
}};
function collect() {{
const anchors = Array.from(document.querySelectorAll('a[href]')).map(anchor => anchor.href);
if ((document.readyState === 'complete' && anchors.length > 0) || Date.now() > deadline) {{
const image = document.querySelector('#image');
resolve({{
readyState: document.readyState,
title: document.title,
textLength: document.body ? document.body.textContent.length : 0,
links: anchors,
fields: {{
name: text('#name'),
game: text('#game'),
serie: text('#serie'),
imageSrc: image ? image.getAttribute('src') || "" : "",
navText: text('#nav'),
altCount: document.querySelectorAll('#alt a[href]').length
}}
}});
}} else {{
setTimeout(collect, 10);
}}
}}
collect();
}})
"""
evaluate_id = await page.client.send(
"Runtime.evaluate",
{"expression": expression, "awaitPromise": True, "returnByValue": True},
session_id=page.session_id,
)
response, _ = await page.client.recv_until_id(evaluate_id, timeout=timeout_seconds + 1)
value = response.get("result", {}).get("result", {}).get("value")
if not isinstance(value, dict):
raise RuntimeError(f"unexpected crawler evaluate result for {url}: {response}")
links = value.get("links")
if not isinstance(links, list):
links = []
fields = value.get("fields")
if not isinstance(fields, dict):
fields = {}
return {
"url": url,
"elapsed_ms": (time.perf_counter() - started) * 1000.0,
"ready_state": value.get("readyState"),
"title": value.get("title"),
"text_length": value.get("textLength"),
"links": [str(link) for link in links],
"fields": fields,
}
async def _crawl_with_endpoint(
*,
endpoint: str,
pool: int,
limit: int,
url: str,
timeout_seconds: float,
) -> dict[str, Any]:
sessions: list[tuple[int, PageSession]] = []
errors: list[dict[str, Any]] = []
try:
for worker_id in range(pool):
try:
sessions.append((worker_id, await _create_page_session(endpoint, timeout_seconds)))
except Exception as error:
errors.append({"worker": worker_id, "url": None, "stage": "create-session", "error": str(error)})
return await _crawl_with_page_sessions(
sessions=sessions,
initial_errors=errors,
limit=limit,
url=url,
timeout_seconds=timeout_seconds,
)
except BaseException:
await _close_page_sessions(sessions)
raise
async def _close_page_sessions(sessions: list[tuple[int, PageSession]]) -> None:
for _, page in sessions:
with contextlib.suppress(Exception):
await page.client.websocket.close()
async def _crawl_with_page_sessions(
*,
sessions: list[tuple[int, PageSession]],
initial_errors: list[dict[str, Any]],
limit: int,
url: str,
timeout_seconds: float,
) -> dict[str, Any]:
expected = _expected_pages(limit)
queue: asyncio.Queue[str] = asyncio.Queue()
known = {_normalize_url(url)}
pages: list[dict[str, Any]] = []
errors = list(initial_errors)
lock = asyncio.Lock()
await queue.put(_normalize_url(url))
workers: list[asyncio.Task[None]] = []
async def worker(worker_id: int, page: PageSession) -> None:
try:
while True:
current = await queue.get()
try:
result = await _fetch_links(page, current, timeout_seconds)
async with lock:
pages.append({k: v for k, v in result.items() if k != "links"} | {"worker": worker_id, "link_count": len(result["links"])})
for link in result["links"]:
if len(known) >= expected:
break
normalized = _normalize_url(link)
if not _same_origin(url, normalized) or normalized in known:
continue
known.add(normalized)
await queue.put(normalized)
except Exception as error:
async with lock:
errors.append({"worker": worker_id, "url": current, "error": str(error)})
finally:
queue.task_done()
except asyncio.CancelledError:
raise
finally:
await page.client.websocket.close()
try:
if not sessions:
return {
"pages": pages,
"errors": errors,
"known_count": len(known),
"expected_pages": expected,
}
workers = [asyncio.create_task(worker(worker_id, page)) for worker_id, page in sessions]
await queue.join()
finally:
for worker_task in workers:
worker_task.cancel()
if workers:
await asyncio.gather(*workers, return_exceptions=True)
else:
await _close_page_sessions(sessions)
return {
"pages": pages,
"errors": errors,
"known_count": len(known),
"expected_pages": expected,
}
def _summarize_serve_resources(serve_details: list[dict[str, Any]]) -> dict[str, Any]:
resources = [detail.get("resources", {}) for detail in serve_details]
def sum_field(name: str) -> int | float | None:
values = [resource.get(name) for resource in resources if resource.get(name) is not None]
return sum(values) if values else None
def max_field(name: str) -> int | float | None:
values = [resource.get(name) for resource in resources if resource.get(name) is not None]
return max(values) if values else None
return {
"peak_cpu_percent": sum_field("peak_cpu_percent"),
"peak_fd_count": sum_field("peak_fd_count"),
"peak_process_count": sum_field("peak_process_count"),
"peak_pss_bytes": sum_field("peak_pss_bytes"),
"peak_rss_bytes": sum_field("peak_rss_bytes"),
"peak_thread_count": sum_field("peak_thread_count"),
"sample_count": sum_field("sample_count"),
"sampling_method": "aggregate_process_tree",
"worker_count": len(serve_details),
"max_worker_peak_pss_bytes": max_field("peak_pss_bytes"),
}
def _summarize_rows(rows: list[dict[str, Any]]) -> dict[str, Any]:
failure_kinds: dict[str, int] = {}
for row in rows:
failure_kind = row.get("failure_kind")
if isinstance(failure_kind, str) and failure_kind:
failure_kinds[failure_kind] = failure_kinds.get(failure_kind, 0) + 1
return {
"runs": len(rows),
"passes": sum(1 for row in rows if row.get("ok")),
"failures": sum(1 for row in rows if not row.get("ok")),
"failure_kinds": failure_kinds,
"assertion_failures": sum(int(row.get("assertion_failures") or 0) for row in rows),
"elapsed_ms": summarize(row.get("elapsed_ms") for row in rows if row.get("ok") and row.get("elapsed_ms") is not None),
"browser_peak_pss_bytes": summarize(
row.get("browser_peak_pss_bytes")
for row in rows
if row.get("browser_peak_pss_bytes") is not None
),
}
def _row_from_crawl_result(
*,
target: str,
mode: str,
pool: int,
limit: int,
url: str,
elapsed_ms: float,
result: dict[str, Any],
serve_details: dict[str, Any],
) -> tuple[dict[str, Any], dict[str, Any]]:
pages = int(result["known_count"])
expected = _expected_pages(limit)
crawler_errors = len(result["errors"])
assertion_failures = _collect_page_assertion_failures(result["pages"])
ok = pages == expected and not crawler_errors and not assertion_failures
resources = serve_details.get("resources", {})
row = {
"target": target,
"mode": mode,
"pool": pool,
"limit": limit,
"url": url,
"ok": ok,
"elapsed_ms": elapsed_ms,
"pages": pages,
"expected_pages": expected,
"errors": crawler_errors,
"assertion_failures": len(assertion_failures),
"browser_peak_pss_bytes": resources.get("peak_pss_bytes"),
"browser_peak_cpu_percent": resources.get("peak_cpu_percent"),
"error": _row_error_message(
ok=ok,
pages=pages,
expected_pages=expected,
crawler_errors=crawler_errors,
assertion_failures=len(assertion_failures),
),
}
row["failure_kind"] = _row_failure_kind(
ok=ok,
pages=pages,
expected_pages=expected,
crawler_errors=crawler_errors,
assertion_failures=len(assertion_failures),
)
return row, {
**row,
"pages_detail": result["pages"],
"errors_detail": result["errors"],
"assertion_failures_detail": assertion_failures,
"serve": serve_details,
}
def _run_session_mode(
*,
target: str,
binary: Path,
pool: int,
limit: int,
url: str,
timeout_seconds: float,
) -> tuple[dict[str, Any], dict[str, Any]]:
serve = None
started = time.perf_counter()
try:
serve = start_target_serve(target, binary, timeout_seconds)
result = asyncio.run(
asyncio.wait_for(
_crawl_with_endpoint(
endpoint=serve.endpoint,
pool=pool,
limit=limit,
url=url,
timeout_seconds=timeout_seconds,
),
timeout=timeout_seconds,
)
)
elapsed_ms = (time.perf_counter() - started) * 1000.0
serve_details = stop_target_serve(serve)
return _row_from_crawl_result(
target=target,
mode="session",
pool=pool,
limit=limit,
url=url,
elapsed_ms=elapsed_ms,
result=result,
serve_details=serve_details,
)
except Exception as error:
serve_details = stop_target_serve(serve)
elapsed_ms = (time.perf_counter() - started) * 1000.0
if isinstance(error, TimeoutError):
error_message = f"crawler run exceeded {timeout_seconds:.1f}s wall timeout"
else:
error_message = str(error)
row = {
"target": target,
"mode": "session",
"pool": pool,
"limit": limit,
"url": url,
"ok": False,
"elapsed_ms": elapsed_ms,
"pages": None,
"expected_pages": _expected_pages(limit),
"errors": 0,
"assertion_failures": 0,
"browser_peak_pss_bytes": serve_details.get("resources", {}).get("peak_pss_bytes"),
"browser_peak_cpu_percent": serve_details.get("resources", {}).get("peak_cpu_percent"),
"error": error_message,
"failure_kind": _row_failure_kind(
ok=False,
pages=None,
expected_pages=_expected_pages(limit),
crawler_errors=0,
assertion_failures=0,
error=error_message,
),
}
return row, {**row, "serve": serve_details}
def _run_process_mode(
*,
target: str,
binary: Path,
pool: int,
limit: int,
url: str,
timeout_seconds: float,
) -> tuple[dict[str, Any], dict[str, Any]]:
serves: list[Any] = []
started = time.perf_counter()
async def run_crawl() -> dict[str, Any]:
sessions: list[tuple[int, PageSession]] = []
errors: list[dict[str, Any]] = []
try:
for worker_id in range(pool):
try:
serve = start_target_serve(target, binary, timeout_seconds)
serves.append(serve)
sessions.append((worker_id, await _create_page_session(serve.endpoint, timeout_seconds)))
except Exception as error:
errors.append({"worker": worker_id, "url": None, "stage": "create-process-session", "error": str(error)})
return await _crawl_with_page_sessions(
sessions=sessions,
initial_errors=errors,
limit=limit,
url=url,
timeout_seconds=timeout_seconds,
)
except BaseException:
await _close_page_sessions(sessions)
raise
try:
result = asyncio.run(asyncio.wait_for(run_crawl(), timeout=timeout_seconds))
elapsed_ms = (time.perf_counter() - started) * 1000.0
stopped = [stop_target_serve(serve) for serve in serves]
serves = []
serve_details = {
"mode": "process",
"workers": stopped,
"resources": _summarize_serve_resources(stopped),
}
return _row_from_crawl_result(
target=target,
mode="process",
pool=pool,
limit=limit,
url=url,
elapsed_ms=elapsed_ms,
result=result,
serve_details=serve_details,
)
except Exception as error:
stopped = [stop_target_serve(serve) for serve in serves]
serves = []
elapsed_ms = (time.perf_counter() - started) * 1000.0
if isinstance(error, TimeoutError):
error_message = f"crawler run exceeded {timeout_seconds:.1f}s wall timeout"
else:
error_message = str(error)
row = {
"target": target,
"mode": "process",
"pool": pool,
"limit": limit,
"url": url,
"ok": False,
"elapsed_ms": elapsed_ms,
"pages": None,
"expected_pages": _expected_pages(limit),
"errors": 0,
"assertion_failures": 0,
"browser_peak_pss_bytes": None,
"browser_peak_cpu_percent": None,
"error": error_message,
"failure_kind": _row_failure_kind(
ok=False,
pages=None,
expected_pages=_expected_pages(limit),
crawler_errors=0,
assertion_failures=0,
error=error_message,
),
}
return row, {
**row,
"serve": {
"mode": "process",
"workers": stopped,
"resources": _summarize_serve_resources(stopped),
},
}
def run_amiibo_crawler_suite(
*,
output_dir: Path,
target_matrix: dict[str, Any],
profile: str,
targets: tuple[str, ...],
pools: tuple[int, ...],
modes: tuple[str, ...],
runs: int,
limit: int,
timeout_seconds: float,
gate_target: str,
url: str = AMIIBO_URL,
) -> dict[str, Any]:
if profile not in AMIIBO_PROFILES:
raise RuntimeError(f"unknown Amiibo crawler profile: {profile}")
targets = tuple(dict.fromkeys(normalize_cdp_target(target) for target in targets))
gate_target = normalize_cdp_target(gate_target)
unknown_targets = [target for target in targets if target not in TARGETS]
if unknown_targets:
raise RuntimeError(f"unknown target(s): {', '.join(unknown_targets)}")
unknown_modes = [mode for mode in modes if mode not in AMIIBO_MODES]
if unknown_modes:
raise RuntimeError(f"unknown Amiibo crawler mode(s): {', '.join(unknown_modes)}")
invalid_pools = [pool for pool in pools if pool < 1]
if invalid_pools:
raise RuntimeError(f"invalid Amiibo crawler pool(s): {invalid_pools}")
if limit < 0:
raise RuntimeError("Amiibo crawler limit must be 0 or greater")
if runs < 1:
raise RuntimeError("Amiibo crawler runs must be at least 1")
suite_dir = output_dir / "amiibo-crawler"
rows: list[dict[str, Any]] = []
details: list[dict[str, Any]] = []
for target in targets:
metadata = target_metadata(target)
info = target_matrix.get(metadata["binary_key"], {})
path = info.get("path")
for run_id in range(1, runs + 1):
for pool in pools:
for mode in modes:
if not info.get("available") or not path:
row = {
"target": target,
**metadata,
"mode": mode,
"run": run_id,
"pool": pool,
"limit": limit,
"url": url,
"ok": False,
"elapsed_ms": None,
"pages": None,
"expected_pages": _expected_pages(limit),
"errors": 0,
"assertion_failures": 0,
"error": "target binary unavailable",
"failure_kind": "target-unavailable",
}
rows.append(row)
details.append(row)
continue
if mode == "session":
row, detail = _run_session_mode(
target=target,
binary=Path(path),
pool=pool,
limit=limit,
url=url,
timeout_seconds=timeout_seconds,
)
elif mode == "process":
row, detail = _run_process_mode(
target=target,
binary=Path(path),
pool=pool,
limit=limit,
url=url,
timeout_seconds=timeout_seconds,
)
else:
raise RuntimeError(f"unknown Amiibo crawler mode: {mode}")
row["run"] = run_id
row.update(metadata)
detail["run"] = run_id
detail.update(metadata)
rows.append(row)
details.append(detail)
gate_failures = sum(1 for row in rows if row.get("target") == gate_target and not row.get("ok"))
required_pools = set(AMIIBO_CONCURRENCY_MATRIX)
selected_pools = set(pools)
required_modes = set(AMIIBO_MODES)
selected_modes = set(modes)
formal_requirements = {
"runs": {"actual": runs, "required": DEFAULT_AMIIBO_RUNS, "ok": runs >= DEFAULT_AMIIBO_RUNS},
"limit": {"actual": limit, "required": 0, "ok": limit == 0},
"expected_pages": {"actual": _expected_pages(limit), "required": AMIIBO_EXPECTED_PAGES, "ok": _expected_pages(limit) >= AMIIBO_EXPECTED_PAGES},
"pool": {
"actual": list(pools),
"required": list(AMIIBO_CONCURRENCY_MATRIX),
"ok": required_pools.issubset(selected_pools),
},
"modes": {
"actual": list(modes),
"required": list(AMIIBO_MODES),
"ok": required_modes.issubset(selected_modes),
},
}
profile_failures = (
sum(1 for requirement in formal_requirements.values() if not bool(requirement["ok"]))
if profile == "formal"
else 0
)
summary: dict[str, Any] = {
"suite": "amiibo-crawler",
"profile": profile,
"runs": runs,
"url": url,
"limit": limit,
"expected_pages": _expected_pages(limit),
"pool": list(pools),
"modes": list(modes),
"timeout_seconds": timeout_seconds,
"gate_target": gate_target,
"gate_failures": gate_failures + profile_failures,
"profile_failures": profile_failures,
"formal_requirements": formal_requirements,
"targets": {},
"total_failures": sum(1 for row in rows if not row.get("ok")),
}
for target in targets:
target_rows = [row for row in rows if row.get("target") == target]
target_summary = _summarize_rows(target_rows)
target_summary.update(target_metadata(target))
target_summary["modes"] = {
mode: _summarize_rows([row for row in target_rows if row.get("mode") == mode])
for mode in modes
}
summary["targets"][target] = target_summary
write_csv(suite_dir / "raw-runs.csv", rows)
write_json(suite_dir / "runs.json", details)
write_json(suite_dir / "summary.json", summary)
return summary
moli-benchmark/moli_benchmark/artifacts.py
+38
View File
@@ -0,0 +1,38 @@
from __future__ import annotations
import csv
import json
from pathlib import Path
from typing import Any, Iterable
def ensure_dir(path: Path) -> Path:
path.mkdir(parents=True, exist_ok=True)
return path
def write_json(path: Path, data: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
def write_text(path: Path, data: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(data, encoding="utf-8")
def write_csv(path: Path, rows: Iterable[dict[str, Any]]) -> None:
rows = list(rows)
path.parent.mkdir(parents=True, exist_ok=True)
if not rows:
path.write_text("", encoding="utf-8")
return
fieldnames: list[str] = []
for row in rows:
for key in row.keys():
if key not in fieldnames:
fieldnames.append(key)
with path.open("w", newline="", encoding="utf-8") as file:
writer = csv.DictWriter(file, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
moli-benchmark/moli_benchmark/cdp_session.py
+470
View File
@@ -0,0 +1,470 @@
from __future__ import annotations
import asyncio
import json
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from .artifacts import write_csv, write_json
from .raw_cdp import RawCdpClient, connect_raw_cdp
from .stats import summarize
from .synthetic import SYNTHETIC_CASES, SyntheticServer
from .synthetic_compare import TARGETS, normalize_cdp_target, target_metadata, target_uses_external_fixture
from .target_serve import start_target_serve, stop_target_serve
@dataclass
class PageSession:
session_id: str
browser_context_id: str | None
TRACE_METHODS = {
"Runtime.consoleAPICalled",
"Runtime.exceptionThrown",
"Log.entryAdded",
"Network.requestWillBeSent",
"Network.responseReceived",
"Network.loadingFailed",
}
def _compact_remote_object(value: dict[str, Any]) -> Any:
if "value" in value:
return value.get("value")
if "description" in value:
return value.get("description")
return value.get("type")
def _cdp_trace_events(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
events: list[dict[str, Any]] = []
for message in messages:
method = message.get("method")
if method not in TRACE_METHODS:
continue
params = message.get("params", {})
if not isinstance(params, dict):
params = {}
event: dict[str, Any] = {"method": method}
if method == "Runtime.consoleAPICalled":
event.update(
{
"type": params.get("type"),
"text": " ".join(
str(_compact_remote_object(arg))
for arg in params.get("args", [])
if isinstance(arg, dict)
),
"timestamp": params.get("timestamp"),
}
)
elif method == "Runtime.exceptionThrown":
details = params.get("exceptionDetails", {})
if not isinstance(details, dict):
details = {}
event.update(
{
"text": details.get("text"),
"url": details.get("url"),
"line_number": details.get("lineNumber"),
"column_number": details.get("columnNumber"),
}
)
elif method == "Log.entryAdded":
entry = params.get("entry", {})
if not isinstance(entry, dict):
entry = {}
event.update(
{
"level": entry.get("level"),
"source": entry.get("source"),
"text": entry.get("text"),
"url": entry.get("url"),
}
)
elif method == "Network.requestWillBeSent":
request = params.get("request", {})
if not isinstance(request, dict):
request = {}
event.update(
{
"request_id": params.get("requestId"),
"url": request.get("url"),
"request_method": request.get("method"),
"type": params.get("type"),
}
)
elif method == "Network.responseReceived":
response = params.get("response", {})
if not isinstance(response, dict):
response = {}
event.update(
{
"request_id": params.get("requestId"),
"url": response.get("url"),
"status": response.get("status"),
"mime_type": response.get("mimeType"),
"type": params.get("type"),
}
)
elif method == "Network.loadingFailed":
event.update(
{
"request_id": params.get("requestId"),
"type": params.get("type"),
"error_text": params.get("errorText"),
"canceled": params.get("canceled"),
}
)
events.append(event)
return events
def _trace_summary(events: list[dict[str, Any]]) -> dict[str, int]:
console_errors = sum(
1
for event in events
if event.get("method") == "Runtime.consoleAPICalled" and event.get("type") in {"error", "assert"}
)
log_errors = sum(
1
for event in events
if event.get("method") == "Log.entryAdded" and event.get("level") == "error"
)
return {
"console_errors": console_errors + log_errors,
"js_exceptions": sum(1 for event in events if event.get("method") == "Runtime.exceptionThrown"),
"network_failures": sum(1 for event in events if event.get("method") == "Network.loadingFailed"),
}
def _benchmark_marker_expression(*, url: str, case: str) -> str:
expected_url = json.dumps(url)
expected_case = json.dumps(case)
return f"""
(function() {{
const node = document.querySelector('[data-benchmark-case]');
return document.readyState === 'complete'
&& location.href === {expected_url}
&& node !== null
&& node.getAttribute('data-benchmark-case') === {expected_case}
&& document.querySelector('[data-benchmark-status="ok"]') !== null;
}})()
"""
def _benchmark_marker_wait_expression(*, url: str, case: str, timeout_seconds: float) -> str:
marker_expression = _benchmark_marker_expression(url=url, case=case)
return f"""
new Promise(resolve => {{
const deadline = Date.now() + {int(timeout_seconds * 1000)};
function tick() {{
if ({marker_expression}) {{
resolve(true);
}} else if (Date.now() > deadline) {{
resolve(false);
}} else {{
setTimeout(tick, 10);
}}
}}
tick();
}})
"""
def _write_trace_artifact(*, suite_dir: Path, row: dict[str, Any], events: list[dict[str, Any]]) -> str:
trace_dir = suite_dir / "traces"
name = f"{row['target']}-run-{row['run']}-{row['case']}.json"
path = trace_dir / name
write_json(path, {"row": row, "events": events, "summary": _trace_summary(events)})
return str(path.relative_to(suite_dir))
async def _create_page_session(client: RawCdpClient) -> PageSession:
browser_context_id = None
try:
context_id = await client.send("Target.createBrowserContext")
context, _ = await client.recv_until_id(context_id)
value = context.get("result", {}).get("browserContextId")
if isinstance(value, str) and value:
browser_context_id = value
except Exception:
browser_context_id = None
params: dict[str, Any] = {"url": "about:blank"}
if browser_context_id:
params["browserContextId"] = browser_context_id
target_command_id = await client.send("Target.createTarget", params)
target_response, _ = await client.recv_until_id(target_command_id)
target_id = target_response.get("result", {}).get("targetId")
if not isinstance(target_id, str) or not target_id:
raise RuntimeError(f"missing targetId in {target_response}")
attach_id = await client.send("Target.attachToTarget", {"targetId": target_id, "flatten": True})
attach_response, _ = await client.recv_until_id(attach_id)
session_id = attach_response.get("result", {}).get("sessionId")
if not isinstance(session_id, str) or not session_id:
raise RuntimeError(f"missing sessionId in {attach_response}")
for method in ("Runtime.enable", "Page.enable"):
command_id = await client.send(method, session_id=session_id)
await client.recv_until_id(command_id)
for method in ("Network.enable", "Log.enable"):
try:
command_id = await client.send(method, session_id=session_id)
await client.recv_until_id(command_id, timeout=3)
except Exception:
pass
return PageSession(session_id=session_id, browser_context_id=browser_context_id)
async def _navigate_and_wait_marker(
client: RawCdpClient,
session_id: str,
url: str,
case: str,
timeout_seconds: float,
target: str,
) -> tuple[bool, float, int, list[dict[str, Any]]]:
started = time.perf_counter()
navigate_id = await client.send("Page.navigate", {"url": url}, session_id=session_id)
_, navigate_seen = await client.recv_until_id(navigate_id, timeout=timeout_seconds)
if target_metadata(target)["engine"] == "obscura":
seen = list(navigate_seen)
deadline = time.perf_counter() + timeout_seconds
ok = False
while time.perf_counter() < deadline:
evaluate_id = await client.send(
"Runtime.evaluate",
{
"expression": _benchmark_marker_expression(url=url, case=case),
"returnByValue": True,
},
session_id=session_id,
)
response, evaluate_seen = await client.recv_until_id(evaluate_id, timeout=3)
seen.extend(evaluate_seen)
if response.get("result", {}).get("result", {}).get("value") is True:
ok = True
break
await asyncio.sleep(0.01)
elapsed_ms = (time.perf_counter() - started) * 1000.0
return ok, elapsed_ms, len(seen), _cdp_trace_events(seen)
evaluate_id = await client.send(
"Runtime.evaluate",
{"expression": _benchmark_marker_wait_expression(url=url, case=case, timeout_seconds=timeout_seconds), "awaitPromise": True, "returnByValue": True},
session_id=session_id,
)
response, evaluate_seen = await client.recv_until_id(evaluate_id, timeout=timeout_seconds + 1)
elapsed_ms = (time.perf_counter() - started) * 1000.0
ok = response.get("result", {}).get("result", {}).get("value") is True
seen = navigate_seen + evaluate_seen
return ok, elapsed_ms, len(seen), _cdp_trace_events(seen)
async def _run_target_session(
*,
suite_dir: Path,
target: str,
binary: Path,
base_url: SyntheticServer,
cases: tuple[str, ...],
runs: int,
timeout_seconds: float,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
serve = None
rows: list[dict[str, Any]] = []
if target_uses_external_fixture(target) and base_url.external_base_url is None:
for run_id in range(1, runs + 1):
for case in cases:
rows.append(
{
"target": target,
**target_metadata(target),
"case": case,
"run": run_id,
"ok": False,
"elapsed_ms": None,
"messages": 0,
"trace_events": 0,
"console_errors": 0,
"js_exceptions": 0,
"network_failures": 0,
"error": "external fixture address unavailable for Obscura",
}
)
return rows, {}
try:
serve = start_target_serve(target, binary, timeout_seconds)
client = await connect_raw_cdp(serve.endpoint)
try:
page = await _create_page_session(client)
for run_id in range(1, runs + 1):
for case in cases:
ok, elapsed_ms, messages, trace_events = await _navigate_and_wait_marker(
client,
page.session_id,
base_url.url_for_path(case, external=target_uses_external_fixture(target)),
case,
timeout_seconds,
target,
)
trace_counts = _trace_summary(trace_events)
row = {
"target": target,
**target_metadata(target),
"case": case,
"run": run_id,
"ok": ok,
"elapsed_ms": elapsed_ms,
"messages": messages,
"trace_events": len(trace_events),
"console_errors": trace_counts["console_errors"],
"js_exceptions": trace_counts["js_exceptions"],
"network_failures": trace_counts["network_failures"],
"error": None if ok else "marker did not become true",
}
if not ok or any(trace_counts.values()):
row["trace_artifact"] = _write_trace_artifact(
suite_dir=suite_dir,
row=row,
events=trace_events,
)
rows.append(row)
if page.browser_context_id:
try:
dispose_id = await client.send("Target.disposeBrowserContext", {"browserContextId": page.browser_context_id})
await client.recv_until_id(dispose_id, timeout=3)
except Exception:
pass
finally:
await client.websocket.close()
except Exception as error:
for run_id in range(1, runs + 1):
for case in cases:
rows.append(
{
"target": target,
**target_metadata(target),
"case": case,
"run": run_id,
"ok": False,
"elapsed_ms": None,
"messages": 0,
"trace_events": 0,
"console_errors": 0,
"js_exceptions": 0,
"network_failures": 0,
"error": str(error),
}
)
finally:
serve_details = stop_target_serve(serve)
return rows, serve_details
def run_cdp_session_suite(
*,
output_dir: Path,
target_matrix: dict[str, Any],
targets: tuple[str, ...],
cases: tuple[str, ...],
runs: int,
timeout_seconds: float,
gate_target: str,
) -> dict[str, Any]:
targets = tuple(dict.fromkeys(normalize_cdp_target(target) for target in targets))
gate_target = normalize_cdp_target(gate_target)
unknown_targets = [target for target in targets if target not in TARGETS]
if unknown_targets:
raise RuntimeError(f"unknown target(s): {', '.join(unknown_targets)}")
if gate_target not in targets:
raise RuntimeError(f"gate target `{gate_target}` must be included in selected targets")
unknown_cases = [case for case in cases if case not in SYNTHETIC_CASES]
if unknown_cases:
raise RuntimeError(f"unknown synthetic case(s): {', '.join(unknown_cases)}")
suite_dir = output_dir / "cdp-session"
rows: list[dict[str, Any]] = []
serve_details: dict[str, Any] = {}
with SyntheticServer() as server:
for target in targets:
metadata = target_metadata(target)
info = target_matrix.get(metadata["binary_key"], {})
path = info.get("path")
if not info.get("available") or not path:
rows.extend(
{
"target": target,
**metadata,
"case": case,
"run": run_id,
"ok": False,
"elapsed_ms": None,
"messages": 0,
"trace_events": 0,
"console_errors": 0,
"js_exceptions": 0,
"network_failures": 0,
"error": "target binary unavailable",
}
for run_id in range(1, runs + 1)
for case in cases
)
continue
target_rows, target_serve = asyncio.run(
_run_target_session(
suite_dir=suite_dir,
target=target,
binary=Path(path),
base_url=server,
cases=cases,
runs=runs,
timeout_seconds=timeout_seconds,
)
)
rows.extend(target_rows)
serve_details[target] = target_serve
gate_failures = sum(1 for row in rows if row["target"] == gate_target and not row.get("ok"))
summary: dict[str, Any] = {
"suite": "cdp-session",
"runs": runs,
"timeout_seconds": timeout_seconds,
"gate_target": gate_target,
"gate_failures": gate_failures,
"cases": list(cases),
"targets": {},
"total_failures": sum(1 for row in rows if not row.get("ok")),
"total_trace_events": sum(1 for row in rows if row.get("trace_artifact")),
}
for target in targets:
target_rows = [row for row in rows if row["target"] == target]
target_summary = {
**target_metadata(target),
"failures": sum(1 for row in target_rows if not row.get("ok")),
"cases": {},
}
for case in cases:
case_rows = [row for row in target_rows if row["case"] == case]
target_summary["cases"][case] = {
"elapsed_ms": summarize(row["elapsed_ms"] for row in case_rows if row.get("ok") and row.get("elapsed_ms") is not None),
"failures": sum(1 for row in case_rows if not row.get("ok")),
"console_errors": sum(int(row.get("console_errors", 0) or 0) for row in case_rows),
"js_exceptions": sum(int(row.get("js_exceptions", 0) or 0) for row in case_rows),
"network_failures": sum(int(row.get("network_failures", 0) or 0) for row in case_rows),
}
target_summary["console_errors"] = sum(int(row.get("console_errors", 0) or 0) for row in target_rows)
target_summary["js_exceptions"] = sum(int(row.get("js_exceptions", 0) or 0) for row in target_rows)
target_summary["network_failures"] = sum(int(row.get("network_failures", 0) or 0) for row in target_rows)
summary["targets"][target] = target_summary
write_csv(suite_dir / "runs.csv", rows)
write_json(suite_dir / "runs.json", rows)
write_json(suite_dir / "serve.json", serve_details)
write_json(suite_dir / "summary.json", summary)
return summary
moli-benchmark/moli_benchmark/cdp_smoke.py
+349
View File
@@ -0,0 +1,349 @@
from __future__ import annotations
import json
import os
import shutil
from pathlib import Path
from typing import Any
from .artifacts import write_json, write_text
from .config import REPO_ROOT, clear_proxy_env
from .process import run_process
CDP_SMOKE_PROFILES = ("smoke", "formal")
CDP_CLIENTS = ("raw_cdp", "playwright", "puppeteer")
PUPPETEER_MODULE_ENV = "PUPPETEER_CORE_MODULE"
def _extract_json_payload(stdout: bytes) -> dict[str, Any] | None:
text = stdout.decode("utf-8", errors="replace").strip()
if not text:
return None
try:
return json.loads(text)
except json.JSONDecodeError:
start = text.find("{")
end = text.rfind("}")
if start >= 0 and end > start:
return json.loads(text[start : end + 1])
return None
def _default_command() -> list[str]:
if shutil.which("uv"):
return ["uv", "run", "moli-cdp-smoke"]
return ["python3", "-m", "moli_cdp_smoke"]
def _discover_group_listing(smoke_command: list[str], timeout_seconds: float, env: dict[str, str]) -> list[dict[str, Any]]:
result = run_process(
[*smoke_command, "--list-groups"],
cwd=REPO_ROOT / "moli-cdp-smoke",
timeout_seconds=min(timeout_seconds, 30.0),
env=env,
sample_resources=False,
)
payload = _extract_json_payload(result.stdout)
if payload is None and result.stderr:
payload = _extract_json_payload(result.stderr)
groups = payload.get("groups") if isinstance(payload, dict) else None
if result.returncode != 0 or result.timed_out or not isinstance(groups, list):
stderr = result.stderr[-2048:].decode("utf-8", errors="replace")
raise RuntimeError(f"failed to list CDP smoke groups: returncode={result.returncode} timed_out={result.timed_out} {stderr}")
return [group for group in groups if isinstance(group, dict)]
def _group_client(group: dict[str, Any]) -> str | None:
name = str(group.get("name", ""))
phase = str(group.get("phase", ""))
if phase == "raw":
return "raw_cdp"
if phase in {"page", "browser"}:
return "playwright"
if phase == "external" and name == "puppeteer":
return "puppeteer"
return None
def _effective_cdp_smoke_groups(
profile: str,
groups: tuple[str, ...],
group_listing: list[dict[str, Any]],
) -> tuple[str, ...]:
if groups:
return groups
selected: list[str] = []
if profile == "formal":
for wanted_client in CDP_CLIENTS:
for group in group_listing:
name = group.get("name")
if isinstance(name, str) and name and _group_client(group) == wanted_client:
selected.append(name)
return tuple(selected)
for group in group_listing:
name = group.get("name")
if isinstance(name, str) and name and group.get("default") is True:
selected.append(name)
return tuple(selected)
def _selected_groups_by_client(
selected_groups: tuple[str, ...],
group_listing: list[dict[str, Any]],
) -> dict[str, list[str]]:
by_name = {str(group.get("name")): group for group in group_listing if group.get("name") is not None}
grouped = {client: [] for client in CDP_CLIENTS}
for group_name in selected_groups:
client = _group_client(by_name.get(group_name, {"name": group_name}))
if client in grouped:
grouped[client].append(group_name)
return grouped
def _record_client(record: Any) -> str | None:
if not isinstance(record, dict):
return None
name = record.get("name")
if not isinstance(name, str):
return None
if name.startswith("raw_cdp_"):
return "raw_cdp"
if name.startswith("puppeteer_"):
return "puppeteer"
return "playwright"
def _records_by_client(records: list[Any]) -> dict[str, list[dict[str, Any]]]:
grouped = {client: [] for client in CDP_CLIENTS}
for record in records:
client = _record_client(record)
if client in grouped and isinstance(record, dict):
grouped[client].append(record)
return grouped
def _executable_available(value: str) -> bool:
if os.sep in value:
return Path(value).exists()
return shutil.which(value) is not None
def _collect_preflight(timeout_seconds: float, env: dict[str, str]) -> dict[str, Any]:
node = env.get("NODE", "node")
node_available = _executable_available(node)
module_name = env.get(PUPPETEER_MODULE_ENV, "puppeteer-core")
puppeteer_core = {
"module": module_name,
"available": False,
"error": None,
}
if node_available:
result = run_process(
[node, "-e", f"require({module_name!r})"],
cwd=REPO_ROOT / "moli-cdp-smoke",
timeout_seconds=min(timeout_seconds, 10.0),
env=env,
sample_resources=False,
)
puppeteer_core["available"] = result.returncode == 0 and not result.timed_out
if not puppeteer_core["available"]:
puppeteer_core["error"] = result.stderr[-2048:].decode("utf-8", errors="replace")
else:
puppeteer_core["error"] = f"executable `{node}` not found"
return {
"node": {"executable": node, "available": node_available},
"puppeteer_core": puppeteer_core,
}
def _client_failure_kind(
*,
client: str,
covered: bool,
selected_groups: list[str],
required: bool,
preflight: dict[str, Any],
process_ok: bool,
timed_out: bool,
) -> str | None:
if covered:
return None
if client == "puppeteer":
node = preflight.get("node", {})
puppeteer_core = preflight.get("puppeteer_core", {})
if not node.get("available") or not puppeteer_core.get("available"):
return "dependency-missing"
if timed_out:
return "timeout"
if selected_groups and not process_ok:
return "process-failed"
if required:
return "no-records"
return "not-selected"
def _client_rows(
*,
profile: str,
selected_groups: tuple[str, ...],
group_listing: list[dict[str, Any]],
records: list[Any],
preflight: dict[str, Any],
process_ok: bool,
timed_out: bool,
) -> list[dict[str, Any]]:
groups_by_client = _selected_groups_by_client(selected_groups, group_listing)
records_by_client = _records_by_client(records)
rows = []
for client in CDP_CLIENTS:
client_records = records_by_client[client]
required = profile == "formal"
covered = bool(client_records)
failure_kind = _client_failure_kind(
client=client,
covered=covered,
selected_groups=groups_by_client[client],
required=required,
preflight=preflight,
process_ok=process_ok,
timed_out=timed_out,
)
rows.append(
{
"client": client,
"required": required,
"covered": covered,
"ok": covered,
"gate_ok": (not required) or covered,
"failure_kind": failure_kind,
"groups": groups_by_client[client],
"record_count": len(client_records),
"records": [record.get("name") for record in client_records if record.get("name")],
}
)
return rows
def _client_coverage(client_rows: list[dict[str, Any]]) -> dict[str, bool]:
return {str(row["client"]): bool(row.get("covered")) for row in client_rows}
def _formal_requirements(client_rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
return {
str(row["client"]): {
"required": bool(row.get("required")),
"ok": bool(row.get("gate_ok")),
"actual": bool(row.get("covered")),
"failure_kind": row.get("failure_kind"),
}
for row in client_rows
}
def _markdown(summary: dict[str, Any]) -> str:
lines = [
"# CDP smoke benchmark",
"",
f"- ok: `{summary['ok']}`",
f"- profile: `{summary['profile']}`",
f"- total records: `{summary['total_records']}`",
f"- elapsed ms: `{summary['elapsed_ms']:.3f}`",
f"- timeout: `{summary['timed_out']}`",
f"- raw CDP coverage: `{summary['client_coverage']['raw_cdp']}`",
f"- Playwright coverage: `{summary['client_coverage']['playwright']}`",
f"- Puppeteer coverage: `{summary['client_coverage']['puppeteer']}`",
"",
]
if summary["groups"]:
lines.append("Groups:")
for group in summary["groups"]:
lines.append(f"- `{group}`")
lines.append("")
if summary["error"]:
lines.append("Error:")
lines.append("")
lines.append("```text")
lines.append(str(summary["error"])[-4000:])
lines.append("```")
lines.append("")
return "\n".join(lines)
def run_cdp_smoke_suite(
*,
output_dir: Path,
moli_bin: Path,
timeout_seconds: float,
groups: tuple[str, ...],
profile: str = "smoke",
command: tuple[str, ...] | None = None,
) -> dict[str, Any]:
if profile not in CDP_SMOKE_PROFILES:
raise RuntimeError(f"unknown CDP smoke profile `{profile}`")
suite_dir = output_dir / "cdp-smoke"
smoke_command = list(command or _default_command())
env = clear_proxy_env(os.environ)
env["MOLI_BIN"] = str(moli_bin)
group_listing = _discover_group_listing(smoke_command, timeout_seconds, env)
preflight = _collect_preflight(timeout_seconds, env)
effective_groups = _effective_cdp_smoke_groups(profile, groups, group_listing)
for group in effective_groups:
smoke_command.extend(["--group", group])
result = run_process(
smoke_command,
cwd=REPO_ROOT / "moli-cdp-smoke",
timeout_seconds=timeout_seconds,
env=env,
)
payload = _extract_json_payload(result.stdout)
if payload is None and result.stderr:
payload = _extract_json_payload(result.stderr)
records = payload.get("results", []) if isinstance(payload, dict) else []
ok = result.returncode == 0 and not result.timed_out and isinstance(payload, dict) and payload.get("ok") is True
client_rows = _client_rows(
profile=profile,
selected_groups=effective_groups,
group_listing=group_listing,
records=records if isinstance(records, list) else [],
preflight=preflight,
process_ok=ok,
timed_out=result.timed_out,
)
coverage = _client_coverage(client_rows)
formal_requirements = _formal_requirements(client_rows)
profile_failures = sum(1 for requirement in formal_requirements.values() if not bool(requirement["ok"]))
total_failures = 0 if ok else 1
error = None if ok else (payload.get("error") if isinstance(payload, dict) else result.stderr[-4096:].decode("utf-8", errors="replace"))
summary = {
"suite": "cdp-smoke",
"ok": ok,
"profile": profile,
"total_failures": total_failures,
"profile_failures": profile_failures,
"gate_failures": total_failures + profile_failures,
"groups": list(effective_groups),
"explicit_groups": bool(groups),
"group_selection_source": "explicit" if groups else "group-listing",
"group_listing": group_listing,
"preflight": preflight,
"client_rows": client_rows,
"client_coverage": coverage,
"formal_requirements": formal_requirements,
"total_records": len(records) if isinstance(records, list) else 0,
"elapsed_ms": result.elapsed_ms,
"timed_out": result.timed_out,
"returncode": result.returncode,
"peak_pss_bytes": result.resources.get("peak_pss_bytes"),
"peak_cpu_percent": result.resources.get("peak_cpu_percent"),
"error": error,
}
write_json(suite_dir / "group-listing.json", {"groups": group_listing})
write_json(suite_dir / "preflight.json", preflight)
write_json(suite_dir / "client-rows.json", {"rows": client_rows})
write_json(suite_dir / "moli-cdp-smoke.json", payload or {})
write_json(suite_dir / "process.json", result.json_summary(include_output=not ok))
write_json(suite_dir / "summary.json", summary)
write_text(suite_dir / "summary.md", _markdown(summary))
return summary
moli-benchmark/moli_benchmark/chrome_dcl.py
+524
View File
@@ -0,0 +1,524 @@
from __future__ import annotations
import asyncio
import os
import signal
import subprocess
import tempfile
import time
from pathlib import Path
from typing import Any
from .config import REPO_ROOT, reserve_port
from .process import ProcessResult
from .raw_cdp import RawCdpClient, RawCdpError, connect_raw_cdp
from .sampling import ResourceSampler
from .target_serve import start_target_serve, stop_target_serve
DEFAULT_CHROME_DCL_USER_AGENT = (
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"
)
# Only Page.navigate uses this grace. It preserves a late protocol error that
# races the benchmark deadline, but a late successful response is still a DCL
# timeout. Keep this small so failed navigations do not distort timing reports.
CDP_LATE_ERROR_GRACE_SECONDS = 2.0
class CdpDclDumpTimeoutError(TimeoutError):
def __init__(self, stage: str, error: BaseException) -> None:
self.stage = stage
self.original_error = error
detail = str(error)
super().__init__(f"{stage}: {detail}" if detail else stage)
@property
def detail(self) -> str:
detail = str(self.original_error)
return detail if detail else self.stage
def _chrome_command(binary: Path, port: int, profile_dir: Path) -> list[str]:
return [
str(binary),
"--headless=new",
"--disable-gpu",
"--no-sandbox",
"--disable-dev-shm-usage",
"--no-first-run",
"--no-default-browser-check",
"--remote-debugging-address=127.0.0.1",
f"--remote-debugging-port={port}",
f"--user-data-dir={profile_dir}",
f"--user-agent={DEFAULT_CHROME_DCL_USER_AGENT}",
"about:blank",
]
async def _wait_for_cdp(endpoint: str, process: subprocess.Popen[bytes], timeout_seconds: float) -> RawCdpClient:
deadline = time.perf_counter() + timeout_seconds
last_error: Exception | None = None
while time.perf_counter() < deadline:
if process.poll() is not None:
raise RawCdpError(f"chromium exited before CDP became available: rc={process.returncode}")
try:
return await connect_raw_cdp(endpoint)
except Exception as error: # noqa: BLE001 - surface the last startup failure in context.
last_error = error
await asyncio.sleep(0.05)
raise TimeoutError(f"timed out waiting for Chrome CDP at {endpoint}; last_error={last_error!r}")
async def _recv_command_response(
client: RawCdpClient,
message_id: int,
*,
deadline: float,
stage: str,
late_error_grace_seconds: float = 0.0,
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
try:
return await client.recv_until_id(
message_id,
timeout=max(0.1, deadline - time.perf_counter()),
)
except TimeoutError as error:
if late_error_grace_seconds > 0.0:
await _raise_late_command_error_or_timeout(
client,
message_id,
stage=stage,
timeout_error=error,
grace_seconds=late_error_grace_seconds,
)
raise CdpDclDumpTimeoutError(stage, error) from error
async def _raise_late_command_error_or_timeout(
client: RawCdpClient,
message_id: int,
*,
stage: str,
timeout_error: BaseException,
grace_seconds: float,
) -> None:
"""Surface a command error that arrives just after the benchmark deadline.
The grace path is only for classification. A late successful response still
remains a timeout, so the DCL benchmark does not accept pages that complete
after its configured deadline.
"""
grace_deadline = time.perf_counter() + grace_seconds
while True:
remaining = grace_deadline - time.perf_counter()
if remaining <= 0.0:
raise CdpDclDumpTimeoutError(stage, timeout_error) from timeout_error
try:
message = await asyncio.wait_for(client.recv(), timeout=remaining)
except TimeoutError:
raise CdpDclDumpTimeoutError(stage, timeout_error) from timeout_error
if message.get("id") != message_id:
continue
if "error" in message:
raise RawCdpError(f"CDP command id={message_id} failed: {message['error']}")
raise CdpDclDumpTimeoutError(stage, timeout_error) from timeout_error
def _is_dcl_event(message: dict[str, Any], session_id: str, frame_id: str | None) -> bool:
if message.get("sessionId") != session_id:
return False
method = message.get("method")
if method == "Page.domContentEventFired":
return True
if method != "Page.lifecycleEvent":
return False
params = message.get("params")
if not isinstance(params, dict):
return False
if frame_id is not None and params.get("frameId") != frame_id:
return False
return params.get("name") in {"DOMContentLoaded", "domContentLoaded"}
_BINARY_DOCUMENT_MIME_PREFIXES = (
"audio/",
"font/",
"image/",
"video/",
)
_BINARY_DOCUMENT_MIME_TYPES = {
"application/gzip",
"application/octet-stream",
"application/pdf",
"application/vnd.ms-fontobject",
"application/x-7z-compressed",
"application/x-bzip2",
"application/x-gzip",
"application/x-rar-compressed",
"application/x-tar",
"application/zip",
}
def _is_binary_document_mime_type(mime_type: str) -> bool:
normalized = mime_type.split(";", 1)[0].strip().lower()
return normalized in _BINARY_DOCUMENT_MIME_TYPES or normalized.startswith(
_BINARY_DOCUMENT_MIME_PREFIXES
)
def _binary_main_resource_body(mime_type: str) -> str:
normalized = mime_type.split(";", 1)[0].strip().lower()
if normalized == "application/pdf":
return "%PDF-1.7\n% moli benchmark CDP binary main resource\n" + ("\0" * 512)
return f"moli benchmark CDP binary main resource: {normalized}\n" + ("\0" * 512)
def _binary_main_resource_body_from_message(
message: dict[str, Any],
*,
session_id: str,
frame_id: str | None,
) -> str | None:
if message.get("sessionId") != session_id:
return None
if message.get("method") != "Network.responseReceived":
return None
params = message.get("params")
if not isinstance(params, dict):
return None
if params.get("type") != "Document":
return None
if frame_id is not None and params.get("frameId") != frame_id:
return None
response = params.get("response")
if not isinstance(response, dict):
return None
status = response.get("status")
if isinstance(status, (int, float)) and not 200 <= status < 400:
return None
mime_type = response.get("mimeType")
if not isinstance(mime_type, str) or not _is_binary_document_mime_type(mime_type):
return None
return _binary_main_resource_body(mime_type)
async def _recv_until_dcl_or_binary_main_resource(
client: RawCdpClient,
*,
session_id: str,
frame_id: str | None,
deadline: float,
seen: list[dict[str, Any]],
) -> str | None:
for message in seen:
binary_body = _binary_main_resource_body_from_message(
message,
session_id=session_id,
frame_id=frame_id,
)
if binary_body is not None:
return binary_body
if any(_is_dcl_event(message, session_id, frame_id) for message in seen):
return None
while True:
remaining = deadline - time.perf_counter()
if remaining <= 0:
raise TimeoutError("timed out waiting for Page.domContentEventFired")
message = await asyncio.wait_for(client.recv(), timeout=remaining)
binary_body = _binary_main_resource_body_from_message(
message,
session_id=session_id,
frame_id=frame_id,
)
if binary_body is not None:
return binary_body
if _is_dcl_event(message, session_id, frame_id):
return None
async def _dump_dcl_html(endpoint: str, process: subprocess.Popen[bytes], url: str, timeout_seconds: float) -> str:
deadline = time.perf_counter() + timeout_seconds
try:
client = await _wait_for_cdp(endpoint, process, min(5.0, max(0.1, timeout_seconds)))
except TimeoutError as error:
raise CdpDclDumpTimeoutError("startup", error) from error
target_id: str | None = None
try:
create_id = await client.send("Target.createTarget", {"url": "about:blank"})
create_response, _ = await _recv_command_response(
client,
create_id,
deadline=deadline,
stage="Target.createTarget",
)
target_id = str(create_response["result"]["targetId"])
attach_id = await client.send("Target.attachToTarget", {"targetId": target_id, "flatten": True})
attach_response, _ = await _recv_command_response(
client,
attach_id,
deadline=deadline,
stage="Target.attachToTarget",
)
session_id = str(attach_response["result"]["sessionId"])
for method in ("Page.enable", "Runtime.enable", "Network.enable"):
message_id = await client.send(method, session_id=session_id)
await _recv_command_response(
client,
message_id,
deadline=deadline,
stage=method,
)
lifecycle_id = await client.send("Page.setLifecycleEventsEnabled", {"enabled": True}, session_id=session_id)
await _recv_command_response(
client,
lifecycle_id,
deadline=deadline,
stage="Page.setLifecycleEventsEnabled",
)
navigate_id = await client.send("Page.navigate", {"url": url}, session_id=session_id)
navigate_response, seen = await _recv_command_response(
client,
navigate_id,
deadline=deadline,
stage="Page.navigate",
late_error_grace_seconds=CDP_LATE_ERROR_GRACE_SECONDS,
)
frame_id = navigate_response.get("result", {}).get("frameId")
if frame_id is not None:
frame_id = str(frame_id)
try:
binary_body = await _recv_until_dcl_or_binary_main_resource(
client,
session_id=session_id,
frame_id=frame_id,
deadline=deadline,
seen=seen,
)
except TimeoutError as error:
raise CdpDclDumpTimeoutError("DCL", error) from error
if binary_body is not None:
return binary_body
evaluate_id = await client.send(
"Runtime.evaluate",
{
"expression": "document.documentElement ? document.documentElement.outerHTML : ''",
"returnByValue": True,
},
session_id=session_id,
)
evaluate_response, _ = await _recv_command_response(
client,
evaluate_id,
deadline=deadline,
stage="outerHTML",
)
result = evaluate_response.get("result", {}).get("result", {})
value = result.get("value", "")
return value if isinstance(value, str) else ""
finally:
if target_id is not None:
try:
close_id = await client.send("Target.closeTarget", {"targetId": target_id})
await client.recv_until_id(close_id, timeout=1.0)
except Exception:
pass
await client.websocket.close()
def _terminate_process_group(process: subprocess.Popen[bytes]) -> None:
if process.poll() is not None:
return
try:
os.killpg(process.pid, signal.SIGTERM)
except OSError:
pass
try:
process.wait(timeout=2)
return
except subprocess.TimeoutExpired:
pass
try:
os.killpg(process.pid, signal.SIGKILL)
except OSError:
pass
try:
process.wait(timeout=2)
except subprocess.TimeoutExpired:
pass
def _read_tempfile(file_obj: Any) -> bytes:
file_obj.flush()
file_obj.seek(0)
return file_obj.read()
def _process_returncode_or(process: subprocess.Popen[bytes], fallback: int) -> int:
returncode = process.poll()
return int(returncode) if returncode is not None else fallback
def run_chrome_dcl_dump(
binary: Path,
url: str,
*,
cwd: Path = REPO_ROOT,
timeout_seconds: float,
env: dict[str, str] | None = None,
sample_resources: bool = True,
) -> ProcessResult:
started = time.perf_counter()
command: list[str] = []
stdout = b""
stderr = b""
browser_stdout = b""
error_suffix = b""
timed_out = False
returncode: int | None = None
with tempfile.TemporaryDirectory(prefix="moli-benchmark-chrome-") as temp_dir:
profile_dir = Path(temp_dir) / "profile"
profile_dir.mkdir()
with tempfile.TemporaryFile() as stdout_file, tempfile.TemporaryFile() as stderr_file:
reserved_port = reserve_port()
try:
port = reserved_port.port
command = _chrome_command(binary, port, profile_dir)
endpoint = f"http://127.0.0.1:{port}"
reserved_port.release_socket()
process = subprocess.Popen(
command,
cwd=cwd,
env=env,
stdout=stdout_file,
stderr=stderr_file,
start_new_session=True,
)
except BaseException:
reserved_port.close()
raise
sampler = ResourceSampler(process.pid) if sample_resources else None
if sampler is not None:
sampler.start()
try:
try:
html = asyncio.run(_dump_dcl_html(endpoint, process, url, timeout_seconds))
stdout = html.encode("utf-8", errors="replace")
returncode = 0
except CdpDclDumpTimeoutError as error:
timed_out = True
returncode = _process_returncode_or(process, 124)
error_suffix = (
f"\nchrome CDP {error.stage} timeout: {error.detail}\n"
).encode("utf-8", errors="replace")
except TimeoutError as error:
timed_out = True
returncode = _process_returncode_or(process, 124)
error_suffix = f"\nchrome CDP timeout: {error}\n".encode("utf-8", errors="replace")
except Exception as error: # noqa: BLE001 - convert CDP/browser failures into benchmark process output.
returncode = _process_returncode_or(process, 1)
error_suffix = f"\nchrome CDP DCL error: {error}\n".encode("utf-8", errors="replace")
finally:
_terminate_process_group(process)
reserved_port.close()
if returncode is None:
returncode = process.returncode
browser_stdout = _read_tempfile(stdout_file)
stderr = _read_tempfile(stderr_file) + error_suffix
finally:
elapsed_ms = (time.perf_counter() - started) * 1000.0
resources = sampler.stop() if sampler is not None else {}
if browser_stdout.strip() and not stdout:
stdout = browser_stdout
return ProcessResult(
command=command,
returncode=returncode,
elapsed_ms=elapsed_ms,
stdout=stdout,
stderr=stderr,
timed_out=timed_out,
resources=resources,
)
def run_served_cdp_dcl_dump(
target: str,
binary: Path,
url: str,
*,
cwd: Path = REPO_ROOT,
timeout_seconds: float,
env: dict[str, str] | None = None,
) -> ProcessResult:
del cwd, env
started = time.perf_counter()
command: list[str] = [str(binary), "serve"]
stdout = b""
stderr = b""
error_suffix = b""
timed_out = False
returncode: int | None = None
resources: dict[str, Any] = {}
serve = None
try:
serve = start_target_serve(target, binary, timeout_seconds)
command = serve.command
try:
html = asyncio.run(_dump_dcl_html(serve.endpoint, serve.process, url, timeout_seconds))
stdout = html.encode("utf-8", errors="replace")
returncode = 0
except CdpDclDumpTimeoutError as error:
timed_out = True
returncode = _process_returncode_or(serve.process, 124)
error_suffix = (
f"\n{target} CDP {error.stage} timeout: {error.detail}\n"
).encode("utf-8", errors="replace")
except TimeoutError as error:
timed_out = True
returncode = _process_returncode_or(serve.process, 124)
error_suffix = f"\n{target} CDP timeout: {error}\n".encode("utf-8", errors="replace")
except Exception as error: # noqa: BLE001 - convert CDP/browser failures into benchmark process output.
returncode = _process_returncode_or(serve.process, 1)
error_suffix = f"\n{target} CDP DCL error: {error}\n".encode("utf-8", errors="replace")
finally:
stopped = stop_target_serve(serve)
serve = None
if returncode is None:
stopped_returncode = stopped.get("returncode")
returncode = int(stopped_returncode) if isinstance(stopped_returncode, int) else None
stopped_resources = stopped.get("resources")
resources = stopped_resources if isinstance(stopped_resources, dict) else {}
log_tail = stopped.get("log_tail")
if isinstance(log_tail, list) and log_tail:
stderr = "\n".join(str(line) for line in log_tail).encode("utf-8", errors="replace")
stderr += error_suffix
except TimeoutError as error:
timed_out = True
error_suffix = f"\n{target} CDP timeout: {error}\n".encode("utf-8", errors="replace")
stderr += error_suffix
except Exception as error: # noqa: BLE001 - startup failures are benchmark process errors.
error_suffix = f"\n{target} CDP DCL error: {error}\n".encode("utf-8", errors="replace")
stderr += error_suffix
finally:
if serve is not None:
stopped = stop_target_serve(serve)
stopped_resources = stopped.get("resources")
if isinstance(stopped_resources, dict) and not resources:
resources = stopped_resources
elapsed_ms = (time.perf_counter() - started) * 1000.0
return ProcessResult(
command=command,
returncode=returncode,
elapsed_ms=elapsed_ms,
stdout=stdout,
stderr=stderr,
timed_out=timed_out,
resources=resources,
)
moli-benchmark/moli_benchmark/cli.py
+1183
View File
File diff suppressed because it is too large Load Diff
moli-benchmark/moli_benchmark/config.py
+176
View File
@@ -0,0 +1,176 @@
from __future__ import annotations
import os
import random
import socket
import shutil
import threading
from dataclasses import dataclass
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
PROJECT_ROOT = REPO_ROOT / "moli-benchmark"
RESULTS_ROOT = PROJECT_ROOT / "results"
FORMAL_RESULTS_ROOT = REPO_ROOT / "benchmarks" / "results"
PROXY_ENV_KEYS = (
"HTTP_PROXY",
"HTTPS_PROXY",
"ALL_PROXY",
"http_proxy",
"https_proxy",
"all_proxy",
)
_RESERVED_PORTS: set[int] = set()
_RESERVED_PORTS_LOCK = threading.Lock()
_SERVER_PORT_RANDOM = random.SystemRandom()
_SERVER_PORT_FALLBACK_RANGE = (10_000, 32_767)
def _linux_ephemeral_port_range() -> tuple[int, int] | None:
try:
raw = Path("/proc/sys/net/ipv4/ip_local_port_range").read_text(encoding="utf-8").strip()
low_raw, high_raw = raw.split()
low = int(low_raw)
high = int(high_raw)
except (OSError, ValueError):
return None
if 0 < low <= high <= 65_535:
return low, high
return None
def _server_port_range() -> tuple[int, int]:
ephemeral = _linux_ephemeral_port_range()
if ephemeral is None:
return _SERVER_PORT_FALLBACK_RANGE
low, _ = ephemeral
candidate = (_SERVER_PORT_FALLBACK_RANGE[0], min(_SERVER_PORT_FALLBACK_RANGE[1], low - 1))
if candidate[0] <= candidate[1]:
return candidate
return _SERVER_PORT_FALLBACK_RANGE
def clear_proxy_env(env: dict[str, str]) -> dict[str, str]:
next_env = dict(env)
for key in PROXY_ENV_KEYS:
next_env.pop(key, None)
next_env["NO_PROXY"] = "*"
next_env["no_proxy"] = "*"
return next_env
def clear_current_proxy_env() -> None:
next_env = clear_proxy_env(os.environ)
os.environ.clear()
os.environ.update(next_env)
@dataclass
class ReservedPort:
socket: socket.socket | None
_port: int
_closed: bool = False
@property
def port(self) -> int:
return self._port
def release_socket(self) -> None:
sock = self.socket
if sock is not None:
self.socket = None
sock.close()
def close(self) -> None:
if self._closed:
return
self.release_socket()
with _RESERVED_PORTS_LOCK:
_RESERVED_PORTS.discard(self._port)
self._closed = True
def __enter__(self) -> "ReservedPort":
return self
def __exit__(self, exc_type: object, exc: object, tb: object) -> None:
self.close()
def reserve_port() -> ReservedPort:
port_low, port_high = _server_port_range()
last_error: BaseException | None = None
for _ in range(512):
port = _SERVER_PORT_RANDOM.randint(port_low, port_high)
with _RESERVED_PORTS_LOCK:
if port in _RESERVED_PORTS:
continue
_RESERVED_PORTS.add(port)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.bind(("127.0.0.1", port))
bound_port = int(sock.getsockname()[1])
if bound_port != port:
raise RuntimeError(f"reserved unexpected benchmark port {bound_port}, expected {port}")
return ReservedPort(sock, port)
except BaseException as error:
with _RESERVED_PORTS_LOCK:
_RESERVED_PORTS.discard(port)
last_error = error
sock.close()
if isinstance(error, OSError):
continue
raise
raise RuntimeError(f"failed to reserve a unique local benchmark port in {port_low}-{port_high}") from last_error
def moli_binary(override: str | None = None) -> Path:
raw_override = override or os.environ.get("MOLI_BIN")
if raw_override:
path = Path(raw_override).expanduser().resolve()
if not path.exists():
raise RuntimeError(f"MOLI_BIN does not exist: {path}")
return path
candidates = [
REPO_ROOT / "target" / "release" / "moli",
]
existing = [candidate for candidate in candidates if candidate.exists()]
if existing:
return max(existing, key=lambda path: path.stat().st_mtime)
path_candidate = shutil.which("moli")
if path_candidate:
return Path(path_candidate).resolve()
raise RuntimeError("missing moli binary; run `cargo build --release` or set MOLI_BIN")
def optional_binary(env_key: str, names: tuple[str, ...], override: str | None = None) -> Path | None:
raw_override = override or os.environ.get(env_key)
if raw_override:
path = Path(raw_override).expanduser().resolve()
if not path.exists():
raise RuntimeError(f"{env_key} does not exist: {path}")
return path
for name in names:
candidate = shutil.which(name)
if candidate:
return Path(candidate).resolve()
return None
def lightpanda_binary(override: str | None = None) -> Path | None:
return optional_binary("LIGHTPANDA_BIN", ("lightpanda",), override)
def obscura_binary(override: str | None = None) -> Path | None:
return optional_binary("OBSCURA_BIN", ("obscura",), override)
def chrome_binary(override: str | None = None) -> Path | None:
return optional_binary(
"CHROME_BIN",
("google-chrome", "google-chrome-stable", "chrome", "chromium", "chromium-browser"),
override,
)
moli-benchmark/moli_benchmark/crawler.py
+122
View File
@@ -0,0 +1,122 @@
from __future__ import annotations
import os
from pathlib import Path
from typing import Any
from .artifacts import write_csv, write_json
from .config import REPO_ROOT, clear_proxy_env
from .process import run_process
from .stats import summarize
from .synthetic import SYNTHETIC_CASES, SyntheticServer
from .synthetic_compare import FETCH_TARGETS, _command_for_target, target_metadata, target_uses_external_fixture
def run_crawler_suite(
*,
output_dir: Path,
target_matrix: dict[str, Any],
targets: tuple[str, ...],
pages: int,
runs: int,
timeout_seconds: float,
gate_target: str,
) -> dict[str, Any]:
unknown_targets = [target for target in targets if target not in FETCH_TARGETS]
if unknown_targets:
raise RuntimeError(f"unknown fetch target(s): {', '.join(unknown_targets)}; use cdp-session for CDP targets")
if gate_target not in FETCH_TARGETS:
raise RuntimeError(f"unknown gate target: {gate_target}")
suite_dir = output_dir / "crawler"
rows: list[dict[str, Any]] = []
cases = list(SYNTHETIC_CASES)
with SyntheticServer() as server:
for target in targets:
metadata = target_metadata(target)
info = target_matrix.get(metadata["binary_key"], {})
path = info.get("path")
urls = [
f"{server.url_for_path(cases[index % len(cases)], external=target_uses_external_fixture(target))}?crawl={index}"
for index in range(pages)
]
for run_id in range(1, runs + 1):
for index, url in enumerate(urls):
case = cases[index % len(cases)]
if not info.get("available") or not path:
rows.append(
{
"target": target,
**metadata,
"run": run_id,
"page": index,
"case": case,
"ok": False,
"elapsed_ms": None,
"error": "target binary unavailable",
}
)
continue
if target_uses_external_fixture(target) and server.external_base_url is None:
rows.append(
{
"target": target,
**metadata,
"run": run_id,
"page": index,
"case": case,
"ok": False,
"elapsed_ms": None,
"error": "external fixture address unavailable for Obscura",
}
)
continue
result = run_process(
_command_for_target(target, Path(path), url, timeout_seconds),
cwd=REPO_ROOT,
timeout_seconds=timeout_seconds + 2,
env=clear_proxy_env(os.environ),
)
ok = result.returncode == 0 and not result.timed_out and b"data-benchmark-status=\"ok\"" in result.stdout
rows.append(
{
"target": target,
**metadata,
"run": run_id,
"page": index,
"case": case,
"ok": ok,
"elapsed_ms": result.elapsed_ms,
"returncode": result.returncode,
"timed_out": result.timed_out,
"peak_pss_bytes": result.resources.get("peak_pss_bytes"),
"error": None if ok else "marker missing or process failed",
}
)
gate_failures = sum(1 for row in rows if row["target"] == gate_target and not row.get("ok"))
summary: dict[str, Any] = {
"suite": "crawler",
"runs": runs,
"pages": pages,
"timeout_seconds": timeout_seconds,
"gate_target": gate_target,
"gate_failures": gate_failures,
"targets": {},
"total_failures": sum(1 for row in rows if not row.get("ok")),
}
for target in targets:
target_rows = [row for row in rows if row["target"] == target]
summary["targets"][target] = {
**target_metadata(target),
"pages": len(target_rows),
"runs": runs,
"passes": sum(1 for row in target_rows if row.get("ok")),
"failures": sum(1 for row in target_rows if not row.get("ok")),
"elapsed_ms": summarize(row["elapsed_ms"] for row in target_rows if row.get("ok") and row.get("elapsed_ms") is not None),
"peak_pss_bytes": summarize(row["peak_pss_bytes"] for row in target_rows if row.get("peak_pss_bytes") is not None),
}
write_csv(suite_dir / "raw-runs.csv", rows)
write_json(suite_dir / "runs.json", rows)
write_json(suite_dir / "summary.json", summary)
return summary
moli-benchmark/moli_benchmark/environment.py
+71
View File
@@ -0,0 +1,71 @@
from __future__ import annotations
import os
import platform
import socket
import subprocess
from pathlib import Path
from typing import Any
from .config import PROXY_ENV_KEYS
def _read_first_existing(paths: list[Path]) -> str | None:
for path in paths:
try:
if path.exists():
return path.read_text(encoding="utf-8", errors="replace").strip()
except OSError:
continue
return None
def _command_version(command: list[str]) -> str | None:
try:
completed = subprocess.run(command, capture_output=True, text=True, timeout=5, check=False)
except (OSError, subprocess.TimeoutExpired):
return None
output = (completed.stdout or completed.stderr).strip()
return output.splitlines()[0] if output else None
def collect_environment() -> dict[str, Any]:
os_release = _read_first_existing([Path("/etc/os-release")])
cpuinfo = _read_first_existing([Path("/proc/cpuinfo")])
meminfo = _read_first_existing([Path("/proc/meminfo")])
cpu_model = None
if cpuinfo:
for line in cpuinfo.splitlines():
if line.startswith("model name"):
cpu_model = line.split(":", 1)[1].strip()
break
memory_total_kb = None
if meminfo:
for line in meminfo.splitlines():
if line.startswith("MemTotal:"):
parts = line.split()
if len(parts) >= 2:
memory_total_kb = int(parts[1])
break
cgroup = {
"memory_current": _read_first_existing([Path("/sys/fs/cgroup/memory.current")]),
"memory_peak": _read_first_existing([Path("/sys/fs/cgroup/memory.peak")]),
"cpu_stat": _read_first_existing([Path("/sys/fs/cgroup/cpu.stat")]),
}
return {
"hostname": socket.gethostname(),
"platform": platform.platform(),
"machine": platform.machine(),
"processor": platform.processor(),
"python": platform.python_version(),
"os_release": os_release,
"kernel": platform.release(),
"glibc": _command_version(["ldd", "--version"]),
"cpu_model": cpu_model,
"logical_cores": os.cpu_count(),
"memory_total_kb": memory_total_kb,
"cgroup": cgroup,
"proxy_env": {key: os.environ.get(key) for key in PROXY_ENV_KEYS if os.environ.get(key)},
}
moli-benchmark/moli_benchmark/html_report.py
+2172
View File
File diff suppressed because it is too large Load Diff
moli-benchmark/moli_benchmark/process.py
+195
View File
@@ -0,0 +1,195 @@
from __future__ import annotations
import os
import signal
import subprocess
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from .sampling import ResourceSampler, collect_cgroup_artifacts
@dataclass(frozen=True)
class ProcessResult:
command: list[str]
returncode: int | None
elapsed_ms: float
stdout: bytes
stderr: bytes
timed_out: bool
resources: dict[str, Any]
time_verbose: dict[str, Any] | None = None
cgroup: dict[str, Any] | None = None
def output_digest_material(self) -> bytes:
return self.stdout + b"\n--- stderr ---\n" + self.stderr
def json_summary(self, include_output: bool = False) -> dict[str, Any]:
summary: dict[str, Any] = {
"command": self.command,
"returncode": self.returncode,
"elapsed_ms": self.elapsed_ms,
"timed_out": self.timed_out,
"resources": self.resources,
"stdout_bytes": len(self.stdout),
"stderr_bytes": len(self.stderr),
}
if self.time_verbose is not None:
summary["time_verbose"] = self.time_verbose
if self.cgroup is not None:
summary["cgroup"] = self.cgroup
if include_output:
summary["stdout_tail"] = self.stdout[-4096:].decode("utf-8", errors="replace")
summary["stderr_tail"] = self.stderr[-4096:].decode("utf-8", errors="replace")
return summary
def _kill_process_group(pid: int, sig: signal.Signals) -> bool:
try:
os.killpg(pid, sig)
return True
except OSError:
return False
def _parse_elapsed_seconds(value: str) -> float | None:
parts = value.strip().split(":")
try:
if len(parts) == 3:
hours, minutes, seconds = parts
return int(hours) * 3600 + int(minutes) * 60 + float(seconds)
if len(parts) == 2:
minutes, seconds = parts
return int(minutes) * 60 + float(seconds)
return float(value)
except ValueError:
return None
def parse_time_verbose_output(text: str) -> dict[str, Any]:
parsed: dict[str, Any] = {}
if text.startswith("unavailable:"):
return {"available": False, "error": text.strip()}
for line in text.splitlines():
if ": " not in line:
continue
key, value = line.split(": ", 1)
key = key.strip()
value = value.strip()
try:
integer = int(value)
except ValueError:
integer = None
if key == "User time (seconds)":
try:
parsed["user_seconds"] = float(value)
except ValueError:
pass
elif key == "System time (seconds)":
try:
parsed["system_seconds"] = float(value)
except ValueError:
pass
elif key == "Elapsed (wall clock) time (h:mm:ss or m:ss)":
parsed["elapsed_seconds"] = _parse_elapsed_seconds(value)
elif key == "Maximum resident set size (kbytes)" and integer is not None:
parsed["max_rss_bytes"] = integer * 1024
elif key == "Major (requiring I/O) page faults" and integer is not None:
parsed["major_page_faults"] = integer
elif key == "Minor (reclaiming a frame) page faults" and integer is not None:
parsed["minor_page_faults"] = integer
elif key == "Voluntary context switches" and integer is not None:
parsed["voluntary_context_switches"] = integer
elif key == "Involuntary context switches" and integer is not None:
parsed["involuntary_context_switches"] = integer
elif key == "File system inputs" and integer is not None:
parsed["file_system_inputs"] = integer
elif key == "File system outputs" and integer is not None:
parsed["file_system_outputs"] = integer
elif key == "Exit status" and integer is not None:
parsed["exit_status"] = integer
if parsed:
parsed["available"] = True
return parsed
def read_time_verbose_file(path: Path) -> dict[str, Any] | None:
try:
raw = path.read_text(encoding="utf-8", errors="replace")
except OSError:
return None
parsed = parse_time_verbose_output(raw)
parsed["raw_path"] = str(path)
parsed["raw_bytes"] = len(raw.encode("utf-8"))
return parsed
def time_verbose_command(command: list[str], output_path: Path | None) -> list[str]:
if output_path is None:
return command
output_path.parent.mkdir(parents=True, exist_ok=True)
time_bin = Path("/usr/bin/time")
if not time_bin.exists():
output_path.write_text("unavailable: /usr/bin/time executable not found\n", encoding="utf-8")
return command
return [str(time_bin), "-v", "-o", str(output_path), *command]
def run_process(
command: list[str],
*,
cwd: Path,
timeout_seconds: float,
env: dict[str, str] | None = None,
sample_resources: bool = True,
time_verbose_path: Path | None = None,
cgroup_artifact_dir: Path | None = None,
) -> ProcessResult:
started = time.perf_counter()
effective_command = time_verbose_command(command, time_verbose_path)
process = subprocess.Popen(
effective_command,
cwd=cwd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
start_new_session=True,
)
sampler = ResourceSampler(process.pid) if sample_resources else None
cgroup = collect_cgroup_artifacts(process.pid, cgroup_artifact_dir) if cgroup_artifact_dir is not None else None
if sampler is not None:
sampler.start()
timed_out = False
stdout = b""
stderr = b""
try:
try:
stdout, stderr = process.communicate(timeout=timeout_seconds)
except subprocess.TimeoutExpired:
timed_out = True
_kill_process_group(process.pid, signal.SIGTERM)
try:
stdout, stderr = process.communicate(timeout=2)
except subprocess.TimeoutExpired:
_kill_process_group(process.pid, signal.SIGKILL)
try:
stdout, stderr = process.communicate(timeout=2)
except subprocess.TimeoutExpired:
pass
finally:
elapsed_ms = (time.perf_counter() - started) * 1000.0
resources = sampler.stop() if sampler is not None else {}
time_verbose = read_time_verbose_file(time_verbose_path) if time_verbose_path is not None else None
return ProcessResult(
command=command,
returncode=process.returncode,
elapsed_ms=elapsed_ms,
stdout=stdout,
stderr=stderr,
timed_out=timed_out,
resources=resources,
time_verbose=time_verbose,
cgroup=cgroup,
)
moli-benchmark/moli_benchmark/publish_readiness.py
+308
View File
@@ -0,0 +1,308 @@
from __future__ import annotations
from pathlib import Path
from typing import Any
REQUIRED_TARGETS = ("moli", "lightpanda", "chrome", "obscura")
TOP_LEVEL_ARTIFACTS = (
"environment.json",
"versions.json",
"summary.json",
"summary.md",
"publish-readiness.json",
"report-data.json",
"index.html",
)
def _summary_by_suite(summaries: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
return {str(summary.get("suite")): summary for summary in summaries if summary.get("suite")}
def _suite_failures(summary: dict[str, Any] | None) -> int | None:
if not summary:
return None
value = summary.get("gate_failures", summary.get("total_failures"))
try:
return int(value or 0)
except (TypeError, ValueError):
return None
def _compact_cdp_client_rows(rows: list[Any]) -> list[dict[str, Any]]:
compact = []
for row in rows:
if not isinstance(row, dict):
continue
compact.append(
{
"client": row.get("client"),
"required": row.get("required"),
"covered": row.get("covered"),
"gate_ok": row.get("gate_ok"),
"failure_kind": row.get("failure_kind"),
"record_count": row.get("record_count"),
"groups": row.get("groups"),
}
)
return compact
def _compact_gate_rows(rows: list[Any]) -> list[dict[str, Any]]:
compact = []
for row in rows:
if not isinstance(row, dict):
continue
compact.append(
{
"gate": row.get("gate"),
"ok": row.get("ok"),
"actual": row.get("actual"),
"required": row.get("required"),
"failure_kind": row.get("failure_kind"),
}
)
return compact
def _target_is_available(versions: dict[str, Any], name: str) -> bool:
target = (versions.get("targets") or {}).get(name)
return isinstance(target, dict) and bool(target.get("available"))
def _check(
checks: list[dict[str, Any]],
*,
name: str,
ok: bool,
required: Any,
actual: Any,
detail: str,
) -> None:
checks.append(
{
"name": name,
"ok": bool(ok),
"required": required,
"actual": actual,
"detail": detail,
}
)
def _has_full_target_matrix(summary: dict[str, Any] | None) -> bool:
if not summary:
return False
targets = summary.get("targets")
if not isinstance(targets, dict):
return False
engines: set[str] = set()
for target, result in targets.items():
if isinstance(result, dict) and isinstance(result.get("engine"), str):
engines.add(result["engine"])
elif isinstance(target, str):
engines.add(target.removesuffix("-cdp"))
return all(target in engines for target in REQUIRED_TARGETS)
def build_publish_readiness(
*,
output_dir: Path,
versions: dict[str, Any],
summaries: list[dict[str, Any]],
) -> dict[str, Any]:
by_suite = _summary_by_suite(summaries)
checks: list[dict[str, Any]] = []
missing_top_level = [path for path in TOP_LEVEL_ARTIFACTS if not (output_dir / path).exists()]
_check(
checks,
name="top-level artifacts",
ok=not missing_top_level,
required=list(TOP_LEVEL_ARTIFACTS),
actual=[path for path in TOP_LEVEL_ARTIFACTS if path not in missing_top_level],
detail="Formal reports must carry machine-readable metadata, summaries, and the human summary.",
)
available_targets = [target for target in REQUIRED_TARGETS if _target_is_available(versions, target)]
_check(
checks,
name="target matrix",
ok=len(available_targets) == len(REQUIRED_TARGETS),
required=list(REQUIRED_TARGETS),
actual=available_targets,
detail="Publishable comparison reports require local Moli, Lightpanda, Chrome, and Obscura measurements.",
)
synthetic = by_suite.get("synthetic-matrix")
synthetic_gate_rows = synthetic.get("formal_gate_rows", []) if isinstance(synthetic, dict) else []
if not isinstance(synthetic_gate_rows, list):
synthetic_gate_rows = []
_check(
checks,
name="synthetic formal matrix",
ok=bool(
synthetic
and synthetic.get("profile") == "formal"
and _suite_failures(synthetic) == 0
and synthetic_gate_rows
and all(isinstance(row, dict) and row.get("ok") is True for row in synthetic_gate_rows)
),
required="synthetic-matrix formal profile with every formal gate row passing",
actual={
"suite": synthetic.get("suite") if synthetic else None,
"profile": synthetic.get("profile") if synthetic else None,
"gate_failures": _suite_failures(synthetic),
"formal_gate_rows": _compact_gate_rows(synthetic_gate_rows),
},
detail="Suite B is the first formalization target because it is local, deterministic, and regression-friendly.",
)
startup = by_suite.get("startup")
startup_gate_rows = startup.get("formal_gate_rows", []) if isinstance(startup, dict) else []
if not isinstance(startup_gate_rows, list):
startup_gate_rows = []
_check(
checks,
name="startup and size",
ok=bool(
startup
and startup.get("profile") == "formal"
and _suite_failures(startup) == 0
and startup_gate_rows
and all(isinstance(row, dict) and row.get("ok") is True for row in startup_gate_rows)
),
required="startup formal profile with every formal gate row passing",
actual={
"suite": startup.get("suite") if startup else None,
"profile": startup.get("profile") if startup else None,
"gate_failures": _suite_failures(startup),
"formal_gate_rows": _compact_gate_rows(startup_gate_rows),
},
detail="Suite E must archive startup, size, CDP first page, warm page, idle footprint, resource, and readiness evidence.",
)
wpt = by_suite.get("wpt")
wpt_summary = wpt.get("summary", {}) if isinstance(wpt, dict) else {}
_check(
checks,
name="wpt p0 smoke",
ok=bool(
wpt
and int(wpt.get("cases", 0) or 0) > 0
and _suite_failures(wpt) == 0
and int(wpt_summary.get("unexpected_fail", 0) or 0) == 0
and int(wpt_summary.get("skip", 0) or 0) == 0
and int(wpt_summary.get("known_fail", 0) or 0) == 0
),
required="non-empty WPT P0 smoke report with 0 unexpected fail, 0 skip, 0 known-fail",
actual={
"cases": wpt.get("cases") if wpt else None,
"gate_failures": _suite_failures(wpt),
"unexpected_fail": wpt_summary.get("unexpected_fail"),
"skip": wpt_summary.get("skip"),
"known_fail": wpt_summary.get("known_fail"),
},
detail="WPT remains the compatibility gate; real-site results cannot replace it.",
)
cdp = by_suite.get("cdp-smoke")
cdp_client_rows = cdp.get("client_rows", []) if isinstance(cdp, dict) else []
if not isinstance(cdp_client_rows, list):
cdp_client_rows = []
cdp_clients = {
str(row.get("client")): row
for row in cdp_client_rows
if isinstance(row, dict) and row.get("client") is not None
}
_check(
checks,
name="cdp workflow",
ok=bool(
cdp
and cdp.get("profile") == "formal"
and _suite_failures(cdp) == 0
and cdp_clients.get("raw_cdp", {}).get("gate_ok") is True
and cdp_clients.get("playwright", {}).get("gate_ok") is True
and cdp_clients.get("puppeteer", {}).get("gate_ok") is True
),
required="formal CDP smoke with raw CDP, Playwright, and Puppeteer coverage and zero gate failures",
actual={
"suite": cdp.get("suite") if cdp else None,
"profile": cdp.get("profile") if cdp else None,
"gate_failures": _suite_failures(cdp),
"client_rows": _compact_cdp_client_rows(cdp_client_rows),
},
detail="Suite D must prove real ecosystem clients, not only raw protocol navigation.",
)
amiibo = by_suite.get("amiibo-crawler")
_check(
checks,
name="amiibo formal crawler",
ok=bool(amiibo and amiibo.get("profile") == "formal" and _suite_failures(amiibo) == 0),
required="Amiibo crawler formal profile with all 933 pages and zero gate failures",
actual={
"suite": amiibo.get("suite") if amiibo else None,
"profile": amiibo.get("profile") if amiibo else None,
"gate_failures": _suite_failures(amiibo),
},
detail="Suite A must use the real Lightpanda Amiibo workload for external credibility.",
)
wild_web = by_suite.get("wild-web")
selected_seeds = wild_web.get("seeds", []) if isinstance(wild_web, dict) else []
wild_targets = wild_web.get("targets", {}) if isinstance(wild_web, dict) else {}
wild_extraction_failures = sum(
int(target.get("extraction_failures", 0) or 0)
for target in wild_targets.values()
if isinstance(target, dict)
)
_check(
checks,
name="wild-web p0 seeds",
ok=bool(
wild_web
and _suite_failures(wild_web) == 0
and wild_extraction_failures == 0
and {"zhihu-home", "toutiao-home"}.issubset(set(selected_seeds))
),
required={"seeds": ["zhihu-home", "toutiao-home"], "extraction_failures": 0},
actual={"seeds": selected_seeds, "gate_failures": _suite_failures(wild_web), "extraction_failures": wild_extraction_failures},
detail="Wild-web is a product-value signal and implementation-gap discovery source.",
)
full_compare = any(_has_full_target_matrix(summary) for summary in summaries)
_check(
checks,
name="horizontal comparison",
ok=full_compare,
required=list(REQUIRED_TARGETS),
actual=[
{
"suite": summary.get("suite"),
"targets": list((summary.get("targets") or {}).keys()),
}
for summary in summaries
if summary.get("targets")
],
detail="External reports need at least one suite that measured Moli, Lightpanda, Chrome, and Obscura side by side.",
)
failed = [check for check in checks if not check["ok"]]
status = "publishable" if not failed else "investigation"
return {
"schema_version": 1,
"status": status,
"checks": checks,
"known_invalid_items": [
{
"name": check["name"],
"required": check["required"],
"actual": check["actual"],
"detail": check["detail"],
}
for check in failed
],
}
moli-benchmark/moli_benchmark/raw_cdp.py
+371
View File
@@ -0,0 +1,371 @@
from __future__ import annotations
import asyncio
import bisect
import contextlib
import json
import time
import urllib.request
from dataclasses import dataclass
from typing import Any, Callable
import websockets
from websockets.asyncio.client import ClientConnection
class RawCdpError(RuntimeError):
pass
class RawCdpTimeoutError(RawCdpError, TimeoutError):
def __init__(
self,
message: str,
*,
method: str | None = None,
message_id: int | None = None,
messages: list[dict[str, Any]] | None = None,
) -> None:
super().__init__(message)
self.method = method
self.message_id = message_id
self.messages = list(messages or ())
class RawCdpCommandError(RawCdpError):
def __init__(
self,
*,
method: str,
message_id: int,
error: dict[str, Any],
messages: list[dict[str, Any]],
elapsed_ms: float,
) -> None:
super().__init__(f"CDP {method} failed: {error}")
self.method = method
self.message_id = message_id
self.error = error
self.messages = messages
self.elapsed_ms = elapsed_ms
class RawCdpConnectionClosed(RawCdpError):
pass
def _read_json_url(url: str) -> dict[str, Any]:
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
with opener.open(url, timeout=2) as response:
return json.loads(response.read().decode("utf-8"))
async def discover_websocket_url(endpoint: str) -> str:
payload = await asyncio.to_thread(_read_json_url, endpoint.rstrip("/") + "/json/version")
websocket_url = payload.get("webSocketDebuggerUrl")
if not isinstance(websocket_url, str) or not websocket_url:
raise RawCdpError(f"CDP discovery response did not include webSocketDebuggerUrl: {payload}")
return websocket_url
@dataclass
class RawCdpClient:
websocket: ClientConnection
next_id: int = 1
command_count: int = 0
async def send(self, method: str, params: dict[str, Any] | None = None, *, session_id: str | None = None) -> int:
message_id = self.next_id
self.next_id += 1
self.command_count += 1
message: dict[str, Any] = {"id": message_id, "method": method}
if params is not None:
message["params"] = params
if session_id is not None:
message["sessionId"] = session_id
await self.websocket.send(json.dumps(message, separators=(",", ":")))
return message_id
async def recv(self) -> dict[str, Any]:
raw = await self.websocket.recv()
if isinstance(raw, bytes):
raw = raw.decode("utf-8")
payload = json.loads(raw)
if not isinstance(payload, dict):
raise RawCdpError(f"unexpected CDP payload: {payload!r}")
return payload
async def recv_until_id(self, message_id: int, *, timeout: float = 10.0) -> tuple[dict[str, Any], list[dict[str, Any]]]:
seen: list[dict[str, Any]] = []
deadline = asyncio.get_running_loop().time() + timeout
while True:
remaining = deadline - asyncio.get_running_loop().time()
if remaining <= 0:
raise RawCdpTimeoutError(f"timed out waiting for CDP response id={message_id}; seen={seen[-20:]}")
message = await asyncio.wait_for(self.recv(), timeout=remaining)
seen.append(message)
if message.get("id") == message_id:
if "error" in message:
raise RawCdpError(f"CDP command id={message_id} failed: {message['error']}")
return message, seen
async def connect_raw_cdp(endpoint: str) -> RawCdpClient:
websocket_url = await discover_websocket_url(endpoint)
websocket = await websockets.connect(websocket_url, open_timeout=5, proxy=None, max_size=None)
return RawCdpClient(websocket=websocket)
@dataclass(frozen=True)
class RecordedCdpMessage:
sequence: int
received_monotonic: float
received_epoch: float
payload: dict[str, Any]
def json_value(self) -> dict[str, Any]:
return {
"sequence": self.sequence,
"received_monotonic": self.received_monotonic,
"received_epoch": self.received_epoch,
"payload": self.payload,
}
@dataclass(frozen=True)
class RoutedCommandResult:
message_id: int
response: dict[str, Any]
messages: list[RecordedCdpMessage]
elapsed_ms: float
class RoutedRawCdpClient:
"""Raw CDP client with one receiver and lossless event recording.
``RawCdpClient`` intentionally mirrors the small sequential harness used by
older suites. Agent episodes need a stronger boundary: a background reader
owns the WebSocket, responses are routed by id, and every intervening event
remains available for ordering checks and failure artifacts.
"""
def __init__(self, websocket: ClientConnection) -> None:
self.websocket = websocket
self.next_id = 1
self.command_count = 0
self._send_lock = asyncio.Lock()
self._condition = asyncio.Condition()
self._pending: dict[int, asyncio.Future[RecordedCdpMessage]] = {}
self._messages: list[RecordedCdpMessage] = []
self._receiver_task: asyncio.Task[None] | None = None
self._closed_error: BaseException | None = None
self._closing = False
def start(self) -> None:
if self._receiver_task is not None:
raise RuntimeError("routed CDP receiver already started")
self._receiver_task = asyncio.create_task(
self._receive_loop(),
name="moli-benchmark-cdp-receiver",
)
@property
def current_sequence(self) -> int:
return self._messages[-1].sequence if self._messages else 0
def recorded_messages(self) -> list[dict[str, Any]]:
return [message.json_value() for message in self._messages]
def _message_index_after(self, sequence: int) -> int:
return bisect.bisect_right(
self._messages,
sequence,
key=lambda message: message.sequence,
)
def messages_since(self, sequence: int) -> list[RecordedCdpMessage]:
return self._messages[self._message_index_after(sequence) :]
async def command(
self,
method: str,
params: dict[str, Any] | None = None,
*,
session_id: str | None = None,
timeout: float = 10.0,
) -> RoutedCommandResult:
if timeout <= 0:
raise ValueError("CDP command timeout must be positive")
if self._closed_error is not None:
raise RawCdpConnectionClosed(str(self._closed_error))
loop = asyncio.get_running_loop()
future: asyncio.Future[RecordedCdpMessage] = loop.create_future()
async with self._send_lock:
if self._closed_error is not None:
raise RawCdpConnectionClosed(str(self._closed_error))
message_id = self.next_id
self.next_id += 1
self.command_count += 1
sequence_before_send = self.current_sequence
message: dict[str, Any] = {"id": message_id, "method": method}
if params is not None:
message["params"] = params
if session_id is not None:
message["sessionId"] = session_id
self._pending[message_id] = future
started = time.perf_counter()
try:
await self.websocket.send(json.dumps(message, separators=(",", ":")))
except BaseException:
self._pending.pop(message_id, None)
future.cancel()
raise
try:
record = await asyncio.wait_for(asyncio.shield(future), timeout=timeout)
except TimeoutError as error:
self._pending.pop(message_id, None)
future.cancel()
seen = [message.json_value() for message in self.messages_since(sequence_before_send)]
raise RawCdpTimeoutError(
f"timed out waiting for CDP {method} response id={message_id}",
method=method,
message_id=message_id,
messages=seen,
) from error
elapsed_ms = (time.perf_counter() - started) * 1000.0
messages = self.messages_since(sequence_before_send)
response = record.payload
error_value = response.get("error")
if isinstance(error_value, dict):
raise RawCdpCommandError(
method=method,
message_id=message_id,
error=error_value,
messages=[message.json_value() for message in messages],
elapsed_ms=elapsed_ms,
)
return RoutedCommandResult(
message_id=message_id,
response=response,
messages=messages,
elapsed_ms=elapsed_ms,
)
async def wait_for_event(
self,
method: str,
*,
after_sequence: int = 0,
session_id: str | None = None,
predicate: Callable[[dict[str, Any]], bool] | None = None,
timeout: float = 10.0,
) -> RecordedCdpMessage:
deadline = asyncio.get_running_loop().time() + timeout
async with self._condition:
scan_index = self._message_index_after(after_sequence)
while True:
while scan_index < len(self._messages):
message = self._messages[scan_index]
scan_index += 1
payload = message.payload
if payload.get("method") != method:
continue
if session_id is not None and payload.get("sessionId") != session_id:
continue
if predicate is not None and not predicate(payload):
continue
return message
if self._closed_error is not None:
raise RawCdpConnectionClosed(str(self._closed_error))
remaining = deadline - asyncio.get_running_loop().time()
if remaining <= 0:
raise RawCdpTimeoutError(
f"timed out waiting for CDP event {method}",
method=method,
messages=[
message.json_value()
for message in self.messages_since(after_sequence)
],
)
try:
await asyncio.wait_for(self._condition.wait(), timeout=remaining)
except TimeoutError as error:
raise RawCdpTimeoutError(
f"timed out waiting for CDP event {method}",
method=method,
messages=[
message.json_value()
for message in self.messages_since(after_sequence)
],
) from error
async def close(self) -> None:
if self._closing:
return
self._closing = True
with contextlib.suppress(Exception):
await self.websocket.close()
task = self._receiver_task
if task is not None:
with contextlib.suppress(asyncio.CancelledError, Exception):
await task
async def _receive_loop(self) -> None:
failure: BaseException | None = None
try:
while True:
raw = await self.websocket.recv()
if isinstance(raw, bytes):
try:
raw = raw.decode("utf-8")
except UnicodeDecodeError as error:
raise RawCdpError("received non-UTF-8 binary CDP frame") from error
payload = json.loads(raw)
if not isinstance(payload, dict):
raise RawCdpError(f"unexpected CDP payload: {payload!r}")
record = RecordedCdpMessage(
sequence=self.current_sequence + 1,
received_monotonic=time.perf_counter(),
received_epoch=time.time(),
payload=payload,
)
response_id = payload.get("id")
future = self._pending.pop(response_id, None) if isinstance(response_id, int) else None
async with self._condition:
self._messages.append(record)
self._condition.notify_all()
if future is not None and not future.done():
future.set_result(record)
except asyncio.CancelledError:
if not self._closing:
failure = RawCdpConnectionClosed("CDP receiver was cancelled")
except BaseException as error:
if not self._closing:
failure = RawCdpConnectionClosed(
f"CDP WebSocket receiver stopped: {type(error).__name__}: {error}"
)
finally:
if failure is not None:
self._closed_error = failure
pending = list(self._pending.values())
self._pending.clear()
for future in pending:
if not future.done():
future.set_exception(failure)
async with self._condition:
self._condition.notify_all()
async def connect_routed_raw_cdp(endpoint: str) -> RoutedRawCdpClient:
websocket_url = await discover_websocket_url(endpoint)
websocket = await websockets.connect(
websocket_url,
open_timeout=5,
proxy=None,
max_size=None,
)
client = RoutedRawCdpClient(websocket)
client.start()
return client
moli-benchmark/moli_benchmark/render_compare.py
+603
View File
@@ -0,0 +1,603 @@
from __future__ import annotations
import html.parser
import os
import re
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Any
from .artifacts import write_csv, write_json, write_text
from .config import REPO_ROOT, clear_proxy_env
from .process import ProcessResult
from .stats import summarize
from .synthetic_compare import WEBFETCH_TARGETS, target_metadata
from .top_sites import (
DEFAULT_TOP_SITES_MIN_BODY_BYTES,
DEFAULT_TOP_SITES_PARALLELISM,
DEFAULT_TOP_SITES_PROFILE,
DEFAULT_TOP_SITES_SOURCE,
TOP_SITES_PROFILES,
_classify,
_ok_categories,
_run_top_sites_target,
_top_sites_target_metadata,
load_top_sites_entries,
resolve_top_sites_source,
)
DEFAULT_RENDER_COMPARE_BASELINE = "chrome"
DEFAULT_RENDER_COMPARE_NGRAM_SIZE = 4
DEFAULT_RENDER_COMPARE_MATCH_THRESHOLD = 0.65
DEFAULT_RENDER_COMPARE_PARTIAL_THRESHOLD = 0.35
DEFAULT_RENDER_COMPARE_KEY_HIT_THRESHOLD = 0.70
DEFAULT_RENDER_COMPARE_PARTIAL_KEY_HIT_THRESHOLD = 0.40
DEFAULT_RENDER_COMPARE_MIN_BASELINE_TEXT_CHARS = 500
DEFAULT_RENDER_COMPARE_KEY_PHRASES = 12
_TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
_PHRASE_SPLIT_RE = re.compile(r"[\n\r。！？!?；;：:]+")
class _VisibleTextExtractor(html.parser.HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self._skip_depth = 0
self.parts: list[str] = []
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
if tag.lower() in {"script", "style", "noscript", "svg", "template"}:
self._skip_depth += 1
def handle_endtag(self, tag: str) -> None:
if tag.lower() in {"script", "style", "noscript", "svg", "template"} and self._skip_depth:
self._skip_depth -= 1
def handle_data(self, data: str) -> None:
if self._skip_depth:
return
text = " ".join(data.split())
if text:
self.parts.append(text)
def _decode_output(stdout: bytes) -> str:
return stdout.decode("utf-8", errors="replace")
def extract_visible_text(stdout: bytes) -> dict[str, Any]:
html_text = _decode_output(stdout)
title = ""
match = _TITLE_RE.search(html_text)
if match:
title = re.sub(r"\s+", " ", match.group(1)).strip()
parser = _VisibleTextExtractor()
try:
parser.feed(html_text)
except html.parser.HTMLParseError:
pass
visible_text = " ".join(parser.parts)
return {
"title": title,
"visible_text": visible_text,
"visible_text_length": len(visible_text),
"visible_text_sample": visible_text[:500],
}
def _compact_text(text: str) -> str:
lowered = text.lower()
return "".join(ch for ch in lowered if ch.isalnum() or "\u4e00" <= ch <= "\u9fff")
def _ngrams(text: str, size: int) -> set[str]:
compact = _compact_text(text)
if len(compact) < size:
return {compact} if compact else set()
return {compact[index : index + size] for index in range(0, len(compact) - size + 1)}
def _ngram_containment(baseline_text: str, target_text: str, size: int) -> float:
baseline_grams = _ngrams(baseline_text, size)
if not baseline_grams:
return 0.0
target_grams = _ngrams(target_text, size)
return len(baseline_grams & target_grams) / len(baseline_grams)
def _ngram_jaccard(baseline_text: str, target_text: str, size: int) -> float:
baseline_grams = _ngrams(baseline_text, size)
target_grams = _ngrams(target_text, size)
union = baseline_grams | target_grams
if not union:
return 0.0
return len(baseline_grams & target_grams) / len(union)
def _key_phrases(text: str, limit: int) -> list[str]:
phrases: list[str] = []
seen: set[str] = set()
for part in _PHRASE_SPLIT_RE.split(text):
normalized = " ".join(part.split())
compact = _compact_text(normalized)
if len(compact) < 12:
continue
phrase = normalized[:80]
key = _compact_text(phrase)
if key in seen:
continue
seen.add(key)
phrases.append(phrase)
if len(phrases) >= limit:
break
return phrases
def _phrase_hit_rate(phrases: list[str], text: str) -> float:
if not phrases:
return 0.0
compact_text = _compact_text(text)
hits = sum(1 for phrase in phrases if _compact_text(phrase) in compact_text)
return hits / len(phrases)
def _capped_ratio(numerator: int, denominator: int) -> float:
if denominator <= 0:
return 0.0
return min(1.0, max(0.0, numerator / denominator))
def _quality_score(*, containment: float, key_hit_rate: float, text_ratio: float) -> float:
score = 100.0 * (0.55 * containment + 0.35 * key_hit_rate + 0.10 * text_ratio)
return round(max(0.0, min(100.0, score)), 2)
def compare_to_baseline(
*,
baseline_stdout: bytes,
baseline_category: str,
target_stdout: bytes,
target_stderr: bytes,
target_category: str,
ngram_size: int = DEFAULT_RENDER_COMPARE_NGRAM_SIZE,
match_threshold: float = DEFAULT_RENDER_COMPARE_MATCH_THRESHOLD,
partial_threshold: float = DEFAULT_RENDER_COMPARE_PARTIAL_THRESHOLD,
key_hit_threshold: float = DEFAULT_RENDER_COMPARE_KEY_HIT_THRESHOLD,
partial_key_hit_threshold: float = DEFAULT_RENDER_COMPARE_PARTIAL_KEY_HIT_THRESHOLD,
min_baseline_text_chars: int = DEFAULT_RENDER_COMPARE_MIN_BASELINE_TEXT_CHARS,
key_phrase_limit: int = DEFAULT_RENDER_COMPARE_KEY_PHRASES,
) -> dict[str, Any]:
baseline_snapshot = extract_visible_text(baseline_stdout)
target_snapshot = extract_visible_text(target_stdout)
baseline_text = str(baseline_snapshot["visible_text"])
target_text = str(target_snapshot["visible_text"])
phrases = _key_phrases(baseline_text, key_phrase_limit)
containment = _ngram_containment(baseline_text, target_text, ngram_size)
jaccard = _ngram_jaccard(baseline_text, target_text, ngram_size)
key_hit_rate = _phrase_hit_rate(phrases, target_text)
raw_text = _decode_output(target_stdout) + "\n" + target_stderr.decode("utf-8", errors="replace")
raw_ngram_containment = _ngram_containment(baseline_text, raw_text, ngram_size)
raw_key_hit_rate = _phrase_hit_rate(phrases, raw_text)
baseline_visible_length = int(baseline_snapshot["visible_text_length"])
target_visible_length = int(target_snapshot["visible_text_length"])
visible_text_ratio = _capped_ratio(target_visible_length, baseline_visible_length)
raw_text_ratio = _capped_ratio(len(_compact_text(raw_text)), max(len(_compact_text(baseline_text)), 1))
render_quality_score = _quality_score(
containment=containment,
key_hit_rate=key_hit_rate,
text_ratio=visible_text_ratio,
)
raw_content_score = _quality_score(
containment=raw_ngram_containment,
key_hit_rate=raw_key_hit_rate,
text_ratio=raw_text_ratio,
)
if baseline_category not in _ok_categories():
category = "baseline-unusable"
elif baseline_visible_length < min_baseline_text_chars:
category = "baseline-thin"
elif (
(raw_ngram_containment >= partial_threshold or raw_key_hit_rate >= partial_key_hit_threshold)
and containment < partial_threshold
and key_hit_rate < partial_key_hit_threshold
):
category = "state-only-content"
elif target_category not in _ok_categories():
category = target_category
elif containment >= match_threshold and key_hit_rate >= key_hit_threshold:
category = "render-match"
elif containment >= max(match_threshold, 0.80):
category = "render-match"
elif containment >= partial_threshold or key_hit_rate >= partial_key_hit_threshold:
category = "render-partial"
else:
category = "render-mismatch"
excluded = category.startswith("baseline-")
return {
"category": category,
"excluded": excluded,
"ok": category == "render-match",
"baseline_title": baseline_snapshot["title"],
"target_title": target_snapshot["title"],
"baseline_visible_text_length": baseline_visible_length,
"target_visible_text_length": target_visible_length,
"visible_text_ratio": visible_text_ratio,
"baseline_visible_text_sample": baseline_snapshot["visible_text_sample"],
"target_visible_text_sample": target_snapshot["visible_text_sample"],
"ngram_size": ngram_size,
"ngram_containment": containment,
"ngram_jaccard": jaccard,
"key_phrase_count": len(phrases),
"key_phrase_hit_rate": key_hit_rate,
"raw_ngram_containment": raw_ngram_containment,
"raw_key_phrase_hit_rate": raw_key_hit_rate,
"render_quality_score": render_quality_score,
"raw_content_score": raw_content_score,
"key_phrases": phrases,
}
def _domain_to_url(domain: str) -> str:
if domain.startswith("http://") or domain.startswith("https://"):
return domain
return f"https://{domain}"
def _execute_fetch(
*,
target: str,
info: dict[str, Any],
rank: int,
domain: str,
timeout_seconds: float,
min_body_bytes: int,
proc_env: dict[str, str],
) -> dict[str, Any]:
metadata = _top_sites_target_metadata(target)
url = _domain_to_url(domain)
if not info.get("available") or not info.get("path"):
return {
"target": target,
**metadata,
"rank": rank,
"domain": domain,
"url": url,
"category": "target-unavailable",
"ok": False,
"returncode": None,
"timed_out": False,
"elapsed_ms": None,
"stdout_bytes": 0,
"stderr_bytes": 0,
"stdout": b"",
"stderr": b"",
"peak_pss_bytes": None,
}
result: ProcessResult = _run_top_sites_target(
target=target,
binary=Path(info["path"]),
url=url,
timeout_seconds=timeout_seconds,
proc_env=proc_env,
)
category = _classify(result.stdout, result.stderr, result.returncode, result.timed_out, min_body_bytes)
return {
"target": target,
**metadata,
"rank": rank,
"domain": domain,
"url": url,
"category": category,
"ok": category in _ok_categories(),
"returncode": result.returncode,
"timed_out": result.timed_out,
"elapsed_ms": result.elapsed_ms,
"stdout_bytes": len(result.stdout),
"stderr_bytes": len(result.stderr),
"stdout": result.stdout,
"stderr": result.stderr,
"stderr_tail": result.stderr[-512:].decode("utf-8", errors="replace"),
"peak_pss_bytes": result.resources.get("peak_pss_bytes"),
"peak_rss_bytes": result.resources.get("peak_rss_bytes"),
}
def _baseline_site_row(
*,
baseline: dict[str, Any],
baseline_target: str,
min_baseline_text_chars: int,
) -> dict[str, Any]:
snapshot = extract_visible_text(baseline["stdout"])
visible_length = int(snapshot["visible_text_length"])
if str(baseline["category"]) not in _ok_categories():
category = "baseline-unusable"
usable = False
elif visible_length < min_baseline_text_chars:
category = "baseline-thin"
usable = False
else:
category = "baseline-usable"
usable = True
return {
"rank": baseline["rank"],
"domain": baseline["domain"],
"url": baseline["url"],
"baseline_target": baseline_target,
"baseline_fetch_category": baseline["category"],
"category": category,
"usable": usable,
"baseline_title": snapshot["title"],
"baseline_visible_text_length": visible_length,
"baseline_visible_text_sample": snapshot["visible_text_sample"],
"baseline_elapsed_ms": baseline["elapsed_ms"],
"baseline_returncode": baseline["returncode"],
"baseline_timed_out": baseline["timed_out"],
"baseline_stdout_bytes": baseline["stdout_bytes"],
"baseline_stderr_bytes": baseline["stderr_bytes"],
"baseline_peak_pss_bytes": baseline.get("peak_pss_bytes"),
"baseline_peak_rss_bytes": baseline.get("peak_rss_bytes"),
"baseline_stderr_tail": baseline.get("stderr_tail", ""),
}
def _write_compare_artifact(*, suite_dir: Path, row: dict[str, Any], target_result: dict[str, Any]) -> str:
failures_dir = suite_dir / "failures"
domain = re.sub(r"[^A-Za-z0-9._-]+", "_", str(row["domain"])).strip("._-") or "site"
name = f"{row['target']}-rank{int(row['rank']):03d}-{domain[:80]}"
json_path = failures_dir / f"{name}.json"
write_json(json_path, row)
if target_result.get("stderr"):
write_text(json_path.with_suffix(".stderr.txt"), target_result["stderr"][-16 * 1024 :].decode("utf-8", errors="replace"))
if target_result.get("stdout"):
write_text(json_path.with_suffix(".stdout.html"), target_result["stdout"][-32 * 1024 :].decode("utf-8", errors="replace"))
return str(json_path.relative_to(suite_dir))
def run_render_compare_suite(
*,
output_dir: Path,
target_matrix: dict[str, Any],
targets: tuple[str, ...],
baseline_target: str = DEFAULT_RENDER_COMPARE_BASELINE,
profile: str = DEFAULT_TOP_SITES_PROFILE,
list_path: Path | None = None,
source: str = DEFAULT_TOP_SITES_SOURCE,
timeout_seconds: float = 30.0,
gate_target: str = "moli",
parallelism: int = DEFAULT_TOP_SITES_PARALLELISM,
min_body_bytes: int = DEFAULT_TOP_SITES_MIN_BODY_BYTES,
limit_override: int | None = None,
ngram_size: int = DEFAULT_RENDER_COMPARE_NGRAM_SIZE,
match_threshold: float = DEFAULT_RENDER_COMPARE_MATCH_THRESHOLD,
partial_threshold: float = DEFAULT_RENDER_COMPARE_PARTIAL_THRESHOLD,
key_hit_threshold: float = DEFAULT_RENDER_COMPARE_KEY_HIT_THRESHOLD,
partial_key_hit_threshold: float = DEFAULT_RENDER_COMPARE_PARTIAL_KEY_HIT_THRESHOLD,
min_baseline_text_chars: int = DEFAULT_RENDER_COMPARE_MIN_BASELINE_TEXT_CHARS,
) -> dict[str, Any]:
selected_targets = tuple(dict.fromkeys(targets))
all_targets = tuple(dict.fromkeys((baseline_target, *selected_targets)))
unknown_targets = [target for target in all_targets if target not in WEBFETCH_TARGETS]
if unknown_targets:
raise RuntimeError(f"unknown webfetch target(s): {', '.join(unknown_targets)}")
if baseline_target not in WEBFETCH_TARGETS:
raise RuntimeError(f"unknown baseline target: {baseline_target}")
if gate_target not in WEBFETCH_TARGETS:
raise RuntimeError(f"unknown gate target: {gate_target}")
if gate_target not in selected_targets:
raise RuntimeError("render-compare gate target must be one of the selected --target values")
if profile not in TOP_SITES_PROFILES:
raise RuntimeError(f"unknown top-sites profile `{profile}`; expected one of {sorted(TOP_SITES_PROFILES)}")
limit = int(limit_override if limit_override is not None else TOP_SITES_PROFILES[profile]["limit"])
if limit <= 0:
raise RuntimeError("render-compare limit must be positive")
if parallelism <= 0:
raise RuntimeError("render-compare parallelism must be positive")
suite_dir = output_dir / "render-compare"
resolved_source, primary_path = resolve_top_sites_source(source, list_path)
entries_all, list_source_labels = load_top_sites_entries(resolved_source, list_path)
entries = entries_all[:limit]
proc_env = clear_proxy_env(os.environ)
baseline_rows: list[dict[str, Any]] = []
baseline_info = target_matrix.get(target_metadata(baseline_target)["binary_key"], {})
with ThreadPoolExecutor(max_workers=parallelism) as executor:
future_to_spec = {
executor.submit(
_execute_fetch,
target=baseline_target,
info=baseline_info,
rank=rank,
domain=domain,
timeout_seconds=timeout_seconds,
min_body_bytes=min_body_bytes,
proc_env=proc_env,
): (baseline_target, rank, domain)
for rank, domain in entries
}
for future in as_completed(future_to_spec):
row = future.result()
row["stage"] = "baseline"
baseline_rows.append(row)
baseline_rows.sort(key=lambda row: int(row["rank"]))
baseline_site_rows = [
_baseline_site_row(
baseline=row,
baseline_target=baseline_target,
min_baseline_text_chars=min_baseline_text_chars,
)
for row in baseline_rows
]
baseline_usable_entries = [
(int(row["rank"]), str(row["domain"]))
for row in baseline_site_rows
if row["usable"]
]
target_fetch_rows: list[dict[str, Any]] = []
target_job_specs = [
(target, target_matrix.get(target_metadata(target)["binary_key"], {}), rank, domain)
for target in selected_targets
if target != baseline_target
for rank, domain in baseline_usable_entries
]
if target_job_specs:
with ThreadPoolExecutor(max_workers=parallelism) as executor:
future_to_spec = {
executor.submit(
_execute_fetch,
target=target,
info=info,
rank=rank,
domain=domain,
timeout_seconds=timeout_seconds,
min_body_bytes=min_body_bytes,
proc_env=proc_env,
): (target, rank, domain)
for target, info, rank, domain in target_job_specs
}
for future in as_completed(future_to_spec):
row = future.result()
row["stage"] = "target"
target_fetch_rows.append(row)
target_fetch_rows.sort(key=lambda row: (int(row["rank"]), row["target"]))
fetch_rows = [*baseline_rows, *target_fetch_rows]
by_site_target = {(int(row["rank"]), str(row["domain"]), row["target"]): row for row in fetch_rows}
compare_rows: list[dict[str, Any]] = []
baseline_site_by_key = {(int(row["rank"]), str(row["domain"])): row for row in baseline_site_rows}
for rank, domain in baseline_usable_entries:
baseline = by_site_target[(rank, domain, baseline_target)]
for target in selected_targets:
target_result = by_site_target[(rank, domain, target)] if target != baseline_target else baseline
comparison = compare_to_baseline(
baseline_stdout=baseline["stdout"],
baseline_category=str(baseline["category"]),
target_stdout=target_result["stdout"],
target_stderr=target_result["stderr"],
target_category=str(target_result["category"]),
ngram_size=ngram_size,
match_threshold=match_threshold,
partial_threshold=partial_threshold,
key_hit_threshold=key_hit_threshold,
partial_key_hit_threshold=partial_key_hit_threshold,
min_baseline_text_chars=min_baseline_text_chars,
)
row = {
"target": target,
**target_metadata(target),
"rank": rank,
"domain": domain,
"url": _domain_to_url(domain),
"baseline_target": baseline_target,
"baseline_category": baseline["category"],
"baseline_gate_category": baseline_site_by_key[(rank, domain)]["category"],
"target_fetch_category": target_result["category"],
"category": comparison["category"],
"ok": comparison["ok"],
"elapsed_ms": target_result["elapsed_ms"],
"baseline_elapsed_ms": baseline["elapsed_ms"],
"returncode": target_result["returncode"],
"timed_out": target_result["timed_out"],
"stdout_bytes": target_result["stdout_bytes"],
"stderr_bytes": target_result["stderr_bytes"],
"peak_pss_bytes": target_result["peak_pss_bytes"],
"peak_rss_bytes": target_result.get("peak_rss_bytes"),
"stderr_tail": target_result.get("stderr_tail", ""),
**comparison,
}
if not row["ok"] and not row["excluded"]:
row["failure_artifact"] = _write_compare_artifact(
suite_dir=suite_dir,
row={key: value for key, value in row.items() if key != "key_phrases"},
target_result=target_result,
)
compare_rows.append(row)
csv_rows = [{key: value for key, value in row.items() if key != "key_phrases"} for row in compare_rows]
fetch_json_rows = [
{key: value for key, value in row.items() if key not in {"stdout", "stderr"}}
for row in fetch_rows
]
baseline_json_rows = [
{key: value for key, value in row.items() if key not in {"stdout", "stderr"}}
for row in baseline_rows
]
gate_failures = sum(
1
for row in compare_rows
if row["target"] == gate_target and not row["excluded"] and not row["ok"]
)
total_failures = sum(1 for row in compare_rows if not row["excluded"] and not row["ok"])
summary: dict[str, Any] = {
"suite": "render-compare",
"profile": profile,
"limit": limit,
"source": resolved_source,
"list_source": str(primary_path.relative_to(REPO_ROOT)) if primary_path.is_relative_to(REPO_ROOT) else str(primary_path),
"list_sources": list_source_labels,
"site_count": len(entries),
"baseline_site_count": len(baseline_site_rows),
"evaluated_site_count": len(baseline_usable_entries),
"baseline_excluded_site_count": len(baseline_site_rows) - len(baseline_usable_entries),
"baseline_target": baseline_target,
"targets": {},
"timeout_seconds": timeout_seconds,
"parallelism": parallelism,
"min_body_bytes": min_body_bytes,
"min_baseline_text_chars": min_baseline_text_chars,
"ngram_size": ngram_size,
"match_threshold": match_threshold,
"partial_threshold": partial_threshold,
"key_hit_threshold": key_hit_threshold,
"partial_key_hit_threshold": partial_key_hit_threshold,
"gate_target": gate_target,
"gate_failures": gate_failures,
"total_failures": total_failures,
"excluded_rows": (len(baseline_site_rows) - len(baseline_usable_entries)) * len(selected_targets),
"skipped_target_rows": (len(baseline_site_rows) - len(baseline_usable_entries)) * len(selected_targets),
}
baseline_categories: dict[str, int] = {}
for row in baseline_site_rows:
baseline_categories[str(row["category"])] = baseline_categories.get(str(row["category"]), 0) + 1
summary["baseline_categories"] = baseline_categories
for target in selected_targets:
target_rows = [row for row in compare_rows if row["target"] == target]
evaluated_rows = [row for row in target_rows if not row["excluded"]]
categories: dict[str, int] = {}
for row in target_rows:
categories[str(row["category"])] = categories.get(str(row["category"]), 0) + 1
summary["targets"][target] = {
**_top_sites_target_metadata(target),
"sites": len(entries),
"evaluated_sites": len(evaluated_rows),
"excluded_sites": len(baseline_site_rows) - len(baseline_usable_entries),
"passes": sum(1 for row in evaluated_rows if row["ok"]),
"failures": sum(1 for row in evaluated_rows if not row["ok"]),
"categories": categories,
"elapsed_ms": summarize(row["elapsed_ms"] for row in evaluated_rows if row.get("elapsed_ms") is not None),
"ngram_containment": summarize(row["ngram_containment"] for row in evaluated_rows),
"raw_ngram_containment": summarize(row["raw_ngram_containment"] for row in evaluated_rows),
"key_phrase_hit_rate": summarize(row["key_phrase_hit_rate"] for row in evaluated_rows),
"raw_key_phrase_hit_rate": summarize(row["raw_key_phrase_hit_rate"] for row in evaluated_rows),
"render_quality_score": summarize(row["render_quality_score"] for row in evaluated_rows),
"raw_content_score": summarize(row["raw_content_score"] for row in evaluated_rows),
"peak_pss_bytes": summarize(row["peak_pss_bytes"] for row in evaluated_rows if row.get("peak_pss_bytes") is not None),
"peak_rss_bytes": summarize(row["peak_rss_bytes"] for row in evaluated_rows if row.get("peak_rss_bytes") is not None),
}
write_csv(suite_dir / "raw-runs.csv", csv_rows)
write_json(suite_dir / "runs.json", compare_rows)
write_json(suite_dir / "fetch-runs.json", fetch_json_rows)
write_json(suite_dir / "baseline-runs.json", baseline_json_rows)
write_json(suite_dir / "baseline-sites.json", baseline_site_rows)
write_csv(suite_dir / "baseline-sites.csv", baseline_site_rows)
write_json(suite_dir / "summary.json", summary)
return summary
moli-benchmark/moli_benchmark/report_diff.py
+149
View File
@@ -0,0 +1,149 @@
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
def _summary_path(path: Path) -> Path:
if path.is_dir():
return path / "summary.json"
return path
def load_baseline_summary(path: Path) -> dict[str, Any]:
summary_path = _summary_path(path)
try:
data = json.loads(summary_path.read_text(encoding="utf-8"))
except FileNotFoundError as error:
raise RuntimeError(f"missing baseline report summary: {summary_path}") from error
except json.JSONDecodeError as error:
raise RuntimeError(f"invalid baseline report summary JSON: {summary_path}") from error
if not isinstance(data, dict):
raise RuntimeError(f"baseline report summary must be an object: {summary_path}")
suites = data.get("suites")
if suites is not None and not isinstance(suites, list):
raise RuntimeError(f"baseline report summary suites must be a list: {summary_path}")
return data
def _suite_key(summary: dict[str, Any]) -> str | None:
suite = summary.get("suite")
return str(suite) if suite else None
def _by_suite(summaries: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
return {key: summary for summary in summaries if (key := _suite_key(summary)) is not None}
def _int_value(value: Any) -> int | None:
if value is None:
return None
try:
return int(value)
except (TypeError, ValueError):
return None
def _total_failures(summary: dict[str, Any] | None) -> int | None:
if summary is None:
return None
return _int_value(summary.get("total_failures"))
def _gate_failures(summary: dict[str, Any] | None) -> int | None:
if summary is None:
return None
return _int_value(summary.get("gate_failures", summary.get("total_failures")))
def _case_count(summary: dict[str, Any] | None) -> int | None:
if summary is None:
return None
cases = summary.get("cases")
if isinstance(cases, (dict, list, tuple)):
return len(cases)
if cases is not None:
return _int_value(cases)
runs = summary.get("runs")
if isinstance(runs, (dict, list, tuple)):
return len(runs)
return None
def _delta(current: int | None, baseline: int | None) -> int | None:
if current is None or baseline is None:
return None
return current - baseline
def _suite_status(current: dict[str, Any] | None, baseline: dict[str, Any] | None) -> str:
if baseline is None:
return "added"
if current is None:
return "removed"
fields = (
_total_failures(current) != _total_failures(baseline),
_gate_failures(current) != _gate_failures(baseline),
_case_count(current) != _case_count(baseline),
)
return "changed" if any(fields) else "unchanged"
def build_report_diff(
*,
current_summaries: list[dict[str, Any]],
baseline_summary: dict[str, Any],
baseline_path: Path,
) -> dict[str, Any]:
baseline_summaries = [summary for summary in baseline_summary.get("suites", []) if isinstance(summary, dict)]
current_by_suite = _by_suite(current_summaries)
baseline_by_suite = _by_suite(baseline_summaries)
rows: list[dict[str, Any]] = []
for suite in sorted(current_by_suite.keys() | baseline_by_suite.keys()):
current = current_by_suite.get(suite)
baseline = baseline_by_suite.get(suite)
current_total = _total_failures(current)
baseline_total = _total_failures(baseline)
current_gate = _gate_failures(current)
baseline_gate = _gate_failures(baseline)
current_cases = _case_count(current)
baseline_cases = _case_count(baseline)
rows.append(
{
"suite": suite,
"status": _suite_status(current, baseline),
"baseline_total_failures": baseline_total,
"current_total_failures": current_total,
"total_failures_delta": _delta(current_total, baseline_total),
"baseline_gate_failures": baseline_gate,
"current_gate_failures": current_gate,
"gate_failures_delta": _delta(current_gate, baseline_gate),
"baseline_cases": baseline_cases,
"current_cases": current_cases,
"cases_delta": _delta(current_cases, baseline_cases),
}
)
baseline_total_failures = sum(_total_failures(summary) or 0 for summary in baseline_summaries)
current_total_failures = sum(_total_failures(summary) or 0 for summary in current_summaries)
baseline_gate_failures = sum(_gate_failures(summary) or 0 for summary in baseline_summaries)
current_gate_failures = sum(_gate_failures(summary) or 0 for summary in current_summaries)
by_status = {status: sum(1 for row in rows if row["status"] == status) for status in ("added", "removed", "changed", "unchanged")}
return {
"schema_version": 1,
"baseline": str(_summary_path(baseline_path)),
"summary": {
**by_status,
"total_changes": by_status["added"] + by_status["removed"] + by_status["changed"],
"baseline_total_failures": baseline_total_failures,
"current_total_failures": current_total_failures,
"total_failures_delta": current_total_failures - baseline_total_failures,
"baseline_gate_failures": baseline_gate_failures,
"current_gate_failures": current_gate_failures,
"gate_failures_delta": current_gate_failures - baseline_gate_failures,
},
"suites": rows,
}
moli-benchmark/moli_benchmark/sampling.py
+386
View File
@@ -0,0 +1,386 @@
from __future__ import annotations
import os
import subprocess
import threading
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
def _process_tree(root_pid: int) -> list[int]:
ppid_by_pid: dict[int, int] = {}
for stat_path in Path("/proc").glob("[0-9]*/stat"):
try:
raw = stat_path.read_text(encoding="utf-8", errors="replace")
except OSError:
continue
close = raw.rfind(")")
if close < 0:
continue
parts = raw[close + 2 :].split()
if len(parts) < 2:
continue
try:
pid = int(stat_path.parent.name)
ppid = int(parts[1])
except ValueError:
continue
ppid_by_pid[pid] = ppid
result = [root_pid]
seen = {root_pid}
changed = True
while changed:
changed = False
for pid, ppid in ppid_by_pid.items():
if pid not in seen and ppid in seen:
result.append(pid)
seen.add(pid)
changed = True
return result
def _read_pss_bytes(pid: int) -> int | None:
try:
text = Path(f"/proc/{pid}/smaps_rollup").read_text(encoding="utf-8", errors="replace")
except OSError:
return None
for line in text.splitlines():
if line.startswith("Pss:"):
parts = line.split()
if len(parts) >= 2:
return int(parts[1]) * 1024
return None
def _read_rss_bytes(pid: int) -> int | None:
try:
text = Path(f"/proc/{pid}/status").read_text(encoding="utf-8", errors="replace")
except OSError:
return None
for line in text.splitlines():
if line.startswith("VmRSS:"):
parts = line.split()
if len(parts) >= 2:
return int(parts[1]) * 1024
return None
def _read_cpu_percent(pids: list[int]) -> float | None:
if not pids:
return None
try:
completed = subprocess.run(
["ps", "-o", "%cpu=", "-p", ",".join(str(pid) for pid in pids)],
capture_output=True,
text=True,
timeout=1,
check=False,
)
except (OSError, subprocess.TimeoutExpired):
return None
total = 0.0
found = False
for line in completed.stdout.splitlines():
try:
total += float(line.strip())
found = True
except ValueError:
continue
return total if found else None
def _read_thread_count(pid: int) -> int | None:
try:
text = Path(f"/proc/{pid}/status").read_text(encoding="utf-8", errors="replace")
except OSError:
return None
for line in text.splitlines():
if line.startswith("Threads:"):
parts = line.split()
if len(parts) >= 2:
return int(parts[1])
return None
def _read_fd_count(pid: int) -> int | None:
try:
return sum(1 for _ in Path(f"/proc/{pid}/fd").iterdir())
except OSError:
return None
def _read_cpu_identity(pid: int) -> tuple[tuple[int, int], int] | None:
try:
raw = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8", errors="replace")
except OSError:
return None
close = raw.rfind(")")
if close < 0:
return None
fields = raw[close + 2 :].split()
if len(fields) <= 19:
return None
try:
user_ticks = int(fields[11])
system_ticks = int(fields[12])
start_ticks = int(fields[19])
except ValueError:
return None
return (pid, start_ticks), user_ticks + system_ticks
def snapshot_resources(root_pid: int, *, include_lifetime_cpu: bool = True) -> dict[str, Any]:
pids = _process_tree(root_pid)
pss_values = [_read_pss_bytes(pid) for pid in pids]
rss_values = [_read_rss_bytes(pid) for pid in pids]
thread_values = [_read_thread_count(pid) for pid in pids]
fd_values = [_read_fd_count(pid) for pid in pids]
pss_values = [value for value in pss_values if value is not None]
rss_values = [value for value in rss_values if value is not None]
thread_values = [value for value in thread_values if value is not None]
fd_values = [value for value in fd_values if value is not None]
return {
"pids": pids,
"process_count": len(pids),
"pss_bytes": sum(pss_values) if len(pss_values) == len(pids) else None,
"pss_process_count": len(pss_values),
"rss_bytes": sum(rss_values) if len(rss_values) == len(pids) else None,
"rss_process_count": len(rss_values),
"thread_count": sum(thread_values)
if len(thread_values) == len(pids)
else None,
"thread_process_count": len(thread_values),
"fd_count": sum(fd_values) if len(fd_values) == len(pids) else None,
"fd_process_count": len(fd_values),
"cpu_percent": _read_cpu_percent(pids) if include_lifetime_cpu else None,
}
def _sanitize_cgroup_name(name: str) -> str:
return "".join(char if char.isalnum() or char in ("-", "_", ".") else "_" for char in name)
def _read_pid_cgroups(pid: int) -> list[dict[str, Any]]:
try:
lines = Path(f"/proc/{pid}/cgroup").read_text(encoding="utf-8", errors="replace").splitlines()
except OSError:
return []
entries = []
for line in lines:
parts = line.split(":", 2)
if len(parts) != 3:
continue
hierarchy, controllers, relative_path = parts
entries.append(
{
"hierarchy": hierarchy,
"controllers": [controller for controller in controllers.split(",") if controller],
"path": relative_path,
}
)
return entries
def _candidate_cgroup_roots(entry: dict[str, Any]) -> list[Path]:
relative = str(entry.get("path") or "/").lstrip("/")
controllers = entry.get("controllers") or []
roots = [Path("/sys/fs/cgroup") / relative]
for controller in controllers:
roots.append(Path("/sys/fs/cgroup") / str(controller) / relative)
return roots
def collect_cgroup_artifacts(pid: int, output_dir: Path) -> dict[str, Any]:
output_dir.mkdir(parents=True, exist_ok=True)
entries = _read_pid_cgroups(pid)
try:
raw_cgroup = Path(f"/proc/{pid}/cgroup").read_text(encoding="utf-8", errors="replace")
except OSError as error:
raw_cgroup = f"unavailable: {error}\n"
proc_cgroup_path = output_dir / "proc-cgroup.txt"
proc_cgroup_path.write_text(raw_cgroup, encoding="utf-8")
files = [
"cgroup.controllers",
"cgroup.events",
"cgroup.procs",
"cgroup.stat",
"cpu.stat",
"io.stat",
"memory.current",
"memory.events",
"memory.max",
"memory.peak",
"memory.stat",
"pids.current",
"pids.max",
]
copied: list[str] = [str(proc_cgroup_path)]
roots: list[str] = []
unavailable: list[str] = []
seen_roots: set[Path] = set()
for entry_index, entry in enumerate(entries):
for root in _candidate_cgroup_roots(entry):
if root in seen_roots or not root.exists():
continue
seen_roots.add(root)
roots.append(str(root))
label = f"{entry_index}-{_sanitize_cgroup_name(root.name or 'root')}"
root_dir = output_dir / label
root_dir.mkdir(parents=True, exist_ok=True)
for file_name in files:
source = root / file_name
target = root_dir / file_name
try:
target.write_text(source.read_text(encoding="utf-8", errors="replace"), encoding="utf-8")
copied.append(str(target))
except OSError as error:
unavailable.append(f"{source}: {error}")
if unavailable:
unavailable_path = output_dir / "unavailable.txt"
unavailable_path.write_text("\n".join(unavailable) + "\n", encoding="utf-8")
copied.append(str(unavailable_path))
return {
"pid": pid,
"entries": entries,
"roots": roots,
"artifact_dir": str(output_dir),
"artifacts": copied,
"available": bool(roots),
}
@dataclass
class ResourceSampler:
root_pid: int
interval_seconds: float = 0.1
samples: list[dict[str, Any]] = field(default_factory=list)
_stop: threading.Event = field(default_factory=threading.Event)
_thread: threading.Thread | None = None
_started_monotonic: float | None = None
_previous_monotonic: float | None = None
_previous_cpu: dict[tuple[int, int], int] = field(default_factory=dict)
_observer_error: str | None = None
def start(self) -> None:
if self.interval_seconds <= 0:
raise ValueError("resource sample interval must be positive")
if self._thread is not None:
raise RuntimeError("resource sampler already started")
self._started_monotonic = time.perf_counter()
self._thread = threading.Thread(target=self._run, name="moli-resource-sampler", daemon=True)
self._thread.start()
def stop(self) -> dict[str, Any]:
self._stop.set()
if self._thread is not None:
self._thread.join(timeout=2)
return self.summary()
def _run(self) -> None:
try:
next_capture = time.perf_counter()
while not self._stop.is_set():
capture_started = time.perf_counter()
sample = snapshot_resources(self.root_pid, include_lifetime_cpu=False)
captured = time.perf_counter()
identities = dict(
identity
for pid in sample.get("pids", [])
if (identity := _read_cpu_identity(int(pid))) is not None
)
cpu_percent = None
if self._previous_monotonic is not None:
elapsed = captured - self._previous_monotonic
delta_ticks = sum(
max(0, ticks - self._previous_cpu[identity])
for identity, ticks in identities.items()
if identity in self._previous_cpu
)
if elapsed > 0:
try:
clock_ticks = float(os.sysconf("SC_CLK_TCK"))
except (OSError, ValueError):
clock_ticks = 100.0
if clock_ticks > 0:
cpu_percent = delta_ticks / clock_ticks / elapsed * 100.0
self._previous_monotonic = captured
self._previous_cpu = identities
sample["cpu_percent"] = cpu_percent
sample["timestamp"] = time.time()
sample["elapsed_ms"] = (
(captured - self._started_monotonic) * 1000.0
if self._started_monotonic is not None
else 0.0
)
sample["capture_duration_ms"] = (captured - capture_started) * 1000.0
sample["kind"] = "periodic"
self.samples.append(sample)
next_capture += self.interval_seconds
now = time.perf_counter()
if next_capture <= now:
missed = int((now - next_capture) / self.interval_seconds) + 1
next_capture += missed * self.interval_seconds
remaining = max(0.0, next_capture - now)
self._stop.wait(remaining)
except BaseException as error:
self._observer_error = f"{type(error).__name__}: {error}"
def summary(self) -> dict[str, Any]:
pss = [sample["pss_bytes"] for sample in self.samples if sample.get("pss_bytes") is not None]
rss = [sample["rss_bytes"] for sample in self.samples if sample.get("rss_bytes") is not None]
cpu = [sample["cpu_percent"] for sample in self.samples if sample.get("cpu_percent") is not None]
process_counts = [sample["process_count"] for sample in self.samples if sample.get("process_count") is not None]
thread_counts = [sample["thread_count"] for sample in self.samples if sample.get("thread_count") is not None]
fd_counts = [sample["fd_count"] for sample in self.samples if sample.get("fd_count") is not None]
capture_durations = [
float(sample["capture_duration_ms"])
for sample in self.samples
if sample.get("capture_duration_ms") is not None
]
intervals = [
float(current["elapsed_ms"]) - float(previous["elapsed_ms"])
for previous, current in zip(self.samples, self.samples[1:])
if previous.get("elapsed_ms") is not None and current.get("elapsed_ms") is not None
]
complete_pss_samples = sum(
1
for sample in self.samples
if sample.get("pss_bytes") is not None
and sample.get("pss_process_count") == sample.get("process_count")
)
return {
"sample_count": len(self.samples),
"interval_seconds": self.interval_seconds,
"peak_pss_bytes": max(pss) if pss else None,
"peak_rss_bytes": max(rss) if rss else None,
"peak_cpu_percent": max(cpu) if cpu else None,
"peak_process_count": max(process_counts) if process_counts else None,
"peak_thread_count": max(thread_counts) if thread_counts else None,
"peak_fd_count": max(fd_counts) if fd_counts else None,
"average_cpu_percent": sum(cpu) / len(cpu) if cpu else None,
"capture_duration_ms": {
"average": sum(capture_durations) / len(capture_durations)
if capture_durations
else None,
"max": max(capture_durations) if capture_durations else None,
},
"late_sample_count": sum(
1 for interval in intervals if interval > self.interval_seconds * 1000.0 * 1.5
),
"observed_interval_ms": {
"average": sum(intervals) / len(intervals) if intervals else None,
"max": max(intervals) if intervals else None,
},
"pss_complete_samples": complete_pss_samples,
"pss_complete": bool(self.samples)
and complete_pss_samples == len(self.samples),
"observer_error": self._observer_error,
"thread_alive_after_stop": bool(
self._thread is not None and self._thread.is_alive()
),
"sampling_method": "procfs_process_tree_cpu_ticks_smaps_rollup",
}
moli-benchmark/moli_benchmark/serve.py
+215
View File
@@ -0,0 +1,215 @@
from __future__ import annotations
import os
import subprocess
import threading
import time
import urllib.error
import urllib.request
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from .config import REPO_ROOT, ReservedPort, clear_proxy_env, reserve_port
from .process import read_time_verbose_file, time_verbose_command
from .sampling import ResourceSampler, collect_cgroup_artifacts
MAX_SERVE_LOG_LINES = 4096
@dataclass
class ServeHandle:
process: subprocess.Popen[bytes]
endpoint: str
port: int
logs: list[str]
sampler: ResourceSampler
port_lease: ReservedPort
ready_ms: float | None = None
time_verbose_path: Path | None = None
cgroup_artifact_dir: Path | None = None
log_threads: list[threading.Thread] | None = None
def _append_log(logs: list[str], line: str) -> None:
logs.append(line)
overflow = len(logs) - MAX_SERVE_LOG_LINES
if overflow > 0:
del logs[:overflow]
def _read_available(stream: Any, label: str, logs: list[str]) -> None:
if stream is None:
return
try:
while True:
line = stream.readline()
if not line:
return
_append_log(logs, f"{label}: {line.decode('utf-8', errors='replace').rstrip()}")
except OSError:
return
def _drain_stream(stream: Any, label: str, logs: list[str]) -> None:
if stream is None:
return
try:
while True:
line = stream.readline()
if not line:
return
_append_log(logs, f"{label}: {line.decode('utf-8', errors='replace').rstrip()}")
except OSError:
return
def _start_log_drain_threads(process: subprocess.Popen[bytes], logs: list[str]) -> list[threading.Thread]:
threads: list[threading.Thread] = []
for label, stream in (("stdout", process.stdout), ("stderr", process.stderr)):
thread = threading.Thread(
target=_drain_stream,
args=(stream, label, logs),
name=f"moli-benchmark-{label}-drain",
daemon=True,
)
thread.start()
threads.append(thread)
return threads
def _join_log_drain_threads(threads: list[threading.Thread] | None) -> None:
for thread in threads or []:
thread.join(timeout=1)
def _terminate_process(process: subprocess.Popen[bytes]) -> bool:
if process.poll() is not None:
return True
try:
process.terminate()
except OSError:
return process.poll() is not None
try:
process.wait(timeout=2)
return True
except subprocess.TimeoutExpired:
try:
process.kill()
except OSError:
return process.poll() is not None
try:
process.wait(timeout=2)
return True
except subprocess.TimeoutExpired:
return False
def probe_url(url: str, timeout_seconds: float = 0.5) -> bool:
try:
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
with opener.open(url, timeout=timeout_seconds) as response:
response.read()
return True
except (urllib.error.URLError, TimeoutError, OSError):
return False
def start_moli_serve(
moli_bin: Path,
timeout_seconds: float,
*,
time_verbose_path: Path | None = None,
cgroup_artifact_dir: Path | None = None,
) -> ServeHandle:
logs: list[str] = []
reserved_port = reserve_port()
try:
port = reserved_port.port
endpoint = f"http://127.0.0.1:{port}"
command = time_verbose_command(
[
str(moli_bin),
"serve",
"--host",
"127.0.0.1",
"--port",
str(port),
],
time_verbose_path,
)
reserved_port.release_socket()
process = subprocess.Popen(
command,
cwd=REPO_ROOT,
env=clear_proxy_env(os.environ),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
start_new_session=True,
)
log_threads = _start_log_drain_threads(process, logs)
except BaseException:
reserved_port.close()
raise
sampler = ResourceSampler(process.pid)
sampler.start()
handle = ServeHandle(
process=process,
endpoint=endpoint,
port=port,
logs=logs,
sampler=sampler,
port_lease=reserved_port,
time_verbose_path=time_verbose_path,
cgroup_artifact_dir=cgroup_artifact_dir,
log_threads=log_threads,
)
started = time.perf_counter()
deadline = started + timeout_seconds
version_url = endpoint + "/json/version"
while time.perf_counter() < deadline:
if process.poll() is not None:
_join_log_drain_threads(handle.log_threads)
sampler.stop()
reserved_port.close()
raise RuntimeError(f"moli serve exited early with {process.returncode}: {'; '.join(logs[-20:])}")
if probe_url(version_url):
handle.ready_ms = (time.perf_counter() - started) * 1000.0
return handle
time.sleep(0.025)
process_exited = _terminate_process(process)
sampler.stop()
reserved_port.close()
if process_exited:
_join_log_drain_threads(handle.log_threads)
else:
logs.append("process did not exit after SIGKILL; skipped pipe drain")
raise RuntimeError(f"timed out waiting for moli serve at {version_url}: {'; '.join(logs[-20:])}")
def stop_moli_serve(handle: ServeHandle | None) -> dict[str, Any]:
if handle is None:
return {}
cgroup = (
collect_cgroup_artifacts(handle.process.pid, handle.cgroup_artifact_dir)
if handle.cgroup_artifact_dir is not None
else None
)
process_exited = _terminate_process(handle.process)
if process_exited:
_join_log_drain_threads(handle.log_threads)
else:
handle.logs.append("process did not exit after SIGKILL; skipped pipe drain")
resources = handle.sampler.stop()
handle.port_lease.close()
return {
"returncode": handle.process.returncode,
"resources": resources,
"time_verbose": (
read_time_verbose_file(handle.time_verbose_path) if handle.time_verbose_path is not None else None
),
"cgroup": cgroup,
"log_tail": handle.logs[-40:],
}
moli-benchmark/moli_benchmark/startup.py
+948
View File
@@ -0,0 +1,948 @@
from __future__ import annotations
import asyncio
import gzip
import hashlib
import os
import shutil
import subprocess
import tarfile
import tempfile
import time
from pathlib import Path
from typing import Any
from urllib.parse import quote
from .artifacts import write_csv, write_json
from .config import REPO_ROOT, clear_proxy_env
from .process import run_process
from .sampling import snapshot_resources
from .serve import start_moli_serve, stop_moli_serve
from .stats import summarize
from .synthetic import SyntheticServer
from .versions import sha256_file
STARTUP_PROFILES = ("smoke", "formal")
FORMAL_STARTUP_RUNS = 10
FORMAL_STARTUP_WARM_PAGES = 10
FORMAL_STARTUP_IDLE_SECONDS = (1.0, 5.0, 30.0)
def _hash_output(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def _cache_mode_fields(process_cache_mode: str, kernel_cache_mode: str) -> dict[str, str]:
return {
"process_cache_mode": process_cache_mode,
"kernel_cache_mode": kernel_cache_mode,
}
def _drop_os_cache_artifact(suite_dir: Path, case: str, run_id: int) -> dict[str, Any]:
artifact_dir = suite_dir / "cache" / f"{case}-run-{run_id}"
artifact_dir.mkdir(parents=True, exist_ok=True)
artifact_path = artifact_dir / "drop-caches.txt"
if os.geteuid() != 0:
artifact_path.write_text("unavailable: root privileges required for drop_caches\n", encoding="utf-8")
return {
"requested": True,
"ok": False,
"mode": "warm-kernel",
"error": "root privileges required for drop_caches",
"artifact_path": str(artifact_path),
}
try:
subprocess.run(["sync"], check=False)
Path("/proc/sys/vm/drop_caches").write_text("3\n", encoding="utf-8")
except OSError as error:
artifact_path.write_text(f"unavailable: {error}\n", encoding="utf-8")
return {
"requested": True,
"ok": False,
"mode": "warm-kernel",
"error": str(error),
"artifact_path": str(artifact_path),
}
artifact_path.write_text("ok: wrote 3 to /proc/sys/vm/drop_caches\n", encoding="utf-8")
return {
"requested": True,
"ok": True,
"mode": "cold-os-cache",
"artifact_path": str(artifact_path),
}
def _compressed_binary_size(binary: Path) -> int:
with tempfile.TemporaryDirectory(prefix="moli-benchmark-size-") as temp_dir:
archive = Path(temp_dir) / "moli.tar.gz"
with tarfile.open(archive, "w:gz") as tar:
tar.add(binary, arcname=binary.name)
return archive.stat().st_size
def _ldd_dependency_paths(binary: Path) -> tuple[list[Path], str | None]:
ldd_bin = shutil.which("ldd")
if ldd_bin is None:
return [], "ldd executable unavailable"
result = subprocess.run(
[ldd_bin, str(binary)],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
check=False,
)
output = result.stdout + result.stderr
if result.returncode != 0 and "not a dynamic executable" not in output:
return [], output.strip() or f"ldd exited with {result.returncode}"
paths: list[Path] = []
seen: set[Path] = set()
for raw_line in output.splitlines():
line = raw_line.strip()
if not line or line.startswith("linux-vdso"):
continue
candidate = ""
if "=>" in line:
candidate = line.split("=>", 1)[1].strip().split(" ", 1)[0]
else:
candidate = line.split(" ", 1)[0]
if not candidate.startswith("/"):
continue
path = Path(candidate)
if path.exists() and path not in seen:
seen.add(path)
paths.append(path)
return paths, None
def _add_path_to_tar(tar: tarfile.TarFile, source: Path, arcname: str) -> None:
tar.add(source, arcname=arcname, recursive=False)
if source.is_symlink():
resolved = source.resolve()
if resolved.exists():
tar.add(resolved, arcname=str(resolved).lstrip("/"), recursive=False)
def _build_minimal_rootfs_image_artifacts(binary: Path, suite_dir: Path) -> dict[str, Any]:
image_dir = suite_dir / "image-size"
image_dir.mkdir(parents=True, exist_ok=True)
dependencies, dependency_error = _ldd_dependency_paths(binary)
rootfs_tar = image_dir / "moli-rootfs.tar"
rootfs_targz = image_dir / "moli-rootfs.tar.gz"
manifest = {
"format": "minimal-rootfs-tar",
"binary": str(binary),
"binary_arcname": "usr/local/bin/moli",
"dependencies": [str(path) for path in dependencies],
"dependency_error": dependency_error,
"note": "Daemonless startup/deploy size artifact. Container image measurement is intentionally out of scope.",
}
write_json(image_dir / "manifest.json", manifest)
with tarfile.open(rootfs_tar, "w") as tar:
_add_path_to_tar(tar, binary, "usr/local/bin/moli")
for dependency in dependencies:
_add_path_to_tar(tar, dependency, str(dependency).lstrip("/"))
with rootfs_tar.open("rb") as source, rootfs_targz.open("wb") as target:
with gzip.GzipFile(fileobj=target, mode="wb", compresslevel=1, mtime=0) as gzip_file:
shutil.copyfileobj(source, gzip_file)
return {
"format": manifest["format"],
"uncompressed_bytes": rootfs_tar.stat().st_size,
"compressed_bytes": rootfs_targz.stat().st_size,
"dependency_count": len(dependencies),
"dependency_error": dependency_error,
"rootfs_tar_path": str(rootfs_tar),
"rootfs_targz_path": str(rootfs_targz),
"manifest_path": str(image_dir / "manifest.json"),
}
def _stripped_binary_size(binary: Path) -> tuple[int | None, str | None]:
strip_bin = shutil.which("strip")
if strip_bin is None:
return None, "strip executable unavailable"
with tempfile.TemporaryDirectory(prefix="moli-benchmark-strip-") as temp_dir:
candidate = Path(temp_dir) / binary.name
shutil.copy2(binary, candidate)
result = subprocess.run(
[strip_bin, "--strip-unneeded", str(candidate)],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=False,
)
if result.returncode != 0:
stderr = result.stderr.decode("utf-8", errors="replace").strip()
return None, stderr or f"strip exited with {result.returncode}"
return candidate.stat().st_size, None
def _time_verbose_path(suite_dir: Path, case: str, run_id: int) -> Path:
return suite_dir / "time" / f"{case}-run-{run_id}.time.txt"
def _cgroup_artifact_dir(suite_dir: Path, case: str, run_id: int) -> Path:
return suite_dir / "cgroup" / f"{case}-run-{run_id}"
def _time_verbose_row_fields(time_verbose: dict[str, Any] | None) -> dict[str, Any]:
if not time_verbose:
return {
"time_max_rss_bytes": None,
"time_elapsed_seconds": None,
"time_user_seconds": None,
"time_system_seconds": None,
"time_raw_path": None,
}
return {
"time_max_rss_bytes": time_verbose.get("max_rss_bytes"),
"time_elapsed_seconds": time_verbose.get("elapsed_seconds"),
"time_user_seconds": time_verbose.get("user_seconds"),
"time_system_seconds": time_verbose.get("system_seconds"),
"time_raw_path": time_verbose.get("raw_path"),
}
def _startup_formal_gate_rows(
*,
profile: str,
runs: int,
include_cdp_first_page: bool,
include_cdp_warm_pages: bool,
cdp_warm_pages: int,
idle_seconds: tuple[float, ...],
total_failures: int,
) -> list[dict[str, Any]]:
idle_set = {float(seconds) for seconds in idle_seconds}
requirements = [
{
"gate": "profile",
"ok": profile == "formal",
"actual": profile,
"required": "formal",
"failure_kind": "profile-requirement",
},
{
"gate": "runs",
"ok": runs >= FORMAL_STARTUP_RUNS,
"actual": runs,
"required": FORMAL_STARTUP_RUNS,
"failure_kind": "profile-requirement",
},
{
"gate": "cdp-first-page",
"ok": include_cdp_first_page,
"actual": include_cdp_first_page,
"required": True,
"failure_kind": "profile-requirement",
},
{
"gate": "cdp-warm-pages",
"ok": include_cdp_warm_pages and cdp_warm_pages >= FORMAL_STARTUP_WARM_PAGES,
"actual": {"enabled": include_cdp_warm_pages, "pages": cdp_warm_pages if include_cdp_warm_pages else 0},
"required": {"enabled": True, "pages": FORMAL_STARTUP_WARM_PAGES},
"failure_kind": "profile-requirement",
},
{
"gate": "idle-footprint",
"ok": all(required in idle_set for required in FORMAL_STARTUP_IDLE_SECONDS),
"actual": sorted(idle_set),
"required": list(FORMAL_STARTUP_IDLE_SECONDS),
"failure_kind": "profile-requirement",
},
{
"gate": "workload-failures",
"ok": total_failures == 0,
"actual": total_failures,
"required": 0,
"failure_kind": "workload-failure",
},
]
return requirements
async def _cdp_first_page_flow(endpoint: str, timeout_seconds: float) -> dict[str, Any]:
from .raw_cdp import connect_raw_cdp
client = await connect_raw_cdp(endpoint)
try:
target_command_id = await client.send("Target.createTarget", {"url": "about:blank"})
target_response, seen = await client.recv_until_id(target_command_id, timeout=timeout_seconds)
target_id = target_response.get("result", {}).get("targetId")
if not isinstance(target_id, str) or not target_id:
raise RuntimeError(f"missing targetId in {target_response}")
attach_id = await client.send("Target.attachToTarget", {"targetId": target_id, "flatten": True})
attach_response, attach_seen = await client.recv_until_id(attach_id, timeout=timeout_seconds)
seen.extend(attach_seen)
session_id = attach_response.get("result", {}).get("sessionId")
if not isinstance(session_id, str) or not session_id:
raise RuntimeError(f"missing sessionId in {attach_response}")
for method in ("Page.enable", "Runtime.enable"):
command_id = await client.send(method, session_id=session_id)
_, method_seen = await client.recv_until_id(command_id, timeout=timeout_seconds)
seen.extend(method_seen)
html = "<!doctype html><title>cdp-first-page</title><body data-benchmark-status='ok'>ok</body>"
navigate_id = await client.send(
"Page.navigate",
{"url": "data:text/html;charset=utf-8," + quote(html, safe=":/;=,'")},
session_id=session_id,
)
_, navigate_seen = await client.recv_until_id(navigate_id, timeout=timeout_seconds)
seen.extend(navigate_seen)
expression = """
new Promise(resolve => {
const deadline = Date.now() + 10000;
function tick() {
if (document.body && document.body.dataset.benchmarkStatus === 'ok') {
resolve(document.title + ':' + document.body.textContent.trim());
} else if (Date.now() > deadline) {
resolve('timeout');
} else {
setTimeout(tick, 10);
}
}
tick();
})
"""
evaluate_id = await client.send(
"Runtime.evaluate",
{"expression": expression, "awaitPromise": True, "returnByValue": True},
session_id=session_id,
)
evaluate_response, evaluate_seen = await client.recv_until_id(evaluate_id, timeout=timeout_seconds + 1)
seen.extend(evaluate_seen)
value = evaluate_response.get("result", {}).get("result", {}).get("value")
return {
"value": value,
"ok": value == "cdp-first-page:ok",
"messages": len(seen),
"command_count": client.command_count,
}
finally:
await client.websocket.close()
async def _cdp_create_page(client: Any, timeout_seconds: float, page_index: int) -> dict[str, Any]:
started = time.perf_counter()
seen: list[dict[str, Any]] = []
html = f"<!doctype html><title>cdp-warm-page-{page_index}</title><body data-benchmark-status='ok'>ok</body>"
target_command_id = await client.send("Target.createTarget", {"url": "about:blank"})
target_response, target_seen = await client.recv_until_id(target_command_id, timeout=timeout_seconds)
seen.extend(target_seen)
target_id = target_response.get("result", {}).get("targetId")
if not isinstance(target_id, str) or not target_id:
raise RuntimeError(f"missing targetId in {target_response}")
try:
attach_id = await client.send("Target.attachToTarget", {"targetId": target_id, "flatten": True})
attach_response, attach_seen = await client.recv_until_id(attach_id, timeout=timeout_seconds)
seen.extend(attach_seen)
session_id = attach_response.get("result", {}).get("sessionId")
if not isinstance(session_id, str) or not session_id:
raise RuntimeError(f"missing sessionId in {attach_response}")
for method in ("Page.enable", "Runtime.enable"):
command_id = await client.send(method, session_id=session_id)
_, method_seen = await client.recv_until_id(command_id, timeout=timeout_seconds)
seen.extend(method_seen)
navigate_id = await client.send(
"Page.navigate",
{"url": "data:text/html;charset=utf-8," + quote(html, safe=":/;=,'")},
session_id=session_id,
)
_, navigate_seen = await client.recv_until_id(navigate_id, timeout=timeout_seconds)
seen.extend(navigate_seen)
evaluate_id = await client.send(
"Runtime.evaluate",
{
"expression": "document.body && document.body.dataset.benchmarkStatus === 'ok'",
"returnByValue": True,
},
session_id=session_id,
)
evaluate_response, evaluate_seen = await client.recv_until_id(evaluate_id, timeout=timeout_seconds)
seen.extend(evaluate_seen)
ok = evaluate_response.get("result", {}).get("result", {}).get("value") is True
return {
"page_index": page_index,
"ok": ok,
"elapsed_ms": (time.perf_counter() - started) * 1000.0,
"target_id": target_id,
"messages": len(seen),
}
finally:
try:
close_id = await client.send("Target.closeTarget", {"targetId": target_id})
await client.recv_until_id(close_id, timeout=3)
except Exception:
pass
async def _cdp_warm_pages_flow(endpoint: str, timeout_seconds: float, pages: int) -> dict[str, Any]:
from .raw_cdp import connect_raw_cdp
client = await connect_raw_cdp(endpoint)
try:
page_results = []
for page_index in range(1, pages + 1):
page_results.append(await _cdp_create_page(client, timeout_seconds, page_index))
return {
"ok": all(bool(page.get("ok")) for page in page_results),
"pages": page_results,
"command_count": client.command_count,
}
finally:
await client.websocket.close()
def _run_cdp_first_page(
moli_bin: Path,
timeout_seconds: float,
*,
time_verbose_path: Path | None = None,
cgroup_artifact_dir: Path | None = None,
) -> tuple[dict[str, Any], dict[str, Any]]:
handle = None
started = time.perf_counter()
try:
handle = start_moli_serve(
moli_bin,
timeout_seconds,
time_verbose_path=time_verbose_path,
cgroup_artifact_dir=cgroup_artifact_dir,
)
result = asyncio.run(_cdp_first_page_flow(handle.endpoint, timeout_seconds))
elapsed_ms = (time.perf_counter() - started) * 1000.0
stop_details = stop_moli_serve(handle)
row = {
"case": "cdp-first-page",
"ok": bool(result.get("ok")),
"elapsed_ms": elapsed_ms,
"serve_ready_ms": handle.ready_ms,
"peak_pss_bytes": stop_details.get("resources", {}).get("peak_pss_bytes"),
"peak_rss_bytes": stop_details.get("resources", {}).get("peak_rss_bytes"),
"peak_cpu_percent": stop_details.get("resources", {}).get("peak_cpu_percent"),
"peak_process_count": stop_details.get("resources", {}).get("peak_process_count"),
"peak_thread_count": stop_details.get("resources", {}).get("peak_thread_count"),
"peak_fd_count": stop_details.get("resources", {}).get("peak_fd_count"),
"cdp_value": result.get("value"),
"cdp_messages": result.get("messages"),
"cdp_command_count": result.get("command_count"),
**_time_verbose_row_fields(stop_details.get("time_verbose")),
}
return row, {**row, "serve": stop_details}
except Exception as error:
elapsed_ms = (time.perf_counter() - started) * 1000.0
stop_details = stop_moli_serve(handle)
row = {
"case": "cdp-first-page",
"ok": False,
"elapsed_ms": elapsed_ms,
"error": str(error),
**_time_verbose_row_fields(stop_details.get("time_verbose")),
}
return row, {**row, "serve": stop_details}
def _run_cdp_warm_pages(
moli_bin: Path,
timeout_seconds: float,
*,
pages: int,
time_verbose_path: Path | None = None,
cgroup_artifact_dir: Path | None = None,
) -> tuple[dict[str, Any], dict[str, Any]]:
handle = None
started = time.perf_counter()
try:
handle = start_moli_serve(
moli_bin,
timeout_seconds,
time_verbose_path=time_verbose_path,
cgroup_artifact_dir=cgroup_artifact_dir,
)
result = asyncio.run(_cdp_warm_pages_flow(handle.endpoint, timeout_seconds, pages))
elapsed_ms = (time.perf_counter() - started) * 1000.0
page_results = [page for page in result.get("pages", []) if isinstance(page, dict)]
page_elapsed = [
float(page["elapsed_ms"])
for page in page_results
if page.get("ok") and page.get("elapsed_ms") is not None
]
page_elapsed_summary = summarize(page_elapsed)
stop_details = stop_moli_serve(handle)
row = {
"case": "cdp-warm-pages",
"ok": bool(result.get("ok")) and len(page_results) == pages,
"elapsed_ms": elapsed_ms,
"serve_ready_ms": handle.ready_ms,
"cdp_pages": pages,
"cdp_page_passes": sum(1 for page in page_results if page.get("ok")),
"cdp_page_elapsed_p50_ms": page_elapsed_summary.get("p50"),
"cdp_page_elapsed_p95_ms": page_elapsed_summary.get("p95"),
"peak_pss_bytes": stop_details.get("resources", {}).get("peak_pss_bytes"),
"peak_rss_bytes": stop_details.get("resources", {}).get("peak_rss_bytes"),
"peak_cpu_percent": stop_details.get("resources", {}).get("peak_cpu_percent"),
"peak_process_count": stop_details.get("resources", {}).get("peak_process_count"),
"peak_thread_count": stop_details.get("resources", {}).get("peak_thread_count"),
"peak_fd_count": stop_details.get("resources", {}).get("peak_fd_count"),
"cdp_command_count": result.get("command_count"),
**_time_verbose_row_fields(stop_details.get("time_verbose")),
}
return row, {**row, "serve": stop_details, "pages": page_results}
except Exception as error:
elapsed_ms = (time.perf_counter() - started) * 1000.0
stop_details = stop_moli_serve(handle)
row = {
"case": "cdp-warm-pages",
"ok": False,
"elapsed_ms": elapsed_ms,
"cdp_pages": pages,
"error": str(error),
**_time_verbose_row_fields(stop_details.get("time_verbose")),
}
return row, {**row, "serve": stop_details}
def run_startup_suite(
*,
moli_bin: Path,
output_dir: Path,
profile: str = "smoke",
runs: int,
timeout_seconds: float,
include_cdp_first_page: bool = False,
include_cdp_warm_pages: bool = False,
cdp_warm_pages: int = 10,
idle_seconds: tuple[float, ...] = (),
drop_os_cache: bool = False,
) -> dict[str, Any]:
if profile not in STARTUP_PROFILES:
raise RuntimeError(f"startup profile must be one of: {', '.join(STARTUP_PROFILES)}")
if include_cdp_warm_pages and cdp_warm_pages < 1:
raise RuntimeError("cdp_warm_pages must be at least 1 when warm page startup is enabled")
suite_dir = output_dir / "startup"
rows: list[dict[str, Any]] = []
details: list[dict[str, Any]] = []
cache_events: list[dict[str, Any]] = []
def prepare_cache(case: str, run_id: int) -> dict[str, str]:
if not drop_os_cache:
return _cache_mode_fields("cold-process", "warm-kernel")
event = _drop_os_cache_artifact(suite_dir, case, run_id)
cache_events.append({"case": case, "run": run_id, **event})
return _cache_mode_fields("cold-process", str(event["mode"]))
stripped_bytes, stripped_error = _stripped_binary_size(moli_bin)
binary_row = {
"case": "binary-size",
"run": 1,
"ok": True,
"elapsed_ms": 0.0,
"binary_bytes": moli_bin.stat().st_size,
"stripped_binary_bytes": stripped_bytes,
"stripped_available": stripped_bytes is not None,
"stripped_error": stripped_error,
"tar_gz_bytes": _compressed_binary_size(moli_bin),
"sha256": sha256_file(moli_bin),
**_cache_mode_fields("artifact", "not-applicable"),
}
rows.append(binary_row)
details.append(binary_row)
image_started = time.perf_counter()
image_artifacts = _build_minimal_rootfs_image_artifacts(moli_bin, suite_dir)
image_row = {
"case": "image-size",
"run": 1,
"ok": True,
"elapsed_ms": (time.perf_counter() - image_started) * 1000.0,
"image_format": image_artifacts["format"],
"image_uncompressed_bytes": image_artifacts["uncompressed_bytes"],
"image_compressed_bytes": image_artifacts["compressed_bytes"],
"image_dependency_count": image_artifacts["dependency_count"],
"image_dependency_error": image_artifacts["dependency_error"],
"image_rootfs_tar_path": image_artifacts["rootfs_tar_path"],
"image_rootfs_targz_path": image_artifacts["rootfs_targz_path"],
"image_manifest_path": image_artifacts["manifest_path"],
**_cache_mode_fields("artifact", "not-applicable"),
}
rows.append(image_row)
details.append({**image_row, "image": image_artifacts})
for run_id in range(1, runs + 1):
handle = None
cache_fields = prepare_cache("serve-ready", run_id)
try:
handle = start_moli_serve(
moli_bin,
timeout_seconds,
time_verbose_path=_time_verbose_path(suite_dir, "serve-ready", run_id),
cgroup_artifact_dir=_cgroup_artifact_dir(suite_dir, "serve-ready", run_id),
)
stop_details = stop_moli_serve(handle)
row = {
"case": "serve-ready",
"run": run_id,
"ok": True,
"elapsed_ms": handle.ready_ms,
"peak_pss_bytes": stop_details.get("resources", {}).get("peak_pss_bytes"),
"peak_rss_bytes": stop_details.get("resources", {}).get("peak_rss_bytes"),
"peak_cpu_percent": stop_details.get("resources", {}).get("peak_cpu_percent"),
"peak_process_count": stop_details.get("resources", {}).get("peak_process_count"),
"peak_thread_count": stop_details.get("resources", {}).get("peak_thread_count"),
"peak_fd_count": stop_details.get("resources", {}).get("peak_fd_count"),
**cache_fields,
**_time_verbose_row_fields(stop_details.get("time_verbose")),
}
rows.append(row)
details.append({**row, "serve": stop_details})
except Exception as error:
stop_details = stop_moli_serve(handle)
row = {
"case": "serve-ready",
"run": run_id,
"ok": False,
"elapsed_ms": None,
"error": str(error),
**cache_fields,
**_time_verbose_row_fields(stop_details.get("time_verbose")),
}
rows.append(row)
details.append({**row, "serve": stop_details})
for run_id in range(1, runs + 1):
cache_fields = prepare_cache("cli-fetch-aboutblank", run_id)
result = run_process(
[
str(moli_bin),
"fetch",
"--dump",
"html",
"--wait-until",
"load",
"--timeout",
str(int(timeout_seconds * 1000)),
"about:blank",
],
cwd=REPO_ROOT,
timeout_seconds=timeout_seconds,
env=clear_proxy_env(os.environ),
time_verbose_path=_time_verbose_path(suite_dir, "cli-fetch-aboutblank", run_id),
cgroup_artifact_dir=_cgroup_artifact_dir(suite_dir, "cli-fetch-aboutblank", run_id),
)
row = {
"case": "cli-fetch-aboutblank",
"run": run_id,
"ok": result.returncode == 0 and not result.timed_out,
"elapsed_ms": result.elapsed_ms,
"returncode": result.returncode,
"timed_out": result.timed_out,
"peak_pss_bytes": result.resources.get("peak_pss_bytes"),
"peak_rss_bytes": result.resources.get("peak_rss_bytes"),
"peak_cpu_percent": result.resources.get("peak_cpu_percent"),
"peak_process_count": result.resources.get("peak_process_count"),
"peak_thread_count": result.resources.get("peak_thread_count"),
"peak_fd_count": result.resources.get("peak_fd_count"),
"output_sha256": _hash_output(result.output_digest_material()),
**cache_fields,
**_time_verbose_row_fields(result.time_verbose),
}
rows.append(row)
details.append({**row, "process": result.json_summary(include_output=True)})
if include_cdp_first_page:
for run_id in range(1, runs + 1):
cache_fields = prepare_cache("cdp-first-page", run_id)
row, detail = _run_cdp_first_page(
moli_bin,
timeout_seconds,
time_verbose_path=_time_verbose_path(suite_dir, "cdp-first-page", run_id),
cgroup_artifact_dir=_cgroup_artifact_dir(suite_dir, "cdp-first-page", run_id),
)
row["run"] = run_id
row.update(cache_fields)
detail["run"] = run_id
detail.update(cache_fields)
rows.append(row)
details.append(detail)
if include_cdp_warm_pages:
for run_id in range(1, runs + 1):
cache_fields = prepare_cache("cdp-warm-pages", run_id)
row, detail = _run_cdp_warm_pages(
moli_bin,
timeout_seconds,
pages=cdp_warm_pages,
time_verbose_path=_time_verbose_path(suite_dir, "cdp-warm-pages", run_id),
cgroup_artifact_dir=_cgroup_artifact_dir(suite_dir, "cdp-warm-pages", run_id),
)
row["run"] = run_id
row.update(cache_fields)
detail["run"] = run_id
detail.update(cache_fields)
rows.append(row)
details.append(detail)
for seconds in idle_seconds:
for run_id in range(1, runs + 1):
handle = None
started = time.perf_counter()
cache_fields = prepare_cache(f"idle-footprint-{seconds:g}s", run_id)
try:
handle = start_moli_serve(
moli_bin,
timeout_seconds,
time_verbose_path=_time_verbose_path(suite_dir, f"idle-footprint-{seconds:g}s", run_id),
cgroup_artifact_dir=_cgroup_artifact_dir(suite_dir, f"idle-footprint-{seconds:g}s", run_id),
)
time.sleep(seconds)
snapshot = snapshot_resources(handle.process.pid)
elapsed_ms = (time.perf_counter() - started) * 1000.0
stop_details = stop_moli_serve(handle)
row = {
"case": f"idle-footprint-{seconds:g}s",
"run": run_id,
"ok": True,
"elapsed_ms": elapsed_ms,
"idle_seconds": seconds,
"serve_ready_ms": handle.ready_ms,
"pss_bytes": snapshot.get("pss_bytes"),
"rss_bytes": snapshot.get("rss_bytes"),
"cpu_percent": snapshot.get("cpu_percent"),
"process_count": snapshot.get("process_count"),
"thread_count": snapshot.get("thread_count"),
"fd_count": snapshot.get("fd_count"),
"peak_pss_bytes": stop_details.get("resources", {}).get("peak_pss_bytes"),
"peak_rss_bytes": stop_details.get("resources", {}).get("peak_rss_bytes"),
"peak_cpu_percent": stop_details.get("resources", {}).get("peak_cpu_percent"),
"peak_process_count": stop_details.get("resources", {}).get("peak_process_count"),
"peak_thread_count": stop_details.get("resources", {}).get("peak_thread_count"),
"peak_fd_count": stop_details.get("resources", {}).get("peak_fd_count"),
**cache_fields,
**_time_verbose_row_fields(stop_details.get("time_verbose")),
}
rows.append(row)
details.append({**row, "snapshot": snapshot, "serve": stop_details})
except Exception as error:
elapsed_ms = (time.perf_counter() - started) * 1000.0
stop_details = stop_moli_serve(handle)
row = {
"case": f"idle-footprint-{seconds:g}s",
"run": run_id,
"ok": False,
"elapsed_ms": elapsed_ms,
"idle_seconds": seconds,
"error": str(error),
**cache_fields,
**_time_verbose_row_fields(stop_details.get("time_verbose")),
}
rows.append(row)
details.append({**row, "serve": stop_details})
with SyntheticServer() as server:
for run_id in range(1, runs + 1):
cache_fields = prepare_cache("cli-fetch-local-js", run_id)
result = run_process(
[
str(moli_bin),
"fetch",
"--dump",
"html",
"--wait-until",
"done",
"--wait-script",
"document.querySelector('[data-benchmark-status=\"ok\"]') !== null",
"--timeout",
str(int(timeout_seconds * 1000)),
f"{server.base_url}/dynamic-script",
],
cwd=REPO_ROOT,
timeout_seconds=timeout_seconds + 1,
env=clear_proxy_env(os.environ),
time_verbose_path=_time_verbose_path(suite_dir, "cli-fetch-local-js", run_id),
cgroup_artifact_dir=_cgroup_artifact_dir(suite_dir, "cli-fetch-local-js", run_id),
)
row = {
"case": "cli-fetch-local-js",
"run": run_id,
"ok": result.returncode == 0 and not result.timed_out,
"elapsed_ms": result.elapsed_ms,
"returncode": result.returncode,
"timed_out": result.timed_out,
"peak_pss_bytes": result.resources.get("peak_pss_bytes"),
"peak_rss_bytes": result.resources.get("peak_rss_bytes"),
"peak_cpu_percent": result.resources.get("peak_cpu_percent"),
"peak_process_count": result.resources.get("peak_process_count"),
"peak_thread_count": result.resources.get("peak_thread_count"),
"peak_fd_count": result.resources.get("peak_fd_count"),
"output_sha256": _hash_output(result.output_digest_material()),
**cache_fields,
**_time_verbose_row_fields(result.time_verbose),
}
rows.append(row)
details.append({**row, "process": result.json_summary(include_output=not row["ok"])})
cases = sorted({str(row["case"]) for row in rows})
elapsed_by_case: dict[str, list[float]] = {case: [] for case in cases}
pss_by_case: dict[str, list[float]] = {case: [] for case in cases}
rss_by_case: dict[str, list[float]] = {case: [] for case in cases}
process_by_case: dict[str, list[float]] = {case: [] for case in cases}
thread_by_case: dict[str, list[float]] = {case: [] for case in cases}
fd_by_case: dict[str, list[float]] = {case: [] for case in cases}
time_rss_by_case: dict[str, list[float]] = {case: [] for case in cases}
for row in rows:
case = str(row["case"])
if row.get("ok") and row.get("elapsed_ms") is not None:
elapsed_by_case.setdefault(case, []).append(float(row["elapsed_ms"]))
for target, keys in (
(pss_by_case, ("pss_bytes", "peak_pss_bytes")),
(rss_by_case, ("rss_bytes", "peak_rss_bytes")),
(process_by_case, ("process_count", "peak_process_count")),
(thread_by_case, ("thread_count", "peak_thread_count")),
(fd_by_case, ("fd_count", "peak_fd_count")),
(time_rss_by_case, ("time_max_rss_bytes",)),
):
for key in keys:
if row.get("ok") and row.get(key) is not None:
target.setdefault(case, []).append(float(row[key]))
break
time_verbose_entries = []
cgroup_entries = []
for detail in details:
process_time = (
detail.get("process", {}).get("time_verbose")
if isinstance(detail.get("process"), dict)
else None
)
process_cgroup = detail.get("process", {}).get("cgroup") if isinstance(detail.get("process"), dict) else None
serve_time = (
detail.get("serve", {}).get("time_verbose")
if isinstance(detail.get("serve"), dict)
else None
)
serve_cgroup = detail.get("serve", {}).get("cgroup") if isinstance(detail.get("serve"), dict) else None
for entry in (process_time, serve_time):
if isinstance(entry, dict):
time_verbose_entries.append(entry)
for entry in (process_cgroup, serve_cgroup):
if isinstance(entry, dict):
cgroup_entries.append(entry)
total_failures = sum(1 for row in rows if not row.get("ok"))
formal_gate_rows = _startup_formal_gate_rows(
profile=profile,
runs=runs,
include_cdp_first_page=include_cdp_first_page,
include_cdp_warm_pages=include_cdp_warm_pages,
cdp_warm_pages=cdp_warm_pages,
idle_seconds=tuple(float(seconds) for seconds in idle_seconds),
total_failures=total_failures,
)
gate_failures = sum(1 for row in formal_gate_rows if row.get("ok") is not True)
summary = {
"suite": "startup",
"profile": profile,
"runs": runs,
"timeout_seconds": timeout_seconds,
"include_cdp_first_page": include_cdp_first_page,
"include_cdp_warm_pages": include_cdp_warm_pages,
"cdp_warm_pages": cdp_warm_pages if include_cdp_warm_pages else 0,
"idle_seconds": list(idle_seconds),
"drop_os_cache": drop_os_cache,
"cache_events": cache_events,
"cache_artifacts": [
event.get("artifact_path") for event in cache_events if isinstance(event.get("artifact_path"), str)
],
"time_verbose_available": any(entry.get("available") for entry in time_verbose_entries),
"time_verbose_artifacts": [
entry.get("raw_path") for entry in time_verbose_entries if isinstance(entry.get("raw_path"), str)
],
"cgroup_available": any(entry.get("available") for entry in cgroup_entries),
"cgroup_artifacts": [
artifact
for entry in cgroup_entries
for artifact in entry.get("artifacts", [])
if isinstance(artifact, str)
],
"cases": {
case: {
"elapsed_ms": summarize(values),
"pss_bytes": summarize(pss_by_case.get(case, [])),
"rss_bytes": summarize(rss_by_case.get(case, [])),
"process_count": summarize(process_by_case.get(case, [])),
"thread_count": summarize(thread_by_case.get(case, [])),
"fd_count": summarize(fd_by_case.get(case, [])),
"time_max_rss_bytes": summarize(time_rss_by_case.get(case, [])),
"binary_bytes": next((row.get("binary_bytes") for row in rows if row.get("case") == case), None),
"stripped_binary_bytes": next(
(row.get("stripped_binary_bytes") for row in rows if row.get("case") == case),
None,
),
"stripped_available": next(
(row.get("stripped_available") for row in rows if row.get("case") == case),
None,
),
"tar_gz_bytes": next((row.get("tar_gz_bytes") for row in rows if row.get("case") == case), None),
"image_format": next((row.get("image_format") for row in rows if row.get("case") == case), None),
"image_uncompressed_bytes": next(
(row.get("image_uncompressed_bytes") for row in rows if row.get("case") == case),
None,
),
"image_compressed_bytes": next(
(row.get("image_compressed_bytes") for row in rows if row.get("case") == case),
None,
),
"image_dependency_count": next(
(row.get("image_dependency_count") for row in rows if row.get("case") == case),
None,
),
"cdp_page_elapsed_p50_ms": next(
(row.get("cdp_page_elapsed_p50_ms") for row in rows if row.get("case") == case),
None,
),
"cdp_page_elapsed_p95_ms": next(
(row.get("cdp_page_elapsed_p95_ms") for row in rows if row.get("case") == case),
None,
),
"cdp_pages": next((row.get("cdp_pages") for row in rows if row.get("case") == case), None),
"cdp_page_passes": next(
(row.get("cdp_page_passes") for row in rows if row.get("case") == case),
None,
),
"process_cache_modes": sorted(
{
str(row.get("process_cache_mode"))
for row in rows
if row.get("case") == case and row.get("process_cache_mode") is not None
}
),
"kernel_cache_modes": sorted(
{
str(row.get("kernel_cache_mode"))
for row in rows
if row.get("case") == case and row.get("kernel_cache_mode") is not None
}
),
"failures": sum(1 for row in rows if row.get("case") == case and not row.get("ok")),
}
for case, values in elapsed_by_case.items()
},
"formal_gate_rows": formal_gate_rows,
"gate_failures": gate_failures,
"total_failures": total_failures,
}
write_csv(suite_dir / "runs.csv", rows)
write_json(suite_dir / "runs.json", details)
write_json(suite_dir / "gate-rows.json", formal_gate_rows)
write_json(suite_dir / "summary.json", summary)
return summary
moli-benchmark/moli_benchmark/stats.py
+39
View File
@@ -0,0 +1,39 @@
from __future__ import annotations
from typing import Iterable
def percentile(values: Iterable[float], percent: float) -> float | None:
ordered = sorted(float(value) for value in values)
if not ordered:
return None
if len(ordered) == 1:
return ordered[0]
rank = (len(ordered) - 1) * (percent / 100.0)
lower = int(rank)
upper = min(lower + 1, len(ordered) - 1)
weight = rank - lower
return ordered[lower] * (1.0 - weight) + ordered[upper] * weight
def summarize(values: Iterable[float]) -> dict[str, float | int | None]:
ordered = sorted(float(value) for value in values)
if not ordered:
return {
"count": 0,
"min": None,
"max": None,
"median": None,
"p50": None,
"p90": None,
"p95": None,
}
return {
"count": len(ordered),
"min": ordered[0],
"max": ordered[-1],
"median": percentile(ordered, 50),
"p50": percentile(ordered, 50),
"p90": percentile(ordered, 90),
"p95": percentile(ordered, 95),
}
moli-benchmark/moli_benchmark/synthetic.py
+212
View File
@@ -0,0 +1,212 @@
from __future__ import annotations
import hashlib
import os
import socket
import subprocess
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Any
from urllib.parse import urlparse
from .artifacts import write_csv, write_json
from .config import REPO_ROOT, clear_proxy_env
from .process import run_process
from .stats import summarize
from .synthetic_cases import SYNTHETIC_CASES, response_for_path
class SyntheticHandler(BaseHTTPRequestHandler):
def do_GET(self) -> None:
parsed = urlparse(self.path)
response = response_for_path(parsed.path)
if response is not None:
if response.delay_seconds:
time.sleep(response.delay_seconds)
self._send(response.content_type, response.body)
return
self.send_error(404)
def log_message(self, format: str, *args: object) -> None:
return
def _send(self, content_type: str, body: bytes) -> None:
self.send_response(200)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
try:
self.wfile.write(body)
except BrokenPipeError:
return
class SyntheticServer:
def __init__(self) -> None:
self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), SyntheticHandler)
self.port = int(self.httpd.server_address[1])
self.thread = threading.Thread(target=self.httpd.serve_forever, name="moli-benchmark-fixture", daemon=True)
self.external_host = _global_ipv6_address()
self.external_httpd: ThreadingHTTPServer | None = None
self.external_port: int | None = None
self.external_thread: threading.Thread | None = None
if self.external_host is not None:
try:
self.external_httpd = _SyntheticIpv6Server((self.external_host, 0), SyntheticHandler)
self.external_port = int(self.external_httpd.server_address[1])
self.external_thread = threading.Thread(
target=self.external_httpd.serve_forever,
name="moli-benchmark-external-fixture",
daemon=True,
)
except OSError:
self.external_host = None
self.external_httpd = None
self.external_port = None
self.external_thread = None
@property
def base_url(self) -> str:
return f"http://127.0.0.1:{self.port}"
@property
def external_base_url(self) -> str | None:
if self.external_host is None or self.external_port is None:
return None
return f"http://[{self.external_host}]:{self.external_port}"
def url_for_path(self, path: str, *, external: bool = False) -> str:
base_url = self.external_base_url if external and self.external_base_url is not None else self.base_url
return f"{base_url}/{path.lstrip('/')}"
def __enter__(self) -> "SyntheticServer":
self.thread.start()
if self.external_thread is not None:
self.external_thread.start()
time.sleep(0.025)
return self
def __exit__(self, exc_type: object, exc: object, tb: object) -> None:
self.httpd.shutdown()
self.httpd.server_close()
self.thread.join(timeout=2)
if self.external_httpd is not None:
self.external_httpd.shutdown()
self.external_httpd.server_close()
if self.external_thread is not None:
self.external_thread.join(timeout=2)
class _SyntheticIpv6Server(ThreadingHTTPServer):
address_family = socket.AF_INET6
def _global_ipv6_address() -> str | None:
try:
output = subprocess.check_output(
["ip", "-o", "-6", "addr", "show", "scope", "global"],
text=True,
stderr=subprocess.DEVNULL,
timeout=2,
)
except (OSError, subprocess.SubprocessError):
return None
for line in output.splitlines():
parts = line.split()
if "inet6" not in parts:
continue
address = parts[parts.index("inet6") + 1].split("/", 1)[0]
if address and not address.lower().startswith("fe80:"):
return address
return None
def _hash_output(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def run_synthetic_suite(
*,
moli_bin: Path,
output_dir: Path,
runs: int,
timeout_seconds: float,
cases: tuple[str, ...],
concurrency: int = 1,
) -> dict[str, Any]:
unknown = [case for case in cases if case not in SYNTHETIC_CASES]
if unknown:
raise RuntimeError(f"unknown synthetic case(s): {', '.join(unknown)}")
suite_dir = output_dir / "synthetic"
rows: list[dict[str, Any]] = []
details: list[dict[str, Any]] = []
def run_one(server: SyntheticServer, case: str, run_id: int) -> tuple[dict[str, Any], dict[str, Any]]:
url = f"{server.base_url}/{case}"
result = run_process(
[
str(moli_bin),
"fetch",
"--dump",
"html",
"--wait-until",
"done",
"--wait-script",
"document.querySelector('[data-benchmark-status=\"ok\"]') !== null",
"--timeout",
str(int(timeout_seconds * 1000)),
url,
],
cwd=REPO_ROOT,
timeout_seconds=timeout_seconds + 1,
env=clear_proxy_env(os.environ),
)
output = result.output_digest_material()
ok = result.returncode == 0 and not result.timed_out and b"data-benchmark-status=\"ok\"" in result.stdout
row = {
"case": case,
"run": run_id,
"concurrency": concurrency,
"ok": ok,
"elapsed_ms": result.elapsed_ms,
"returncode": result.returncode,
"timed_out": result.timed_out,
"peak_pss_bytes": result.resources.get("peak_pss_bytes"),
"peak_cpu_percent": result.resources.get("peak_cpu_percent"),
"output_sha256": _hash_output(output),
}
return row, {**row, "url": url, "process": result.json_summary(include_output=not ok)}
with SyntheticServer() as server:
for case in cases:
with ThreadPoolExecutor(max_workers=max(1, concurrency)) as executor:
futures = [executor.submit(run_one, server, case, run_id) for run_id in range(1, runs + 1)]
for future in as_completed(futures):
row, detail = future.result()
rows.append(row)
details.append(detail)
summary = {
"suite": "synthetic",
"runs": runs,
"timeout_seconds": timeout_seconds,
"concurrency": concurrency,
"cases": {},
"total_failures": sum(1 for row in rows if not row.get("ok")),
}
for case in cases:
case_rows = [row for row in rows if row["case"] == case]
summary["cases"][case] = {
"elapsed_ms": summarize(row["elapsed_ms"] for row in case_rows if row.get("ok")),
"peak_pss_bytes": summarize(row["peak_pss_bytes"] for row in case_rows if row.get("peak_pss_bytes") is not None),
"failures": sum(1 for row in case_rows if not row.get("ok")),
}
write_csv(suite_dir / "runs.csv", rows)
write_json(suite_dir / "runs.json", details)
write_json(suite_dir / "summary.json", summary)
return summary
moli-benchmark/moli_benchmark/synthetic_case_groups/__init__.py
+1
View File
@@ -0,0 +1 @@
from __future__ import annotations
moli-benchmark/moli_benchmark/synthetic_case_groups/basic.py
+106
View File
@@ -0,0 +1,106 @@
from __future__ import annotations
BASIC_CASES = (
"static-html",
"js-xhr-fetch",
"dynamic-script",
"dom-heavy",
"storage-cookie",
"forms-events",
)
def _html(body: str, script: str = "") -> bytes:
return (
"<!doctype html><meta charset=utf-8>"
"<title>moli benchmark</title>"
f"<body>{body}<script>{script}</script></body>"
).encode("utf-8")
def _case_response(case: str) -> tuple[str, bytes, float]:
if case == "static-html":
body = "<main data-benchmark-case=static-html data-benchmark-status=ok>" + "".join(
f"<p>row {index}</p>" for index in range(200)
) + "</main>"
return "text/html; charset=utf-8", _html(body), 0.0
if case == "js-xhr-fetch":
script = """
Promise.all([
fetch('/api/a.json').then(r => r.json()),
new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open('GET', '/api/b.json');
xhr.onload = () => resolve(JSON.parse(xhr.responseText));
xhr.onerror = reject;
xhr.send();
})
]).then(([a, b]) => {
document.body.dataset.benchmarkStatus = a.ok && b.ok ? 'ok' : 'bad';
});
"""
return "text/html; charset=utf-8", _html("<main data-benchmark-case=js-xhr-fetch></main>", script), 0.0
if case == "dynamic-script":
script = """
const s = document.createElement('script');
s.src = '/dynamic-script.js';
document.head.appendChild(s);
"""
return "text/html; charset=utf-8", _html("<main data-benchmark-case=dynamic-script></main>", script), 0.0
if case == "dom-heavy":
script = """
const root = document.querySelector('main');
for (let i = 0; i < 2000; i++) {
const item = document.createElement('button');
item.textContent = 'item-' + i;
item.dataset.index = String(i);
root.appendChild(item);
}
document.body.dataset.benchmarkStatus = String(root.children.length) === '2000' ? 'ok' : 'bad';
"""
return "text/html; charset=utf-8", _html("<main data-benchmark-case=dom-heavy></main>", script), 0.0
if case == "storage-cookie":
script = """
document.cookie = 'lm_bench_cookie=1; path=/';
localStorage.setItem('lm_bench_local', '1');
sessionStorage.setItem('lm_bench_session', '1');
document.body.dataset.benchmarkStatus =
document.cookie.includes('lm_bench_cookie=1') &&
localStorage.getItem('lm_bench_local') === '1' &&
sessionStorage.getItem('lm_bench_session') === '1' ? 'ok' : 'bad';
"""
return "text/html; charset=utf-8", _html("<main data-benchmark-case=storage-cookie></main>", script), 0.0
if case == "forms-events":
script = """
const form = document.querySelector('form');
const input = document.querySelector('input');
let changed = false;
let submitted = false;
input.addEventListener('change', () => changed = true);
form.addEventListener('submit', event => {
event.preventDefault();
submitted = true;
document.body.dataset.benchmarkStatus = changed && submitted ? 'ok' : 'bad';
});
input.value = 'updated';
input.dispatchEvent(new Event('change', { bubbles: true }));
form.requestSubmit();
"""
return (
"text/html; charset=utf-8",
_html("<form data-benchmark-case=forms-events><input name=q value=initial><button>send</button></form>", script),
0.0,
)
raise KeyError(case)
def response_for_basic_path(path: str) -> tuple[str, bytes, float] | None:
case = path.strip("/")
if path == "/api/a.json" or path == "/api/b.json":
return "application/json", b'{"ok":true}', 0.0
if path == "/dynamic-script.js":
return "application/javascript", b"document.body.dataset.benchmarkStatus='ok';", 0.0
if case in BASIC_CASES:
return _case_response(case)
return None
moli-benchmark/moli_benchmark/synthetic_case_groups/dom.py
+63
View File
@@ -0,0 +1,63 @@
from __future__ import annotations
DOM_CASES = (
"custom-elements-shadow",
"message-channel",
"mutation-observer",
)
def _html(body: str, script: str = "") -> bytes:
return (
"<!doctype html><meta charset=utf-8>"
"<title>moli benchmark</title>"
f"<body>{body}<script>{script}</script></body>"
).encode("utf-8")
def _case_response(case: str) -> tuple[str, bytes, float]:
if case == "custom-elements-shadow":
script = """
customElements.define('bench-card', class extends HTMLElement {
connectedCallback() {
const root = this.attachShadow({ mode: 'open' });
root.innerHTML = '<slot></slot><span id="marker">ready</span>';
}
});
const card = document.createElement('bench-card');
card.textContent = 'payload';
document.body.appendChild(card);
const marker = card.shadowRoot && card.shadowRoot.querySelector('#marker');
document.body.dataset.benchmarkStatus = marker && marker.textContent === 'ready' ? 'ok' : 'bad';
"""
return "text/html; charset=utf-8", _html("<main data-benchmark-case=custom-elements-shadow></main>", script), 0.0
if case == "message-channel":
script = """
const channel = new MessageChannel();
channel.port1.onmessage = event => {
document.body.dataset.benchmarkStatus = event.data && event.data.value === 7 ? 'ok' : 'bad';
};
channel.port2.postMessage({ value: 7 });
"""
return "text/html; charset=utf-8", _html("<main data-benchmark-case=message-channel></main>", script), 0.0
if case == "mutation-observer":
script = """
const target = document.querySelector('main');
const observer = new MutationObserver(records => {
document.body.dataset.benchmarkStatus =
records.length === 1 && target.dataset.changed === 'yes' ? 'ok' : 'bad';
observer.disconnect();
});
observer.observe(target, { attributes: true });
target.dataset.changed = 'yes';
"""
return "text/html; charset=utf-8", _html("<main data-benchmark-case=mutation-observer></main>", script), 0.0
raise KeyError(case)
def response_for_dom_path(path: str) -> tuple[str, bytes, float] | None:
case = path.strip("/")
if case in DOM_CASES:
return _case_response(case)
return None
moli-benchmark/moli_benchmark/synthetic_case_groups/io.py
+51
View File
@@ -0,0 +1,51 @@
from __future__ import annotations
IO_CASES = (
"abort-fetch",
"blob-filereader",
)
def _html(body: str, script: str = "") -> bytes:
return (
"<!doctype html><meta charset=utf-8>"
"<title>moli benchmark</title>"
f"<body>{body}<script>{script}</script></body>"
).encode("utf-8")
def _case_response(case: str) -> tuple[str, bytes, float]:
if case == "abort-fetch":
script = """
const controller = new AbortController();
const promise = fetch('/slow.json', { signal: controller.signal });
controller.abort();
promise.then(
() => { document.body.dataset.benchmarkStatus = 'bad'; },
error => { document.body.dataset.benchmarkStatus = error && error.name === 'AbortError' ? 'ok' : 'bad'; }
);
"""
return "text/html; charset=utf-8", _html("<main data-benchmark-case=abort-fetch></main>", script), 0.0
if case == "blob-filereader":
script = """
const blob = new Blob(['hello ', 'world'], { type: 'text/plain' });
const reader = new FileReader();
reader.onload = () => {
document.body.dataset.benchmarkStatus =
reader.result === 'hello world' && blob.type === 'text/plain' ? 'ok' : 'bad';
};
reader.onerror = () => { document.body.dataset.benchmarkStatus = 'bad'; };
reader.readAsText(blob);
"""
return "text/html; charset=utf-8", _html("<main data-benchmark-case=blob-filereader></main>", script), 0.0
raise KeyError(case)
def response_for_io_path(path: str) -> tuple[str, bytes, float] | None:
case = path.strip("/")
if path == "/slow.json":
return "application/json", b'{"ok":true}', 0.2
if case in IO_CASES:
return _case_response(case)
return None
moli-benchmark/moli_benchmark/synthetic_case_groups/modules.py
+47
View File
@@ -0,0 +1,47 @@
from __future__ import annotations
MODULE_CASES = (
"module-graph",
"importmap-module",
)
def _html(body: str, script: str = "") -> bytes:
return (
"<!doctype html><meta charset=utf-8>"
"<title>moli benchmark</title>"
f"<body>{body}<script>{script}</script></body>"
).encode("utf-8")
def _case_response(case: str) -> tuple[str, bytes, float]:
if case == "module-graph":
body = "<main data-benchmark-case=module-graph></main><script type=module src=/module-main.mjs></script>"
return "text/html; charset=utf-8", _html(body), 0.0
if case == "importmap-module":
body = """
<main data-benchmark-case=importmap-module></main>
<script type=importmap>{"imports":{"bench-alias":"/module-alias.mjs"}}</script>
<script type=module>
import { value } from 'bench-alias';
document.body.dataset.benchmarkStatus = value === 42 ? 'ok' : 'bad';
</script>
"""
return "text/html; charset=utf-8", _html(body), 0.0
raise KeyError(case)
def response_for_module_path(path: str) -> tuple[str, bytes, float] | None:
case = path.strip("/")
if path == "/module-main.mjs":
return (
"application/javascript",
b"import { value } from '/module-dep.mjs'; document.body.dataset.benchmarkStatus = value === 42 ? 'ok' : 'bad';",
0.0,
)
if path == "/module-dep.mjs" or path == "/module-alias.mjs":
return "application/javascript", b"export const value = 42;", 0.0
if case in MODULE_CASES:
return _case_response(case)
return None
moli-benchmark/moli_benchmark/synthetic_case_groups/webapi.py
+16
View File
@@ -0,0 +1,16 @@
from __future__ import annotations
from .dom import DOM_CASES, response_for_dom_path
from .io import IO_CASES, response_for_io_path
from .modules import MODULE_CASES, response_for_module_path
WEBAPI_CASES = MODULE_CASES + DOM_CASES + IO_CASES
def response_for_webapi_path(path: str) -> tuple[str, bytes, float] | None:
for resolver in (response_for_module_path, response_for_dom_path, response_for_io_path):
response = resolver(path)
if response is not None:
return response
return None
moli-benchmark/moli_benchmark/synthetic_cases.py
+34
View File
@@ -0,0 +1,34 @@
from __future__ import annotations
from dataclasses import dataclass
from .synthetic_case_groups.basic import BASIC_CASES, response_for_basic_path
from .synthetic_case_groups.webapi import WEBAPI_CASES, response_for_webapi_path
SYNTHETIC_CASES = BASIC_CASES + WEBAPI_CASES
@dataclass(frozen=True)
class SyntheticResponse:
content_type: str
body: bytes
delay_seconds: float = 0.0
def html_document(body: str, script: str = "") -> bytes:
return (
"<!doctype html><meta charset=utf-8>"
"<title>moli benchmark</title>"
f"<body>{body}<script>{script}</script></body>"
).encode("utf-8")
def response_for_path(path: str) -> SyntheticResponse | None:
response = response_for_basic_path(path)
if response is not None:
return SyntheticResponse(*response)
response = response_for_webapi_path(path)
if response is not None:
return SyntheticResponse(*response)
return None
moli-benchmark/moli_benchmark/synthetic_compare.py
+300
View File
@@ -0,0 +1,300 @@
from __future__ import annotations
import os
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Any
from .artifacts import write_csv, write_json
from .config import REPO_ROOT, clear_proxy_env
from .process import run_process
from .stats import summarize
from .synthetic import SYNTHETIC_CASES, SyntheticServer, _hash_output
BASE_TARGETS = ("moli", "lightpanda", "chrome", "obscura")
FETCH_TARGETS = (*BASE_TARGETS, "moli-full")
CDP_TARGETS = (
"moli-cdp",
"lightpanda-cdp",
"chrome-cdp",
"obscura-cdp",
"moli-full-cdp",
)
TARGETS = (*FETCH_TARGETS, *CDP_TARGETS)
WEBFETCH_TARGETS = (
"moli",
"moli-cdp",
"moli-full",
"moli-full-cdp",
"lightpanda",
"lightpanda-cdp",
"chrome",
"obscura",
"obscura-cdp",
)
TARGET_METADATA: dict[str, dict[str, str]] = {
"moli": {"engine": "moli", "driver": "fetch", "label": "moli / fetch", "binary_key": "moli"},
"moli-cdp": {"engine": "moli", "driver": "cdp", "label": "moli / cdp", "binary_key": "moli"},
"moli-full": {"engine": "moli", "driver": "fetch-full", "label": "moli full / fetch", "binary_key": "moli"},
"moli-full-cdp": {"engine": "moli", "driver": "cdp-full", "label": "moli full / cdp", "binary_key": "moli"},
"lightpanda": {"engine": "lightpanda", "driver": "fetch", "label": "lightpanda / fetch", "binary_key": "lightpanda"},
"lightpanda-cdp": {"engine": "lightpanda", "driver": "cdp", "label": "lightpanda / cdp", "binary_key": "lightpanda"},
"chrome": {"engine": "chrome", "driver": "dump-dom", "label": "chrome / dump-dom", "binary_key": "chrome"},
"chrome-cdp": {"engine": "chrome", "driver": "cdp", "label": "chrome / cdp", "binary_key": "chrome"},
"obscura": {"engine": "obscura", "driver": "fetch", "label": "obscura / fetch", "binary_key": "obscura"},
"obscura-cdp": {"engine": "obscura", "driver": "cdp", "label": "obscura / cdp", "binary_key": "obscura"},
}
def target_metadata(target: str) -> dict[str, str]:
return TARGET_METADATA.get(target, {"engine": target, "driver": "unknown", "label": target, "binary_key": target})
def target_binary_key(target: str) -> str:
return target_metadata(target)["binary_key"]
def target_uses_external_fixture(target: str) -> bool:
return target_metadata(target)["engine"] == "obscura"
def normalize_cdp_target(target: str) -> str:
if target in CDP_TARGETS:
return target
if target == "moli-full":
return "moli-full-cdp"
if target in BASE_TARGETS:
return f"{target}-cdp"
return target
def target_enables_all_resource_fetch(target: str) -> bool:
return target in {"moli-full", "moli-full-cdp"}
def target_is_cdp(target: str) -> bool:
return target in CDP_TARGETS
def _command_for_target(target: str, binary: Path, url: str, timeout_seconds: float) -> list[str]:
metadata = target_metadata(target)
if target_is_cdp(target):
raise RuntimeError(f"{target} is a CDP target; use the cdp-session suite")
timeout_ms = str(int(timeout_seconds * 1000))
if target in {"moli", "moli-full"}:
compatibility_args = (
["--layout", "--resource"]
if target_enables_all_resource_fetch(target)
else []
)
return [
str(binary),
"fetch",
*compatibility_args,
"--dump",
"html",
"--wait-until",
"done",
"--wait-script",
"document.querySelector('[data-benchmark-status=\"ok\"]') !== null",
"--timeout",
timeout_ms,
url,
]
if target == "lightpanda":
return [
str(binary),
"fetch",
"--dump",
"html",
"--wait-until",
"done",
"--wait-ms",
timeout_ms,
"--http-timeout",
timeout_ms,
"--terminate-ms",
timeout_ms,
url,
]
if target == "chrome":
return [
str(binary),
"--headless=new",
"--disable-gpu",
"--no-sandbox",
"--disable-dev-shm-usage",
f"--virtual-time-budget={timeout_ms}",
"--dump-dom",
url,
]
if target == "obscura":
return [
str(binary),
"fetch",
"--dump",
"html",
"--wait-until",
"load",
"--wait",
"0",
"--timeout",
str(max(1, int(timeout_seconds))),
url,
]
raise RuntimeError(f"unknown target: {target}")
def run_synthetic_compare_suite(
*,
output_dir: Path,
target_matrix: dict[str, Any],
targets: tuple[str, ...],
runs: int,
timeout_seconds: float,
cases: tuple[str, ...],
concurrency: int,
gate_target: str,
) -> dict[str, Any]:
unknown_cases = [case for case in cases if case not in SYNTHETIC_CASES]
if unknown_cases:
raise RuntimeError(f"unknown synthetic case(s): {', '.join(unknown_cases)}")
unknown_targets = [target for target in targets if target not in FETCH_TARGETS]
if unknown_targets:
raise RuntimeError(f"unknown fetch target(s): {', '.join(unknown_targets)}; use cdp-session for CDP targets")
if gate_target not in FETCH_TARGETS:
raise RuntimeError(f"unknown gate target: {gate_target}")
if gate_target not in targets:
raise RuntimeError(f"gate target `{gate_target}` must be included in selected targets")
suite_dir = output_dir / "synthetic-compare"
rows: list[dict[str, Any]] = []
details: list[dict[str, Any]] = []
def run_one(server: SyntheticServer, target: str, case: str, run_id: int) -> tuple[dict[str, Any], dict[str, Any]]:
metadata = target_metadata(target)
target_info = target_matrix.get(metadata["binary_key"], {})
path = target_info.get("path")
if not target_info.get("available") or not path:
row = {
"target": target,
**metadata,
"case": case,
"run": run_id,
"concurrency": concurrency,
"ok": False,
"elapsed_ms": None,
"returncode": None,
"timed_out": False,
"peak_pss_bytes": None,
"peak_rss_bytes": None,
"peak_cpu_percent": None,
"output_sha256": None,
"error": "target binary unavailable",
}
return row, dict(row)
if target_uses_external_fixture(target) and server.external_base_url is None:
row = {
"target": target,
**metadata,
"case": case,
"run": run_id,
"concurrency": concurrency,
"ok": False,
"elapsed_ms": None,
"returncode": None,
"timed_out": False,
"peak_pss_bytes": None,
"peak_rss_bytes": None,
"peak_cpu_percent": None,
"output_sha256": None,
"error": "external fixture address unavailable for Obscura",
}
return row, dict(row)
url = server.url_for_path(case, external=target_uses_external_fixture(target))
result = run_process(
_command_for_target(target, Path(path), url, timeout_seconds),
cwd=REPO_ROOT,
timeout_seconds=timeout_seconds + 2,
env=clear_proxy_env(os.environ),
)
ok = result.returncode == 0 and not result.timed_out and b"data-benchmark-status=\"ok\"" in result.stdout
row = {
"target": target,
**metadata,
"case": case,
"run": run_id,
"concurrency": concurrency,
"ok": ok,
"elapsed_ms": result.elapsed_ms,
"returncode": result.returncode,
"timed_out": result.timed_out,
"peak_pss_bytes": result.resources.get("peak_pss_bytes"),
"peak_rss_bytes": result.resources.get("peak_rss_bytes"),
"peak_cpu_percent": result.resources.get("peak_cpu_percent"),
"output_sha256": _hash_output(result.output_digest_material()),
"error": None if ok else "benchmark status marker missing or process failed",
}
return row, {**row, "url": url, "process": result.json_summary(include_output=not ok)}
with SyntheticServer() as server:
with ThreadPoolExecutor(max_workers=max(1, concurrency)) as executor:
futures = [
executor.submit(run_one, server, target, case, run_id)
for target in targets
for case in cases
for run_id in range(1, runs + 1)
]
for future in as_completed(futures):
row, detail = future.result()
rows.append(row)
details.append(detail)
gate_failures = sum(1 for row in rows if row["target"] == gate_target and not row.get("ok"))
summary: dict[str, Any] = {
"suite": "synthetic-compare",
"runs": runs,
"timeout_seconds": timeout_seconds,
"concurrency": concurrency,
"gate_target": gate_target,
"gate_failures": gate_failures,
"cases": list(cases),
"targets": {},
"total_failures": sum(1 for row in rows if not row.get("ok")),
}
for target in targets:
target_rows = [row for row in rows if row["target"] == target]
target_summary = {
**target_metadata(target),
"failures": sum(1 for row in target_rows if not row.get("ok")),
"failure_samples": [],
"cases": {},
}
for case in cases:
case_rows = [row for row in target_rows if row["case"] == case]
failure_samples = []
for detail in (detail for detail in details if detail["target"] == target and detail["case"] == case and not detail.get("ok")):
process = detail.get("process") if isinstance(detail.get("process"), dict) else {}
tail = str(process.get("stderr_tail") or process.get("stdout_tail") or detail.get("error") or "")
compact_tail = " ".join(line.strip() for line in tail.splitlines() if line.strip())
failure_samples.append(compact_tail[:500] if compact_tail else str(detail.get("error") or "unknown failure"))
if len(failure_samples) >= 3:
break
target_summary["cases"][case] = {
"elapsed_ms": summarize(row["elapsed_ms"] for row in case_rows if row.get("ok") and row.get("elapsed_ms") is not None),
"peak_pss_bytes": summarize(row["peak_pss_bytes"] for row in case_rows if row.get("peak_pss_bytes") is not None),
"peak_rss_bytes": summarize(row["peak_rss_bytes"] for row in case_rows if row.get("peak_rss_bytes") is not None),
"failures": sum(1 for row in case_rows if not row.get("ok")),
"failure_samples": failure_samples,
}
target_summary["failure_samples"].extend(failure_samples)
summary["targets"][target] = target_summary
write_csv(suite_dir / "runs.csv", rows)
write_json(suite_dir / "runs.json", details)
write_json(suite_dir / "summary.json", summary)
return summary
moli-benchmark/moli_benchmark/synthetic_matrix.py
+213
View File
@@ -0,0 +1,213 @@
from __future__ import annotations
from pathlib import Path
from typing import Any
from .artifacts import write_csv, write_json
from .stats import summarize
from .synthetic import SYNTHETIC_CASES, run_synthetic_suite
DEFAULT_SYNTHETIC_CONCURRENCY_MATRIX = (1, 5, 10, 25, 100)
DEFAULT_STABILITY_THRESHOLD_PERCENT = 10.0
FORMAL_SYNTHETIC_RUNS = 100
FORMAL_SYNTHETIC_REPEATS = 5
SYNTHETIC_MATRIX_PROFILES = ("smoke", "formal")
def _median_drift_percent(values: list[float]) -> float | None:
if len(values) < 2:
return None
summary = summarize(values)
median = summary.get("median")
minimum = summary.get("min")
maximum = summary.get("max")
if median in (None, 0) or minimum is None or maximum is None:
return None
return ((float(maximum) - float(minimum)) / float(median)) * 100.0
def _formal_gate_rows(
*,
profile: str,
formal_requirements: dict[str, dict[str, Any]],
total_failures: int,
stability_failures: int,
) -> list[dict[str, Any]]:
rows = [
{
"gate": "profile",
"actual": profile,
"required": "formal",
"ok": profile == "formal",
"failure_kind": None if profile == "formal" else "profile-not-formal",
}
]
for name, requirement in formal_requirements.items():
ok = bool(requirement.get("ok"))
rows.append(
{
"gate": name,
"actual": requirement.get("actual"),
"required": requirement.get("required"),
"ok": ok,
"failure_kind": None if ok else "formal-requirement",
}
)
rows.extend(
[
{
"gate": "workload_failures",
"actual": total_failures,
"required": 0,
"ok": total_failures == 0,
"failure_kind": None if total_failures == 0 else "workload-failure",
},
{
"gate": "stability_failures",
"actual": stability_failures,
"required": 0,
"ok": stability_failures == 0,
"failure_kind": None if stability_failures == 0 else "stability-drift",
},
]
)
return rows
def run_synthetic_matrix_suite(
*,
moli_bin: Path,
output_dir: Path,
profile: str,
runs: int,
timeout_seconds: float,
cases: tuple[str, ...],
concurrency_levels: tuple[int, ...],
repeats: int,
stability_threshold_percent: float,
) -> dict[str, Any]:
unknown = [case for case in cases if case not in SYNTHETIC_CASES]
if unknown:
raise RuntimeError(f"unknown synthetic case(s): {', '.join(unknown)}")
if not concurrency_levels:
raise RuntimeError("synthetic matrix requires at least one concurrency level")
invalid_concurrency = [value for value in concurrency_levels if value < 1]
if invalid_concurrency:
raise RuntimeError(f"invalid synthetic concurrency level(s): {invalid_concurrency}")
if repeats < 1:
raise RuntimeError("synthetic matrix repeats must be at least 1")
if profile not in SYNTHETIC_MATRIX_PROFILES:
raise RuntimeError(f"unknown synthetic matrix profile: {profile}")
suite_dir = output_dir / "synthetic-matrix"
matrix_rows: list[dict[str, Any]] = []
summaries: list[dict[str, Any]] = []
for repeat in range(1, repeats + 1):
for concurrency in concurrency_levels:
run_output_dir = suite_dir / f"repeat-{repeat}" / f"concurrency-{concurrency}"
summary = run_synthetic_suite(
moli_bin=moli_bin,
output_dir=run_output_dir,
runs=runs,
timeout_seconds=timeout_seconds,
cases=cases,
concurrency=concurrency,
)
summaries.append({**summary, "repeat": repeat})
for case, case_summary in summary.get("cases", {}).items():
elapsed = case_summary.get("elapsed_ms", {})
pss = case_summary.get("peak_pss_bytes", {})
matrix_rows.append(
{
"repeat": repeat,
"concurrency": concurrency,
"case": case,
"runs": runs,
"failures": case_summary.get("failures", 0),
"elapsed_p50_ms": elapsed.get("p50"),
"elapsed_p90_ms": elapsed.get("p90"),
"elapsed_p95_ms": elapsed.get("p95"),
"peak_pss_p50_bytes": pss.get("p50"),
"peak_pss_p95_bytes": pss.get("p95"),
}
)
cases_summary: dict[str, Any] = {}
stability_failures = 0
for case in cases:
by_concurrency: dict[str, Any] = {}
for concurrency in concurrency_levels:
rows = [row for row in matrix_rows if row["case"] == case and row["concurrency"] == concurrency]
medians = [
float(row["elapsed_p50_ms"])
for row in rows
if row.get("failures") == 0 and row.get("elapsed_p50_ms") is not None
]
drift = _median_drift_percent(medians)
stable = drift is None or drift <= stability_threshold_percent
if drift is not None and not stable:
stability_failures += 1
by_concurrency[str(concurrency)] = {
"elapsed_p50_ms": summarize(medians),
"median_drift_percent": drift,
"stable": stable,
"failures": sum(int(row.get("failures", 0) or 0) for row in rows),
}
cases_summary[case] = by_concurrency
total_failures = sum(int(row.get("failures", 0) or 0) for row in matrix_rows)
required_cases = set(SYNTHETIC_CASES)
selected_cases = set(cases)
required_concurrency = set(DEFAULT_SYNTHETIC_CONCURRENCY_MATRIX)
selected_concurrency = set(concurrency_levels)
formal_requirements = {
"runs": {"actual": runs, "required": FORMAL_SYNTHETIC_RUNS, "ok": runs >= FORMAL_SYNTHETIC_RUNS},
"repeats": {"actual": repeats, "required": FORMAL_SYNTHETIC_REPEATS, "ok": repeats >= FORMAL_SYNTHETIC_REPEATS},
"concurrency_levels": {
"actual": list(concurrency_levels),
"required": list(DEFAULT_SYNTHETIC_CONCURRENCY_MATRIX),
"ok": required_concurrency.issubset(selected_concurrency),
},
"cases": {
"actual": list(cases),
"required": list(SYNTHETIC_CASES),
"ok": required_cases.issubset(selected_cases),
},
}
profile_failures = (
sum(1 for requirement in formal_requirements.values() if not bool(requirement["ok"]))
if profile == "formal"
else 0
)
formal_gate_rows = _formal_gate_rows(
profile=profile,
formal_requirements=formal_requirements,
total_failures=total_failures,
stability_failures=stability_failures,
)
summary = {
"suite": "synthetic-matrix",
"profile": profile,
"runs": runs,
"repeats": repeats,
"timeout_seconds": timeout_seconds,
"concurrency_levels": list(concurrency_levels),
"stability_threshold_percent": stability_threshold_percent,
"stability_failures": stability_failures,
"formal_gate_rows": formal_gate_rows,
"formal_requirements": formal_requirements,
"profile_failures": profile_failures,
"gate_target": "moli",
"gate_failures": total_failures + stability_failures + profile_failures,
"cases": cases_summary,
"total_failures": total_failures,
}
write_csv(suite_dir / "matrix.csv", matrix_rows)
write_json(suite_dir / "matrix.json", matrix_rows)
write_json(suite_dir / "gate-rows.json", {"rows": formal_gate_rows})
write_json(suite_dir / "run-summaries.json", summaries)
write_json(suite_dir / "summary.json", summary)
return summary
moli-benchmark/moli_benchmark/target_serve.py
+225
View File
@@ -0,0 +1,225 @@
from __future__ import annotations
import os
import shutil
import subprocess
import tempfile
import threading
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from .config import REPO_ROOT, ReservedPort, clear_proxy_env, reserve_port
from .sampling import ResourceSampler
from .serve import probe_url
from .synthetic_compare import target_enables_all_resource_fetch, target_metadata
class TargetServeError(RuntimeError):
pass
class TargetServeProcessExit(TargetServeError):
pass
class TargetServeReadinessTimeout(TargetServeError):
pass
@dataclass
class TargetServeHandle:
target: str
process: subprocess.Popen[bytes]
endpoint: str
command: list[str]
logs: list[str]
sampler: ResourceSampler
log_threads: list[threading.Thread]
port_lease: ReservedPort
temp_dir: Path | None = None
ready_ms: float | None = None
def _append_log(logs: list[str], line: str) -> None:
logs.append(line)
if len(logs) > 400:
del logs[: len(logs) - 400]
def _read_available(stream: Any, label: str, logs: list[str]) -> None:
if stream is None:
return
try:
while True:
line = stream.readline()
if not line:
return
_append_log(logs, f"{label}: {line.decode('utf-8', errors='replace').rstrip()}")
except OSError:
return
def _terminate_process(process: subprocess.Popen[bytes]) -> bool:
if process.poll() is not None:
return True
try:
process.terminate()
except OSError:
return process.poll() is not None
try:
process.wait(timeout=2)
return True
except subprocess.TimeoutExpired:
try:
process.kill()
except OSError:
return process.poll() is not None
try:
process.wait(timeout=2)
return True
except subprocess.TimeoutExpired:
return False
def _serve_command(
target: str,
binary: Path,
port: int,
temp_dir: Path | None,
extra_args: tuple[str, ...] = (),
) -> list[str]:
engine = target_metadata(target)["engine"]
if engine == "moli" or engine == "lightpanda":
compatibility_args = (
["--layout", "--resource"]
if target_enables_all_resource_fetch(target)
else []
)
return [
str(binary),
"serve",
*compatibility_args,
*extra_args,
"--host",
"127.0.0.1",
"--port",
str(port),
]
if engine == "obscura":
return [str(binary), "serve", "--port", str(port)]
if engine == "chrome":
if temp_dir is None:
raise RuntimeError("chrome target requires a temporary profile directory")
return [
str(binary),
"--headless=new",
"--no-sandbox",
"--disable-gpu",
"--disable-dev-shm-usage",
"--no-first-run",
f"--user-data-dir={temp_dir}",
f"--remote-debugging-port={port}",
*extra_args,
"about:blank",
]
raise RuntimeError(f"unknown CDP target: {target}")
def start_target_serve(
target: str,
binary: Path,
timeout_seconds: float,
extra_args: tuple[str, ...] = (),
*,
sample_interval_seconds: float | None = None,
) -> TargetServeHandle:
engine = target_metadata(target)["engine"]
temp_dir = Path(tempfile.mkdtemp(prefix=f"moli-benchmark-{target}-")) if engine == "chrome" else None
logs: list[str] = []
reserved_port = reserve_port()
try:
port = reserved_port.port
endpoint = f"http://127.0.0.1:{port}"
command = _serve_command(target, binary, port, temp_dir, extra_args)
reserved_port.release_socket()
process = subprocess.Popen(
command,
cwd=REPO_ROOT,
env=clear_proxy_env(os.environ),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
start_new_session=True,
)
except BaseException:
reserved_port.close()
if temp_dir is not None:
shutil.rmtree(temp_dir, ignore_errors=True)
raise
log_threads = [
threading.Thread(target=_read_available, args=(process.stdout, "stdout", logs), daemon=True),
threading.Thread(target=_read_available, args=(process.stderr, "stderr", logs), daemon=True),
]
for thread in log_threads:
thread.start()
sampler = (
ResourceSampler(process.pid)
if sample_interval_seconds is None
else ResourceSampler(process.pid, interval_seconds=sample_interval_seconds)
)
sampler.start()
handle = TargetServeHandle(
target=target,
process=process,
endpoint=endpoint,
command=command,
logs=logs,
sampler=sampler,
log_threads=log_threads,
port_lease=reserved_port,
temp_dir=temp_dir,
)
started = time.perf_counter()
deadline = started + timeout_seconds
version_url = endpoint + "/json/version"
while time.perf_counter() < deadline:
if process.poll() is not None:
stop_target_serve(handle)
raise TargetServeProcessExit(
f"{target} CDP server exited early with {process.returncode}: "
f"{'; '.join(logs[-20:])}"
)
if probe_url(version_url):
handle.ready_ms = (time.perf_counter() - started) * 1000.0
return handle
time.sleep(0.025)
stop_target_serve(handle)
raise TargetServeReadinessTimeout(
f"timed out waiting for {target} CDP server at {version_url}: "
f"{'; '.join(logs[-20:])}"
)
def stop_target_serve(handle: TargetServeHandle | None) -> dict[str, Any]:
if handle is None:
return {}
process_exited = False
resources: dict[str, Any] = {}
try:
process_exited = _terminate_process(handle.process)
if process_exited:
for thread in handle.log_threads:
thread.join(timeout=0.2)
else:
_append_log(handle.logs, "process did not exit after SIGKILL; skipped pipe drain")
resources = handle.sampler.stop()
finally:
handle.port_lease.close()
if handle.temp_dir is not None:
shutil.rmtree(handle.temp_dir, ignore_errors=True)
return {
"returncode": handle.process.returncode,
"resources": resources,
"log_tail": handle.logs[-40:],
}
moli-benchmark/moli_benchmark/targets.py
+46
View File
@@ -0,0 +1,46 @@
from __future__ import annotations
import subprocess
from pathlib import Path
from typing import Any
from .config import chrome_binary, moli_binary, lightpanda_binary, obscura_binary
from .versions import sha256_file
def _version(command: list[str]) -> str | None:
try:
completed = subprocess.run(command, capture_output=True, text=True, timeout=5, check=False)
except (OSError, subprocess.TimeoutExpired):
return None
output = (completed.stdout or completed.stderr).strip()
return output.splitlines()[0] if output else None
def _binary_info(path: Path | None, version_args: tuple[str, ...]) -> dict[str, Any]:
if path is None:
return {"path": None, "available": False}
stat = path.stat()
return {
"path": str(path),
"available": True,
"size_bytes": stat.st_size,
"sha256": sha256_file(path),
"version": _version([str(path), *version_args]),
}
def collect_target_binaries(
*,
moli_override: str | None = None,
lightpanda_override: str | None = None,
chrome_override: str | None = None,
obscura_override: str | None = None,
) -> dict[str, Any]:
moli_path = moli_binary(moli_override)
return {
"moli": _binary_info(moli_path, ("version",)),
"lightpanda": _binary_info(lightpanda_binary(lightpanda_override), ("version",)),
"chrome": _binary_info(chrome_binary(chrome_override), ("--version",)),
"obscura": _binary_info(obscura_binary(obscura_override), ("--help",)),
}
moli-benchmark/moli_benchmark/top_sites.py
+952
View File
@@ -0,0 +1,952 @@
from __future__ import annotations
import os
import re
import html.parser
from hashlib import sha256
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Any
from .artifacts import write_csv, write_json, write_text
from .chrome_dcl import run_chrome_dcl_dump, run_served_cdp_dcl_dump
from .config import REPO_ROOT, clear_proxy_env
from .process import ProcessResult, run_process
from .stats import summarize
from .synthetic_compare import (
WEBFETCH_TARGETS,
target_enables_all_resource_fetch,
target_is_cdp,
target_metadata,
)
TOP_SITES_LIST_PATH = REPO_ROOT / "docs" / "chinese-community-top100-websites.md"
GLOBAL_TOP_SITES_LIST_PATH = REPO_ROOT / "docs" / "global-top-websites-seed-list.md"
WEBFETCH_LONGTAIL_LIST_PATH = REPO_ROOT / "docs" / "webfetch-longtail-seed-list.md"
RENDER_QUALITY_LIST_PATH = REPO_ROOT / "docs" / "render-quality-seed-list.md"
LEGACY_ENCODING_LIST_PATH = REPO_ROOT / "docs" / "legacy-encoding-websites-seed-list.md"
TOP_SITES_SOURCES: dict[str, dict[str, Any]] = {
"chinese-community": {
"path": TOP_SITES_LIST_PATH,
"label": "Chinese community top 100 (curated)",
},
"global": {
"path": GLOBAL_TOP_SITES_LIST_PATH,
"label": "Tranco-derived English-world top sites",
},
"webfetch-longtail": {
"path": WEBFETCH_LONGTAIL_LIST_PATH,
"label": "Observed WebFetch longtail URL failures",
},
"render-quality": {
"path": RENDER_QUALITY_LIST_PATH,
"label": "Curated article/document URLs for rendered-DOM quality checks",
},
"legacy-encoding": {
"path": LEGACY_ENCODING_LIST_PATH,
"label": "Curated non-UTF-8 public pages for document/script encoding checks",
},
}
COMPOSITE_TOP_SITES_SOURCES = ("mixed", "webfetch-mix")
DEFAULT_TOP_SITES_SOURCE = "chinese-community"
TOP_SITES_PROFILES: dict[str, dict[str, Any]] = {
"quick": {"limit": 20, "default_runs": 1},
"full": {"limit": 100, "default_runs": 1},
"webfetch": {"limit": 300, "default_runs": 1},
}
DEFAULT_TOP_SITES_PROFILE = "quick"
DEFAULT_TOP_SITES_MIN_BODY_BYTES = 256
DEFAULT_TOP_SITES_PARALLELISM_CAP = 8
def _default_top_sites_parallelism() -> int:
if hasattr(os, "sched_getaffinity"):
try:
return min(DEFAULT_TOP_SITES_PARALLELISM_CAP, max(1, len(os.sched_getaffinity(0))))
except OSError:
pass
return min(DEFAULT_TOP_SITES_PARALLELISM_CAP, max(1, os.cpu_count() or 1))
DEFAULT_TOP_SITES_PARALLELISM = _default_top_sites_parallelism()
DEFAULT_TOP_SITES_TEXT_SAMPLE_CHARS = 500
_TOP_LIST_HEADING = re.compile(r"^##\s+Top\s+\d+\b", re.IGNORECASE)
_NEXT_SECTION_HEADING = re.compile(r"^##\s+")
_LIST_ENTRY = re.compile(r"^\s*(\d+)\.\s+`([^`]+)`")
_BLOCKED_TITLE_403 = re.compile(r"^\s*(?:http\s*)?403\b")
_SAFE_ARTIFACT_TOKEN = re.compile(r"[^A-Za-z0-9._-]+")
_TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
_NETWORK_ERROR_MARKERS = (
"privacy error",
"your connection is not private",
"net::err_cert",
"this site can't be reached",
"this site can\u2019t be reached",
"err_name_not_resolved",
"err_connection",
"err_timed_out",
"dns_probe_finished",
"could not resolve",
"could not resolve host",
"could not resolve hostname",
"could not connect to server",
"name resolution",
"connection refused",
"connection reset",
"connection timed out",
"no route to host",
"network is unreachable",
"ssl handshake",
"ssl error",
"ssl connect error",
"tls handshake",
"timed out connecting",
"timeout was reached",
"request timeout",
"i/o error",
"broken pipe",
"operation timed out",
"failed to connect",
"dns lookup",
"curl request failed",
"recv failure",
)
_CLI_DEADLINE_COMMAND_MARKERS = (
"fetch_document_allow_http_error_with_wait_until",
"fetch document allow-http-error",
"fetch allow-http-error wait_until",
"fetch wait_until",
)
_CLI_DEADLINE_TIMEOUT_MARKERS = (
"timed out after",
)
_RAW_DOCUMENT_BODY_PROGRESS_RE = re.compile(
r"with\s+(\d+)\s+out\s+of\s+(\d+)\s+bytes\s+received",
re.IGNORECASE,
)
_ELAPSED_FAILURE_TIMEOUT_GRACE_SECONDS = 1.0
_CAPTCHA_MARKERS = (
"captcha",
"安全验证",
"验证你是真人",
"人机验证",
"human verification",
"verification",
"verify you are human",
"verify that you're not a robot",
"verify that you are not a robot",
"are you a robot",
"not a robot",
"bot or not",
"checking your browser",
"perform security check",
"security verification",
"performing security verification",
"请完成验证",
)
_LOGIN_MARKERS = (
"sign in",
"log in",
"login",
"登录",
"注册",
"账号",
)
_LOGIN_FORM_MARKERS = (
"login to your account",
"email/username",
"your password is a required field",
"forgot password",
"create a free account",
)
_LOGIN_CONTEXT_MARKERS = (
"password",
"验证码",
"短信验证码",
"语音验证码",
"手机号",
"手机验证",
"邮箱",
"email",
)
_JS_CHALLENGE_MARKERS = (
"__cf_chl_",
"c2wf946j0/probe",
"cf-challenge",
"__tencent_chaos_vm",
"__eo_jschallenge_vm",
"teojschallengesdk.js",
"eojschallengesdk",
"window.solvechallenge(",
"jsl_clearance",
"acw_sc__v2",
"window._phantom",
"_$jsvmprt",
"awswaf",
"aliyun_waf",
"aliyun waf",
"_waf_",
"just a moment...",
"vercel security checkpoint",
"we're verifying your browser",
"we\u2019re verifying your browser",
"enable javascript and cookies to continue",
"javascript is needed to access this site",
)
_BLOCKED_BODY_MARKERS = (
"unable to give you access to our site",
"access denied",
"access to this page has been denied",
"sorry, you have been blocked",
"security issue was automatically identified",
"waf拦截",
"被waf拦截",
)
_NOT_FOUND_MARKERS = (
"404 not found",
"file not found",
"page not found",
"this page cannot be found",
"page you requested is missing",
"page you requested could not be found",
)
_FORBIDDEN_ERROR_MARKERS = (
"403 forbidden",
"returned 403",
"http 403",
)
_NOT_FOUND_ERROR_MARKERS = (
"returned 404",
"http 404",
)
_SHELL_MARKERS = (
'id="root"',
'id="app"',
'id="__next"',
'data-reactroot',
"webpackJsonp",
"__NUXT__",
"__NEXT_DATA__",
)
class _VisibleTextExtractor(html.parser.HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self._skip_depth = 0
self.parts: list[str] = []
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
if tag.lower() in {"script", "style", "noscript", "svg", "template"}:
self._skip_depth += 1
def handle_endtag(self, tag: str) -> None:
if tag.lower() in {"script", "style", "noscript", "svg", "template"} and self._skip_depth:
self._skip_depth -= 1
def handle_data(self, data: str) -> None:
if self._skip_depth:
return
text = " ".join(data.split())
if text:
self.parts.append(text)
def _extract_title_and_text(stdout: bytes) -> dict[str, Any]:
html_text = stdout.decode("utf-8", errors="replace")
title = ""
match = _TITLE_RE.search(html_text)
if match:
title = re.sub(r"\s+", " ", match.group(1)).strip()
parser = _VisibleTextExtractor()
try:
parser.feed(html_text)
except html.parser.HTMLParseError:
pass
visible_text = " ".join(parser.parts)
visible_text = re.sub(r"\s+", " ", visible_text).strip()
return {
"title": title,
"text_length": len(visible_text),
"text_sample": visible_text[:DEFAULT_TOP_SITES_TEXT_SAMPLE_CHARS],
}
def _looks_like_binary_content(body: bytes) -> bool:
sample = body[:4096]
if sample.startswith(b"%PDF-"):
return True
if b"\x00" in sample:
return True
decoded = sample.decode("utf-8", errors="replace")
replacement_count = decoded.count("\ufffd")
return replacement_count > max(8, len(decoded) // 20)
def _looks_like_raw_binary_main_resource_timeout(text: str, min_body_bytes: int) -> bool:
if "failed to read raw document body" not in text:
return False
if "timeout was reached" not in text and "operation timed out" not in text:
return False
match = _RAW_DOCUMENT_BODY_PROGRESS_RE.search(text)
if match is None:
return False
received = int(match.group(1))
total = int(match.group(2))
return received >= min_body_bytes and total >= received
def _parse_top_sites_sections(path: Path) -> list[tuple[str, list[tuple[int, str]]]]:
if not path.exists():
raise RuntimeError(f"top sites list not found: {path}")
in_section = False
current_heading = ""
current_entries: list[tuple[int, str]] = []
sections: list[tuple[str, list[tuple[int, str]]]] = []
seen_domains: set[str] = set()
for raw in path.read_text(encoding="utf-8").splitlines():
line = raw.rstrip()
if not in_section:
if _TOP_LIST_HEADING.match(line):
in_section = True
current_heading = line
current_entries = []
seen_domains = set()
continue
if _NEXT_SECTION_HEADING.match(line):
if current_entries:
sections.append((current_heading, current_entries))
top_heading = _TOP_LIST_HEADING.match(line)
in_section = bool(top_heading)
current_heading = line if top_heading else ""
current_entries = []
seen_domains = set()
continue
match = _LIST_ENTRY.match(line)
if match:
rank = int(match.group(1))
domain = match.group(2).strip()
if domain and domain not in seen_domains:
seen_domains.add(domain)
current_entries.append((rank, domain))
if in_section and current_entries:
sections.append((current_heading, current_entries))
if not sections:
raise RuntimeError(f"no Top entries parsed from {path}")
return sections
def parse_top_sites_list(path: Path) -> list[tuple[int, str]]:
return _parse_top_sites_sections(path)[0][1]
def _parse_top_sites_list_by_count(path: Path, count: int) -> list[tuple[int, str]]:
heading = re.compile(rf"^##\s+Top\s+{count}\b", re.IGNORECASE)
for section_heading, entries in _parse_top_sites_sections(path):
if heading.match(section_heading):
return entries
raise RuntimeError(f"no Top {count} entries parsed from {path}")
def resolve_top_sites_source(source: str, list_path: Path | None) -> tuple[str, Path]:
if list_path is not None:
return ("custom", list_path)
if source not in TOP_SITES_SOURCES:
if source == "mixed":
return (source, REPO_ROOT / "docs" / "mixed-top-websites")
if source == "webfetch-mix":
return (source, REPO_ROOT / "docs" / "webfetch-mix-websites")
raise RuntimeError(
f"unknown top-sites source `{source}`; expected one of "
f"{sorted(TOP_SITES_SOURCES) + list(COMPOSITE_TOP_SITES_SOURCES)} or a --list-path override"
)
return (source, TOP_SITES_SOURCES[source]["path"])
def _interleave_entries(*entry_lists: list[tuple[int, str]]) -> list[tuple[int, str]]:
interleaved: list[tuple[int, str]] = []
seen: set[str] = set()
max_len = max((len(entries) for entries in entry_lists), default=0)
for index in range(max_len):
for entries in entry_lists:
if index >= len(entries):
continue
_, domain = entries[index]
if domain in seen:
continue
seen.add(domain)
interleaved.append((len(interleaved) + 1, domain))
return interleaved
def _append_unique(
base: list[tuple[int, str]],
extra: list[tuple[int, str]],
*,
limit: int | None = None,
) -> list[tuple[int, str]]:
combined: list[tuple[int, str]] = []
seen: set[str] = set()
for _, domain in [*base, *extra]:
if domain in seen:
continue
seen.add(domain)
combined.append((len(combined) + 1, domain))
if limit is not None and len(combined) >= limit:
break
return combined
def load_top_sites_entries(source: str, list_path: Path | None) -> tuple[list[tuple[int, str]], list[str]]:
"""Return (entries, source_labels) for the requested source.
`mixed` interleaves entries from chinese-community and global sources, taking
one from each in rank order, to make a balanced cross-region list.
`webfetch-mix` keeps that top-site coverage but caps it at 100 entries, then
appends observed longtail URL paths from the WebFetch failure corpus.
"""
if list_path is not None:
return parse_top_sites_list(list_path), [f"custom:{list_path.name}"]
if source == "mixed":
cn = parse_top_sites_list(TOP_SITES_SOURCES["chinese-community"]["path"])
gl = _parse_top_sites_list_by_count(GLOBAL_TOP_SITES_LIST_PATH, 100)
interleaved = _interleave_entries(cn, gl)
labels = [
f"chinese-community:{TOP_SITES_SOURCES['chinese-community']['path'].name}",
f"global:{GLOBAL_TOP_SITES_LIST_PATH.name}",
]
return interleaved, labels
if source == "webfetch-mix":
cn = parse_top_sites_list(TOP_SITES_SOURCES["chinese-community"]["path"])
gl = _parse_top_sites_list_by_count(GLOBAL_TOP_SITES_LIST_PATH, 100)
top_site_mix = _interleave_entries(cn, gl)[:100]
longtail = parse_top_sites_list(WEBFETCH_LONGTAIL_LIST_PATH)
entries = _append_unique(top_site_mix, longtail)
labels = [
f"chinese-community:{TOP_SITES_SOURCES['chinese-community']['path'].name}",
f"global:{GLOBAL_TOP_SITES_LIST_PATH.name}",
f"webfetch-longtail:{WEBFETCH_LONGTAIL_LIST_PATH.name}",
]
return entries, labels
if source not in TOP_SITES_SOURCES:
raise RuntimeError(
f"unknown top-sites source `{source}`; expected one of "
f"{sorted(TOP_SITES_SOURCES) + list(COMPOSITE_TOP_SITES_SOURCES)} or a --list-path override"
)
info = TOP_SITES_SOURCES[source]
entries = (
_parse_top_sites_list_by_count(info["path"], 100)
if source == "global"
else parse_top_sites_list(info["path"])
)
return entries, [f"{source}:{info['path'].name}"]
def _top_command_for_target(target: str, binary: Path, url: str, timeout_seconds: float) -> list[str]:
metadata = target_metadata(target)
if target_is_cdp(target):
raise RuntimeError(f"{target} is a CDP target; use the cdp-session suite")
if target == "chrome":
raise RuntimeError("chrome top-sites uses the CDP DCL runner")
timeout_ms = str(int(timeout_seconds * 1000))
if target in {"moli", "moli-full"}:
compatibility_args = (
["--layout", "--resource"]
if target_enables_all_resource_fetch(target)
else []
)
return [
str(binary),
"fetch",
*compatibility_args,
"--dump",
"html",
"--wait-until",
"domcontentloaded",
"--timeout",
timeout_ms,
"--http-timeout",
timeout_ms,
url,
]
if target == "lightpanda":
return [
str(binary),
"fetch",
"--dump",
"html",
"--wait-until",
"domcontentloaded",
"--wait-ms",
timeout_ms,
"--http-timeout",
timeout_ms,
"--terminate-ms",
timeout_ms,
url,
]
if target == "obscura":
return [
str(binary),
"fetch",
"--dump",
"html",
"--wait-until",
"load",
"--wait",
"0",
"--timeout",
str(max(1, int(timeout_seconds))),
url,
]
raise RuntimeError(f"unknown target: {target}")
def _top_sites_target_metadata(target: str) -> dict[str, str]:
metadata = dict(target_metadata(target))
if target == "chrome" or target_is_cdp(target):
metadata["driver"] = "cdp-dcl"
prefix = "moli full" if target == "moli-full-cdp" else metadata["engine"]
metadata["label"] = f"{prefix} / cdp-dcl"
return metadata
def _run_top_sites_target(
*,
target: str,
binary: Path,
url: str,
timeout_seconds: float,
proc_env: dict[str, str],
) -> ProcessResult:
if target == "chrome":
return run_chrome_dcl_dump(
binary,
url,
cwd=REPO_ROOT,
timeout_seconds=timeout_seconds,
env=proc_env,
)
if target_is_cdp(target):
return run_served_cdp_dcl_dump(
target,
binary,
url,
cwd=REPO_ROOT,
timeout_seconds=timeout_seconds,
env=proc_env,
)
command = _top_command_for_target(target, binary, url, timeout_seconds)
return run_process(
command,
cwd=REPO_ROOT,
timeout_seconds=timeout_seconds + 2,
env=proc_env,
)
def _classify(
stdout: bytes,
stderr: bytes,
returncode: int | None,
timed_out: bool,
min_body_bytes: int,
snapshot: dict[str, Any] | None = None,
) -> str:
if timed_out:
return "timeout"
combined_text = (stdout + b"\n" + stderr).decode("utf-8", errors="replace").lower()
if "operationtimedout" in combined_text and (
"navigate failed" in combined_text or "navigation failed" in combined_text
):
return "timeout"
if (
returncode != 0
and any(marker in combined_text for marker in _CLI_DEADLINE_COMMAND_MARKERS)
and any(marker in combined_text for marker in _CLI_DEADLINE_TIMEOUT_MARKERS)
):
return "timeout"
if returncode != 0 and _looks_like_raw_binary_main_resource_timeout(
combined_text,
min_body_bytes,
):
return "success-binary-main-resource"
if returncode != 0:
if any(marker in combined_text for marker in _FORBIDDEN_ERROR_MARKERS):
return "blocked-or-forbidden"
if any(marker in combined_text for marker in _NOT_FOUND_ERROR_MARKERS):
return "not-found"
if any(marker in combined_text for marker in _NETWORK_ERROR_MARKERS):
return "network-error"
return "process-error"
body = stdout.strip()
if not body:
if any(marker in combined_text for marker in _NETWORK_ERROR_MARKERS):
return "network-error"
return "empty-response"
if len(body) >= min_body_bytes and _looks_like_binary_content(body):
return "success-binary-content"
if snapshot is None:
snapshot = _extract_title_and_text(stdout)
text_length = int(snapshot.get("text_length") or 0)
title = str(snapshot.get("title") or "").lower()
sample = str(snapshot.get("text_sample") or "").lower()
page_text = f"{title}\n{sample}"
if any(marker in page_text for marker in _NETWORK_ERROR_MARKERS):
return "network-error"
if (
"blocked" in title
or "forbidden" in title
or _BLOCKED_TITLE_403.search(title)
or "access restricted" in title
):
return "blocked-or-forbidden"
if any(marker in page_text for marker in _NOT_FOUND_MARKERS):
return "not-found"
if any(marker in page_text for marker in _BLOCKED_BODY_MARKERS):
return "blocked-or-forbidden"
if any(marker in page_text for marker in _CAPTCHA_MARKERS):
return "captcha-or-verification"
if any(marker in combined_text for marker in _JS_CHALLENGE_MARKERS):
return "js-challenge"
if _looks_like_login_wall(title, sample, text_length):
return "login-wall"
if len(body) < min_body_bytes:
return "app-shell-only"
if text_length < min_body_bytes:
return "app-shell-only"
return "success-content"
def _elapsed_failure_reached_timeout(elapsed_ms: float | None, timeout_seconds: float) -> bool:
if elapsed_ms is None or timeout_seconds <= 0:
return False
grace_seconds = min(_ELAPSED_FAILURE_TIMEOUT_GRACE_SECONDS, timeout_seconds * 0.05)
timeout_floor_ms = max(0.0, (timeout_seconds - grace_seconds) * 1000.0)
return elapsed_ms >= timeout_floor_ms
def _looks_like_login_wall(title: str, sample: str, text_length: int) -> bool:
page_text = f"{title}\n{sample}"
if sum(1 for marker in _LOGIN_FORM_MARKERS if marker in page_text) >= 2:
return True
if text_length >= 800:
return False
login_hits = {marker for marker in _LOGIN_MARKERS if marker in page_text}
if not login_hits:
return False
title_has_login = any(marker in title for marker in _LOGIN_MARKERS)
if title_has_login and len(login_hits) >= 2:
return True
if len(login_hits) >= 2 and any(marker in page_text for marker in _LOGIN_CONTEXT_MARKERS):
return True
return False
def _ok_categories() -> set[str]:
return {"success-content", "success-binary-content", "success-binary-main-resource"}
def _failure_kind(category: str, error: str | None) -> str | None:
if error == "target binary unavailable":
return "target-unavailable"
if category not in _ok_categories():
return category
return None
_SITE_UNREACHABLE_FAILURE_KINDS = {
"network-error",
"timeout",
"empty-response",
"process-error",
}
def _site_unreachable_exclusions(rows: list[dict[str, Any]]) -> dict[str, str]:
rows_by_domain: dict[str, list[dict[str, Any]]] = {}
for row in rows:
rows_by_domain.setdefault(str(row["domain"]), []).append(row)
excluded: dict[str, str] = {}
for domain, domain_rows in rows_by_domain.items():
if any(row.get("ok") for row in domain_rows):
continue
failure_kinds = {
str(row.get("failure_kind") or _failure_kind(str(row.get("category")), None) or "")
for row in domain_rows
}
failure_kinds.discard("")
if (
failure_kinds
and failure_kinds <= _SITE_UNREACHABLE_FAILURE_KINDS
and (failure_kinds - {"process-error"})
):
excluded[domain] = "site-unreachable"
return excluded
def _artifact_filename_token(value: str, *, max_length: int = 80) -> str:
token = _SAFE_ARTIFACT_TOKEN.sub("_", value)
while ".." in token:
token = token.replace("..", "_")
token = token.strip("._-") or "site"
if token == value and len(token) <= max_length:
return token
digest = sha256(value.encode("utf-8", errors="surrogatepass")).hexdigest()[:12]
token = token[:max_length].rstrip("._-") or "site"
return f"{token}-{digest}"
def _write_failure_artifact(
*,
suite_dir: Path,
row: dict[str, Any],
stdout: bytes,
stderr: bytes,
) -> str:
domain = _artifact_filename_token(str(row["domain"]))
name = f"{row['target']}-run-{row['run']}-rank{row['rank']:03d}-{domain}"
failures_dir = suite_dir / "failures"
json_path = failures_dir / f"{name}.json"
stderr_path = failures_dir / f"{name}.stderr.txt"
write_json(json_path, row)
write_text(stderr_path, stderr[-16 * 1024 :].decode("utf-8", errors="replace"))
if stdout.strip():
stdout_path = failures_dir / f"{name}.stdout.html"
write_text(stdout_path, stdout[-32 * 1024 :].decode("utf-8", errors="replace"))
return str(json_path.relative_to(suite_dir))
def _domain_to_url(domain: str) -> str:
if domain.startswith("http://") or domain.startswith("https://"):
return domain
return f"https://{domain}"
def _execute_one(
*,
suite_dir: Path,
target: str,
metadata: dict[str, str],
info: dict[str, Any],
run_id: int,
rank: int,
domain: str,
timeout_seconds: float,
min_body_bytes: int,
proc_env: dict[str, str],
) -> dict[str, Any]:
url = _domain_to_url(domain)
if not info.get("available") or not info.get("path"):
row = {
"target": target,
**metadata,
"run": run_id,
"rank": rank,
"domain": domain,
"url": url,
"category": "error",
"ok": False,
"failure_kind": "target-unavailable",
"error": "target binary unavailable",
}
row["failure_artifact"] = _write_failure_artifact(suite_dir=suite_dir, row=row, stdout=b"", stderr=b"")
return row
result = _run_top_sites_target(
target=target,
binary=Path(info["path"]),
url=url,
timeout_seconds=timeout_seconds,
proc_env=proc_env,
)
snapshot = _extract_title_and_text(result.stdout)
category = _classify(
result.stdout,
result.stderr,
result.returncode,
result.timed_out,
min_body_bytes,
snapshot,
)
if category not in _ok_categories() and _elapsed_failure_reached_timeout(result.elapsed_ms, timeout_seconds):
category = "timeout"
ok = category in _ok_categories()
row: dict[str, Any] = {
"target": target,
**metadata,
"run": run_id,
"rank": rank,
"domain": domain,
"url": url,
"command": result.command,
"category": category,
"ok": ok,
"failure_kind": None if ok else _failure_kind(category, None),
"elapsed_ms": result.elapsed_ms,
"returncode": result.returncode,
"timed_out": result.timed_out,
"stdout_bytes": len(result.stdout),
"stderr_bytes": len(result.stderr),
"title": snapshot["title"],
"text_length": snapshot["text_length"],
"text_sample": snapshot["text_sample"],
"stdout_tail": result.stdout[-512:].decode("utf-8", errors="replace"),
"stderr_tail": result.stderr[-512:].decode("utf-8", errors="replace"),
"peak_pss_bytes": result.resources.get("peak_pss_bytes"),
"peak_rss_bytes": result.resources.get("peak_rss_bytes"),
}
if not ok:
row["failure_artifact"] = _write_failure_artifact(
suite_dir=suite_dir, row=row, stdout=result.stdout, stderr=result.stderr
)
return row
def run_top_sites_suite(
*,
output_dir: Path,
target_matrix: dict[str, Any],
targets: tuple[str, ...],
profile: str = DEFAULT_TOP_SITES_PROFILE,
list_path: Path | None = None,
source: str = DEFAULT_TOP_SITES_SOURCE,
runs: int | None = None,
timeout_seconds: float = 15.0,
gate_target: str = "moli",
parallelism: int = DEFAULT_TOP_SITES_PARALLELISM,
chrome_parallelism: int = 1,
min_body_bytes: int = DEFAULT_TOP_SITES_MIN_BODY_BYTES,
limit_override: int | None = None,
) -> dict[str, Any]:
unknown_targets = [target for target in targets if target not in WEBFETCH_TARGETS]
if unknown_targets:
raise RuntimeError(f"unknown webfetch target(s): {', '.join(unknown_targets)}")
if gate_target not in WEBFETCH_TARGETS:
raise RuntimeError(f"unknown gate target: {gate_target}")
if profile not in TOP_SITES_PROFILES:
raise RuntimeError(f"unknown top-sites profile `{profile}`; expected one of {sorted(TOP_SITES_PROFILES)}")
profile_config = TOP_SITES_PROFILES[profile]
limit = int(limit_override if limit_override is not None else profile_config["limit"])
if limit <= 0:
raise RuntimeError("top-sites limit must be positive")
runs_count = int(runs if runs is not None else profile_config["default_runs"])
if runs_count <= 0:
raise RuntimeError("top-sites runs must be positive")
if parallelism <= 0:
raise RuntimeError("top-sites parallelism must be positive")
if chrome_parallelism <= 0:
raise RuntimeError("top-sites chrome parallelism must be positive")
suite_dir = output_dir / "top-sites"
resolved_source, primary_path = resolve_top_sites_source(source, list_path)
entries_all, list_source_labels = load_top_sites_entries(resolved_source, list_path)
entries = entries_all[:limit]
list_source = primary_path
proc_env = clear_proxy_env(os.environ)
rows: list[dict[str, Any]] = []
for target in targets:
metadata = _top_sites_target_metadata(target)
info = target_matrix.get(metadata["binary_key"], {})
job_specs = [
(run_id, rank, domain)
for run_id in range(1, runs_count + 1)
for rank, domain in entries
]
target_workers = chrome_parallelism if target == "chrome" else parallelism
with ThreadPoolExecutor(max_workers=target_workers) as executor:
future_to_spec = {
executor.submit(
_execute_one,
suite_dir=suite_dir,
target=target,
metadata=metadata,
info=info,
run_id=run_id,
rank=rank,
domain=domain,
timeout_seconds=timeout_seconds,
min_body_bytes=min_body_bytes,
proc_env=proc_env,
): (target, run_id, rank, domain)
for run_id, rank, domain in job_specs
}
for future in as_completed(future_to_spec):
rows.append(future.result())
rows.sort(key=lambda row: (row["target"], row["run"], row["rank"]))
excluded_domains = _site_unreachable_exclusions(rows) if len(targets) > 1 else {}
for row in rows:
exclusion_reason = excluded_domains.get(str(row["domain"]))
row["excluded"] = exclusion_reason is not None
row["exclusion_reason"] = exclusion_reason
counted_rows = [row for row in rows if not row.get("excluded")]
gate_failures = sum(1 for row in counted_rows if row["target"] == gate_target and not row.get("ok"))
summary: dict[str, Any] = {
"suite": "top-sites",
"profile": profile,
"limit": limit,
"source": resolved_source,
"list_source": str(list_source.relative_to(REPO_ROOT)) if list_source.is_relative_to(REPO_ROOT) else str(list_source),
"list_sources": list_source_labels,
"site_count": len(entries),
"counted_site_count": len(entries) - len(excluded_domains),
"excluded_site_count": len(excluded_domains),
"excluded_sites": [
{"domain": domain, "reason": reason}
for domain, reason in sorted(
excluded_domains.items(),
key=lambda item: next((rank for rank, entry_domain in entries if entry_domain == item[0]), 0),
)
],
"runs": runs_count,
"timeout_seconds": timeout_seconds,
"min_body_bytes": min_body_bytes,
"parallelism": parallelism,
"chrome_parallelism": chrome_parallelism,
"gate_target": gate_target,
"gate_failures": gate_failures,
"total_failures": sum(1 for row in counted_rows if not row.get("ok")),
"total_excluded_runs": sum(1 for row in rows if row.get("excluded")),
"targets": {},
}
for target in targets:
all_target_rows = [row for row in rows if row["target"] == target]
target_rows = [row for row in all_target_rows if not row.get("excluded")]
categories: dict[str, int] = {}
failure_kinds: dict[str, int] = {}
for row in target_rows:
categories[row["category"]] = categories.get(row["category"], 0) + 1
failure_kind = row.get("failure_kind")
if isinstance(failure_kind, str) and failure_kind:
failure_kinds[failure_kind] = failure_kinds.get(failure_kind, 0) + 1
summary["targets"][target] = {
**_top_sites_target_metadata(target),
"sites": len(target_rows),
"raw_sites": len(all_target_rows),
"excluded_runs": len(all_target_rows) - len(target_rows),
"runs": runs_count,
"passes": sum(1 for row in target_rows if row.get("ok")),
"failures": sum(1 for row in target_rows if not row.get("ok")),
"categories": categories,
"failure_kinds": failure_kinds,
"elapsed_ms": summarize(row["elapsed_ms"] for row in target_rows if row.get("elapsed_ms") is not None),
"peak_pss_bytes": summarize(row["peak_pss_bytes"] for row in target_rows if row.get("peak_pss_bytes") is not None),
"peak_rss_bytes": summarize(row["peak_rss_bytes"] for row in target_rows if row.get("peak_rss_bytes") is not None),
}
write_csv(suite_dir / "raw-runs.csv", rows)
write_json(suite_dir / "runs.json", rows)
write_json(suite_dir / "summary.json", summary)
return summary
moli-benchmark/moli_benchmark/versions.py
+59
View File
@@ -0,0 +1,59 @@
from __future__ import annotations
import hashlib
import subprocess
from pathlib import Path
from typing import Any
from . import __version__
from .config import REPO_ROOT
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as file:
for chunk in iter(lambda: file.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _run(command: list[str], timeout: float = 5.0) -> str | None:
try:
completed = subprocess.run(command, cwd=REPO_ROOT, capture_output=True, text=True, timeout=timeout, check=False)
except (OSError, subprocess.TimeoutExpired):
return None
output = (completed.stdout or completed.stderr).strip()
return output if output else None
def collect_versions(moli_bin: Path | None = None, targets: dict[str, Any] | None = None) -> dict[str, Any]:
moli: dict[str, Any] | None = None
if moli_bin is not None:
moli = {
"path": str(moli_bin),
"size_bytes": moli_bin.stat().st_size,
"sha256": sha256_file(moli_bin),
"version": _run([str(moli_bin), "version"]),
}
elif targets and isinstance(targets.get("moli"), dict):
moli = targets["moli"]
return {
"benchmark": {"version": __version__},
"git": {
"commit": _run(["git", "rev-parse", "HEAD"]),
"branch": _run(["git", "branch", "--show-current"]),
"dirty": _run(["git", "status", "--porcelain"]),
},
"tools": {
"rustc": _run(["rustc", "--version"]),
"cargo": _run(["cargo", "--version"]),
"python": _run(["python", "--version"]),
"python3": _run(["python3", "--version"]),
"node": _run(["node", "--version"]),
"go": _run(["go", "version"]),
"smem": _run(["smem", "--version"]),
},
"moli": moli,
"targets": targets,
}
moli-benchmark/moli_benchmark/wasm_v8_live_binding_audit.py
+477
View File
@@ -0,0 +1,477 @@
"""Audit V8 source readiness for wasm global live-binding work.
This is deliberately a source-tree audit, not a runtime compatibility shim. It
helps distinguish a stock V8 tree from a candidate tree that has started to
carry Moli-owned wasm module-binding changes.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any
VERSION_HEADER = "include/v8-version.h"
@dataclass(frozen=True)
class SourceProbe:
path: str
name: str
pattern: str
description: str
scope_pattern: str | None = None
UNPATCHED_READ_PROBES = [
SourceProbe(
"src/objects/module.cc",
"namespace_get_export_reads_cell_value",
r"JSModuleNamespace::GetExport[\s\S]*?Cell::value|JSModuleNamespace::GetExport[\s\S]*?->value\(",
"module namespace export access still reads ordinary Cell::value",
scope_pattern=r"JSModuleNamespace::GetExport",
),
SourceProbe(
"src/objects/module.cc",
"namespace_get_property_attributes_reads_cell_value",
r"JSModuleNamespace::GetPropertyAttributes[\s\S]*?Cell::value|JSModuleNamespace::GetPropertyAttributes[\s\S]*?->value\(",
"module namespace property attributes still read ordinary Cell::value",
scope_pattern=r"JSModuleNamespace::GetPropertyAttributes",
),
SourceProbe(
"src/objects/source-text-module.cc",
"source_text_load_variable_reads_cell_value",
r"SourceTextModule::LoadVariable[\s\S]*?->value\(",
"SourceTextModule::LoadVariable still reads ordinary Cell::value",
scope_pattern=r"SourceTextModule::LoadVariable",
),
SourceProbe(
"src/interpreter/interpreter-generator.cc",
"ignition_lda_module_variable_direct_cell_offset",
r"IGNITION_HANDLER\(LdaModuleVariable[\s\S]*?Cell::kValueOffset",
"Ignition LdaModuleVariable still loads Cell::kValueOffset directly",
scope_pattern=r"IGNITION_HANDLER\(LdaModuleVariable",
),
SourceProbe(
"src/baseline",
"baseline_lda_module_variable_direct_cell_offset",
r"LdaModuleVariable[\s\S]*?Cell::kValueOffset",
"Baseline LdaModuleVariable still loads Cell::kValueOffset directly",
scope_pattern=r"LdaModuleVariable",
),
SourceProbe(
"src/maglev/maglev-graph-builder.cc",
"maglev_lda_module_variable_direct_cell_offset",
r"VisitLdaModuleVariable[\s\S]*?Cell::kValueOffset",
"Maglev LdaModuleVariable still loads Cell::kValueOffset directly",
scope_pattern=r"VisitLdaModuleVariable",
),
SourceProbe(
"src/compiler/bytecode-graph-builder.cc",
"turbofan_bytecode_graph_builder_load_module",
r"BytecodeGraphBuilder::VisitLdaModuleVariable[\s\S]*?javascript\(\)->LoadModule",
"TurboFan bytecode graph builder still emits JSLoadModule for module binding reads",
scope_pattern=r"BytecodeGraphBuilder::VisitLdaModuleVariable",
),
SourceProbe(
"src/compiler/js-typed-lowering.cc",
"turbofan_jsloadmodule_for_cell_value",
r"ReduceJSLoadModule[\s\S]*?AccessBuilder::ForCellValue",
"TurboFan JSLoadModule lowering still lowers to ForCellValue",
scope_pattern=r"ReduceJSLoadModule",
),
SourceProbe(
"src/compiler/js-native-context-specialization.cc",
"namespace_specialization_for_cell_value",
r"AccessBuilder::ForCellValue",
"optimized module namespace specialization still reads ForCellValue",
),
]
PATCH_MARKER_PROBES = [
SourceProbe(
"include/v8-script.h",
"public_wasm_global_export_api",
r"SetSyntheticModuleExportWasmGlobal",
"public V8 API exposes a wasm-global synthetic export setter",
),
SourceProbe(
"src/objects/synthetic-module.cc",
"synthetic_module_wasm_global_setter",
r"SetExportWasmGlobal|SetSyntheticModuleExportWasmGlobal",
"SyntheticModule has an internal wasm-global export setter",
),
SourceProbe(
"src/objects",
"internal_wasm_global_binding_object",
r"WasmGlobalExportBinding",
"V8 has a distinct wasm global export binding object or descriptor",
),
SourceProbe(
"src/wasm",
"wasm_global_js_value_materializer",
r"GetJSValue|Materialize.*WasmGlobal|WasmGlobal.*Materialize",
"wasm global storage can be materialized as a JS module binding value",
),
]
PATCHED_READ_PATH_PROBES = [
SourceProbe(
"src/objects/module.cc",
"namespace_get_export_materializes_wasm_binding",
r"JSModuleNamespace::GetExport[\s\S]*?(MaterializeModuleBinding|WasmGlobalExportBinding)",
"module namespace export access has an explicit wasm/materialized binding path",
scope_pattern=r"JSModuleNamespace::GetExport",
),
SourceProbe(
"src/objects/module.cc",
"namespace_get_property_attributes_materializes_wasm_binding",
r"JSModuleNamespace::GetPropertyAttributes[\s\S]*?(MaterializeModuleBinding|WasmGlobalExportBinding)",
"module namespace property attributes have an explicit wasm/materialized binding path",
scope_pattern=r"JSModuleNamespace::GetPropertyAttributes",
),
SourceProbe(
"src/objects/source-text-module.cc",
"source_text_load_variable_materializes_wasm_binding",
r"SourceTextModule::LoadVariable[\s\S]*?(MaterializeModuleBinding|WasmGlobalExportBinding)",
"SourceTextModule::LoadVariable has an explicit wasm/materialized binding path",
scope_pattern=r"SourceTextModule::LoadVariable",
),
SourceProbe(
"src/interpreter/interpreter-generator.cc",
"ignition_lda_module_variable_materializes_wasm_binding",
r"IGNITION_HANDLER\(LdaModuleVariable[\s\S]*?(MaterializeModuleBinding|WasmGlobalExportBinding|kMaterializeModuleBindingValue|LoadMaterializedModuleBinding)",
"Ignition LdaModuleVariable materializes wasm-aware module bindings",
scope_pattern=r"IGNITION_HANDLER\(LdaModuleVariable",
),
SourceProbe(
"src/baseline",
"baseline_lda_module_variable_materializes_wasm_binding",
r"LdaModuleVariable[\s\S]*?(MaterializeModuleBinding|WasmGlobalExportBinding|kMaterializeModuleBindingValue|LoadMaterializedModuleBinding)",
"Baseline LdaModuleVariable materializes wasm-aware module bindings",
scope_pattern=r"LdaModuleVariable",
),
SourceProbe(
"src/maglev/maglev-graph-builder.cc",
"maglev_lda_module_variable_materializes_wasm_binding",
r"VisitLdaModuleVariable[\s\S]*?(MaterializeModuleBinding|WasmGlobalExportBinding|kMaterializeModuleBindingValue|LoadMaterializedModuleBinding)",
"Maglev LdaModuleVariable materializes wasm-aware module bindings",
scope_pattern=r"VisitLdaModuleVariable",
),
SourceProbe(
"src/compiler/bytecode-graph-builder.cc",
"turbofan_bytecode_graph_builder_materialized_load_module",
r"BytecodeGraphBuilder::VisitLdaModuleVariable[\s\S]*?(LoadMaterializedModuleBinding|MaterializeModuleBinding|WasmGlobalExportBinding)",
"TurboFan bytecode graph builder emits a materialized module-binding read",
scope_pattern=r"BytecodeGraphBuilder::VisitLdaModuleVariable",
),
SourceProbe(
"src/compiler/js-typed-lowering.cc",
"turbofan_jsloadmodule_materialized_binding",
r"ReduceJSLoadModule[\s\S]*?(MaterializedModuleBinding|MaterializeModuleBinding|WasmGlobalExportBinding)",
"TurboFan JSLoadModule lowering recognizes wasm/materialized module bindings",
scope_pattern=r"ReduceJSLoadModule",
),
SourceProbe(
"src/compiler/js-native-context-specialization.cc",
"namespace_specialization_materialized_binding",
r"(MaterializedModuleBinding|MaterializeModuleBinding|WasmGlobalExportBinding)",
"optimized namespace specialization recognizes wasm/materialized module bindings",
),
]
def _read_text(path: Path) -> str | None:
try:
return path.read_text(encoding="utf-8", errors="replace")
except OSError:
return None
def _mask_cpp_comments_and_literals_preserving_lines(text: str) -> str:
"""Mask C++ comments and literals before source probes, preserving lines."""
def masked(fragment: str) -> str:
return "".join("\n" if ch == "\n" else " " for ch in fragment)
out: list[str] = []
i = 0
length = len(text)
while i < length:
ch = text[i]
next_ch = text[i + 1] if i + 1 < length else ""
if ch == "R" and next_ch == '"':
open_paren = text.find("(", i + 2)
if open_paren != -1:
delimiter = text[i + 2 : open_paren]
end_marker = ")" + delimiter + '"'
end = text.find(end_marker, open_paren + 1)
if end != -1:
end += len(end_marker)
out.append(masked(text[i:end]))
i = end
continue
if ch in {'"', "'"}:
quote = ch
literal_start = i
i += 1
while i < length:
current = text[i]
i += 1
if current == "\\" and i < length:
i += 1
continue
if current == quote:
break
out.append(masked(text[literal_start:i]))
continue
if ch == "/" and next_ch == "/":
comment_start = i
i += 2
while i < length and text[i] != "\n":
i += 1
out.append(masked(text[comment_start:i]))
continue
if ch == "/" and next_ch == "*":
comment_start = i
i += 2
while i < length:
if text[i] == "*" and i + 1 < length and text[i + 1] == "/":
i += 2
break
i += 1
out.append(masked(text[comment_start:i]))
continue
out.append(ch)
i += 1
return "".join(out)
def _iter_source_texts(root: Path, relative: str) -> list[tuple[str, str]]:
target = root / relative
if target.is_file():
text = _read_text(target)
return [] if text is None else [(relative, text)]
if not target.is_dir():
return []
texts: list[tuple[str, str]] = []
for path in sorted(target.rglob("*")):
if not path.is_file() or path.suffix not in {".cc", ".h", ".inc", ".tq"}:
continue
text = _read_text(path)
if text is not None:
texts.append((path.relative_to(root).as_posix(), text))
return texts
def _find_matching_brace(text: str, open_brace: int) -> int | None:
depth = 0
for index in range(open_brace, len(text)):
ch = text[index]
if ch == "{":
depth += 1
elif ch == "}":
depth -= 1
if depth == 0:
return index + 1
return None
def _probe_regions(text: str, probe: SourceProbe) -> list[tuple[int, str]]:
if probe.scope_pattern is None:
return [(0, text)]
regions: list[tuple[int, str]] = []
for match in re.finditer(probe.scope_pattern, text, flags=re.MULTILINE):
open_brace = text.find("{", match.end())
if open_brace == -1:
continue
if ";" in text[match.end() : open_brace]:
continue
end = _find_matching_brace(text, open_brace)
if end is None:
continue
regions.append((match.start(), text[match.start() : end]))
return regions
def _probe(root: Path, probe: SourceProbe) -> dict[str, Any]:
matches: list[dict[str, Any]] = []
for relative, text in _iter_source_texts(root, probe.path):
probe_text = _mask_cpp_comments_and_literals_preserving_lines(text)
for region_start, region in _probe_regions(probe_text, probe):
for match in re.finditer(probe.pattern, region, flags=re.MULTILINE):
line = probe_text.count("\n", 0, region_start + match.start()) + 1
matches.append({"path": relative, "line": line})
return {
"name": probe.name,
"path": probe.path,
"description": probe.description,
"found": bool(matches),
"count": len(matches),
"matches": matches,
}
def _parse_v8_version(root: Path) -> dict[str, int] | None:
text = _read_text(root / VERSION_HEADER)
if text is None:
return None
version: dict[str, int] = {}
for name, value in re.findall(r"#define\s+V8_(MAJOR|MINOR|BUILD|PATCH)_\w*\s+(\d+)", text):
version[name.lower()] = int(value)
required = {"major", "minor", "build", "patch"}
return version if required.issubset(version) else None
def audit_v8_source(root: Path) -> dict[str, Any]:
root = root.resolve()
version = _parse_v8_version(root)
missing_files = [
relative
for relative in {
VERSION_HEADER,
*(probe.path for probe in UNPATCHED_READ_PROBES if not probe.path.endswith("objects")),
}
if not (root / relative).exists()
]
missing_probe_inputs = [
probe.path
for probe in UNPATCHED_READ_PROBES
if not _iter_source_texts(root, probe.path)
]
direct_read_sites = [_probe(root, probe) for probe in UNPATCHED_READ_PROBES]
patch_markers = [_probe(root, probe) for probe in PATCH_MARKER_PROBES]
patched_read_paths = [_probe(root, probe) for probe in PATCHED_READ_PATH_PROBES]
found_direct_reads = sum(probe["count"] for probe in direct_read_sites)
found_patch_markers = sum(probe["count"] for probe in patch_markers)
found_patched_read_paths = sum(probe["count"] for probe in patched_read_paths)
patch_markers_complete = all(probe["found"] for probe in patch_markers)
read_paths_rewritten = found_direct_reads == 0
patched_read_paths_complete = all(probe["found"] for probe in patched_read_paths)
if missing_files or missing_probe_inputs:
diagnosis = "missing-source-files"
elif patch_markers_complete and read_paths_rewritten and patched_read_paths_complete:
diagnosis = "candidate-patched-read-paths-rewritten"
elif patch_markers_complete and read_paths_rewritten:
diagnosis = "candidate-patched-read-path-markers-missing"
elif patch_markers_complete:
diagnosis = "candidate-patched-direct-reads-remain"
elif found_patch_markers:
diagnosis = "partial-patch-markers"
elif found_direct_reads:
diagnosis = "unpatched-stock-v8"
else:
diagnosis = "unknown"
return {
"v8_root": str(root),
"version": version,
"diagnosis": diagnosis,
"missing_files": missing_files,
"missing_probe_inputs": missing_probe_inputs,
"direct_read_sites": direct_read_sites,
"patch_markers": patch_markers,
"patched_read_paths": patched_read_paths,
"counts": {
"direct_read_sites": found_direct_reads,
"patch_markers": found_patch_markers,
"patched_read_paths": found_patched_read_paths,
"missing_files": len(missing_files),
"missing_probe_inputs": len(missing_probe_inputs),
},
"patch_markers_complete": patch_markers_complete,
"read_paths_rewritten": read_paths_rewritten,
"patched_read_paths_complete": patched_read_paths_complete,
"notes": [
"This audit is not proof that a V8 fork implements wasm global live bindings correctly.",
"A passing runtime fix still needs focused Moli regressions and WPT evidence.",
],
}
def _format_human(audit: dict[str, Any]) -> str:
version = audit.get("version")
version_text = "unknown"
if isinstance(version, dict):
version_text = (
f"{version.get('major')}.{version.get('minor')}"
f".{version.get('build')}.{version.get('patch')}"
)
lines = [
f"v8 root: {audit['v8_root']}",
f"version: {version_text}",
f"diagnosis: {audit['diagnosis']}",
f"missing files: {audit['counts']['missing_files']}",
f"missing probe inputs: {audit['counts']['missing_probe_inputs']}",
f"direct module cell read sites: {audit['counts']['direct_read_sites']}",
f"patch markers: {audit['counts']['patch_markers']}",
f"patched read paths: {audit['counts']['patched_read_paths']}",
]
for group in ("direct_read_sites", "patch_markers", "patched_read_paths"):
lines.append(f"{group}:")
for entry in audit[group]:
state = "found" if entry["found"] else "missing"
lines.append(f" - {entry['name']}: {state} ({entry['count']})")
for match in entry["matches"][:5]:
lines.append(f" {match['path']}:{match['line']}")
if entry["count"] > 5:
lines.append(f" ... {entry['count'] - 5} more")
return "\n".join(lines)
def build_arg_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--v8-root", required=True, type=Path, help="V8 source root to audit")
parser.add_argument("--json-output", type=Path, help="write machine-readable audit JSON")
parser.add_argument(
"--json",
action="store_true",
help="print machine-readable audit JSON to stdout instead of a text summary",
)
parser.add_argument(
"--require-patched",
action="store_true",
help=(
"return nonzero unless patch markers are present, source files are "
"complete, and stock module cell read probes no longer match"
),
)
return parser
def main(argv: list[str] | None = None) -> int:
args = build_arg_parser().parse_args(argv)
audit = audit_v8_source(args.v8_root)
if args.json_output:
args.json_output.write_text(json.dumps(audit, indent=2, sort_keys=True), encoding="utf-8")
if args.json:
print(json.dumps(audit, indent=2, sort_keys=True))
else:
print(_format_human(audit))
if audit["missing_files"] or audit["missing_probe_inputs"]:
return 2
if args.require_patched and (
not audit["patch_markers_complete"]
or not audit["read_paths_rewritten"]
or not audit["patched_read_paths_complete"]
):
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))
moli-benchmark/moli_benchmark/wild_web.py
+419
View File
@@ -0,0 +1,419 @@
from __future__ import annotations
import html.parser
import os
from pathlib import Path
from typing import Any
from .artifacts import write_csv, write_json, write_text
from .chrome_dcl import run_chrome_dcl_dump, run_served_cdp_dcl_dump
from .config import REPO_ROOT, clear_proxy_env
from .process import run_process
from .stats import summarize
from .synthetic_compare import (
WEBFETCH_TARGETS,
target_enables_all_resource_fetch,
target_is_cdp,
target_metadata,
)
WILD_WEB_SEEDS: dict[str, str] = {
"baidu-home": "https://www.baidu.com/",
"bilibili-home": "https://www.bilibili.com/",
"zhihu-home": "https://www.zhihu.com/",
"toutiao-home": "https://www.toutiao.com/",
}
WILD_WEB_SEED_ASSERTIONS: dict[str, dict[str, Any]] = {
"baidu-home": {"title_any": ("百度", "baidu"), "text_any": ("百度", "baidu"), "min_text_length": 20},
"bilibili-home": {"title_any": ("哔哩", "bilibili"), "text_any": ("哔哩", "bilibili"), "min_text_length": 20},
"zhihu-home": {"title_any": ("知乎", "zhihu"), "text_any": ("知乎", "zhihu"), "min_text_length": 20},
"toutiao-home": {"title_any": ("头条", "toutiao"), "text_any": ("头条", "toutiao"), "min_text_length": 20},
}
class _TextExtractor(html.parser.HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self._hidden_depth = 0
self._in_title = False
self.title_parts: list[str] = []
self.text_parts: list[str] = []
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
if tag in {"script", "style", "noscript"}:
self._hidden_depth += 1
if tag == "title":
self._in_title = True
def handle_endtag(self, tag: str) -> None:
if tag in {"script", "style", "noscript"} and self._hidden_depth:
self._hidden_depth -= 1
if tag == "title":
self._in_title = False
def handle_data(self, data: str) -> None:
text = " ".join(data.split())
if not text:
return
if self._in_title:
self.title_parts.append(text)
if self._hidden_depth == 0:
self.text_parts.append(text)
def _extract_page_snapshot(stdout: bytes) -> dict[str, Any]:
text = stdout.decode("utf-8", errors="replace")
parser = _TextExtractor()
parser.feed(text)
title = " ".join(parser.title_parts).strip()
body_text = " ".join(parser.text_parts).strip()
return {
"title": title,
"text_length": len(body_text),
"text_sample": body_text[:500],
}
def _wild_web_extraction_failures(seed: str, snapshot: dict[str, Any]) -> list[str]:
assertions = WILD_WEB_SEED_ASSERTIONS.get(seed, {})
failures: list[str] = []
title = str(snapshot.get("title") or "")
sample = str(snapshot.get("text_sample") or "")
text_length = int(snapshot.get("text_length") or 0)
title_lower = title.lower()
sample_lower = sample.lower()
title_any = tuple(str(value).lower() for value in assertions.get("title_any", ()))
text_any = tuple(str(value).lower() for value in assertions.get("text_any", ()))
min_text_length = int(assertions.get("min_text_length", 1))
if not title:
failures.append("missing-title")
elif title_any and not any(value in title_lower for value in title_any):
failures.append("title-keyword-mismatch")
if text_length < min_text_length:
failures.append("short-body-text")
elif text_any and not any(value in sample_lower for value in text_any):
failures.append("text-keyword-mismatch")
return failures
def _wild_command_for_target(target: str, binary: Path, url: str, timeout_seconds: float) -> list[str]:
metadata = target_metadata(target)
if target_is_cdp(target):
raise RuntimeError(f"{target} is a CDP target; use the cdp-session suite")
if target == "chrome":
raise RuntimeError("chrome wild-web uses the CDP DCL runner")
timeout_ms = str(int(timeout_seconds * 1000))
if target in {"moli", "moli-full"}:
compatibility_args = (
["--layout", "--resource"]
if target_enables_all_resource_fetch(target)
else []
)
return [
str(binary),
"fetch",
*compatibility_args,
"--dump",
"html",
"--wait-until",
"done",
"--timeout",
timeout_ms,
url,
]
if target == "lightpanda":
return [
str(binary),
"fetch",
"--dump",
"html",
"--wait-until",
"done",
"--wait-ms",
timeout_ms,
"--http-timeout",
timeout_ms,
"--terminate-ms",
timeout_ms,
url,
]
if target == "obscura":
return [
str(binary),
"fetch",
"--dump",
"html",
"--wait-until",
"load",
"--wait",
"0",
"--timeout",
str(max(1, int(timeout_seconds))),
url,
]
raise RuntimeError(f"unknown target: {target}")
def _wild_web_target_metadata(target: str) -> dict[str, str]:
metadata = dict(target_metadata(target))
if target == "chrome" or target_is_cdp(target):
metadata["driver"] = "cdp-dcl"
prefix = "moli full" if target == "moli-full-cdp" else metadata["engine"]
metadata["label"] = f"{prefix} / cdp-dcl"
return metadata
def _classify(stdout: bytes, stderr: bytes, returncode: int | None, timed_out: bool) -> str:
if timed_out:
return "timeout"
if returncode != 0:
return "error"
snapshot = _extract_page_snapshot(stdout)
visible_text = " ".join(
str(snapshot.get(key) or "") for key in ("title", "text_sample")
).lower()
diagnostic_text = stderr.decode("utf-8", errors="replace").lower()
text = f"{visible_text}\n{diagnostic_text}"
if "captcha" in text or "verify" in text or "challenge" in text or "安全验证" in text:
return "challenge"
if "login" in text or "登录" in text:
return "login"
if "blocked" in text or "forbidden" in text or "403" in text:
return "blocked"
return "success" if stdout.strip() else "empty"
def _failure_kind(category: str, extraction_failures: list[str], error: str | None = None) -> str | None:
if error == "target binary unavailable":
return "target-unavailable"
if category == "timeout":
return "timeout"
if category == "blocked":
return "blocked"
if category == "challenge":
return "challenge"
if category == "login":
return "login"
if category == "empty":
return "empty-response"
if category == "error":
return "process-error"
if extraction_failures:
return "extraction-failure"
return None
def _write_failure_artifacts(
*,
suite_dir: Path,
row: dict[str, Any],
stdout: bytes,
stderr: bytes,
) -> str:
name = f"{row['target']}-run-{row['run']}-{row['seed']}"
failures_dir = suite_dir / "failures"
json_path = failures_dir / f"{name}.json"
stdout_path = failures_dir / f"{name}.stdout.html"
stderr_path = failures_dir / f"{name}.stderr.txt"
write_json(json_path, row)
write_text(stdout_path, stdout[-128 * 1024 :].decode("utf-8", errors="replace"))
write_text(stderr_path, stderr[-32 * 1024 :].decode("utf-8", errors="replace"))
return str(json_path.relative_to(suite_dir))
def _write_replay_artifact(
*,
suite_dir: Path,
row: dict[str, Any],
stdout: bytes,
) -> str:
name = f"{row['target']}-run-{row['run']}-{row['seed']}.html"
replay_dir = suite_dir / "replay"
replay_path = replay_dir / name
write_text(replay_path, stdout.decode("utf-8", errors="replace"))
return str(replay_path.relative_to(suite_dir))
def run_wild_web_suite(
*,
output_dir: Path,
target_matrix: dict[str, Any],
targets: tuple[str, ...],
seeds: tuple[str, ...],
runs: int,
timeout_seconds: float,
gate_target: str,
capture_replay: bool = False,
) -> dict[str, Any]:
unknown_targets = [target for target in targets if target not in WEBFETCH_TARGETS]
if unknown_targets:
raise RuntimeError(f"unknown webfetch target(s): {', '.join(unknown_targets)}")
if gate_target not in WEBFETCH_TARGETS:
raise RuntimeError(f"unknown gate target: {gate_target}")
suite_dir = output_dir / "wild-web"
rows: list[dict[str, Any]] = []
replay_manifest: list[dict[str, Any]] = []
selected = seeds or tuple(WILD_WEB_SEEDS.keys())
for seed in selected:
if seed not in WILD_WEB_SEEDS:
raise RuntimeError(f"unknown wild web seed: {seed}")
for target in targets:
metadata = _wild_web_target_metadata(target)
info = target_matrix.get(metadata["binary_key"], {})
path = info.get("path")
for run_id in range(1, runs + 1):
for seed in selected:
url = WILD_WEB_SEEDS[seed]
if not info.get("available") or not path:
row = {
"target": target,
**metadata,
"run": run_id,
"seed": seed,
"url": url,
"category": "error",
"classification_ok": False,
"extraction_ok": False,
"ok": False,
"extraction_failures": ["target-unavailable"],
"extraction_failure_count": 1,
"failure_kind": "target-unavailable",
"error": "target binary unavailable",
}
row["failure_artifact"] = _write_failure_artifacts(suite_dir=suite_dir, row=row, stdout=b"", stderr=b"")
rows.append(row)
continue
proc_env = clear_proxy_env(os.environ)
if target == "chrome":
result = run_chrome_dcl_dump(
Path(path),
url,
cwd=REPO_ROOT,
timeout_seconds=timeout_seconds,
env=proc_env,
)
elif target_is_cdp(target):
result = run_served_cdp_dcl_dump(
target,
Path(path),
url,
cwd=REPO_ROOT,
timeout_seconds=timeout_seconds,
env=proc_env,
)
else:
result = run_process(
_wild_command_for_target(target, Path(path), url, timeout_seconds),
cwd=REPO_ROOT,
timeout_seconds=timeout_seconds + 2,
env=proc_env,
)
category = _classify(result.stdout, result.stderr, result.returncode, result.timed_out)
snapshot = _extract_page_snapshot(result.stdout)
extraction_failures = _wild_web_extraction_failures(seed, snapshot)
classification_ok = category in {"success", "login", "challenge"}
extraction_ok = not extraction_failures
ok = classification_ok and extraction_ok
row = {
"target": target,
**metadata,
"run": run_id,
"seed": seed,
"url": url,
"category": category,
"classification_ok": classification_ok,
"extraction_ok": extraction_ok,
"ok": ok,
"title": snapshot["title"],
"text_length": snapshot["text_length"],
"text_sample": snapshot["text_sample"],
"extraction_failures": extraction_failures,
"extraction_failure_count": len(extraction_failures),
"failure_kind": None if ok else _failure_kind(category, extraction_failures),
"elapsed_ms": result.elapsed_ms,
"returncode": result.returncode,
"timed_out": result.timed_out,
"stdout_bytes": len(result.stdout),
"stderr_bytes": len(result.stderr),
"stdout_tail": result.stdout[-1024:].decode("utf-8", errors="replace"),
"stderr_tail": result.stderr[-1024:].decode("utf-8", errors="replace"),
"peak_pss_bytes": result.resources.get("peak_pss_bytes"),
}
if not ok:
row["failure_artifact"] = _write_failure_artifacts(
suite_dir=suite_dir,
row=row,
stdout=result.stdout,
stderr=result.stderr,
)
elif capture_replay:
row["replay_artifact"] = _write_replay_artifact(
suite_dir=suite_dir,
row=row,
stdout=result.stdout,
)
replay_manifest.append(
{
"target": target,
**metadata,
"run": run_id,
"seed": seed,
"url": url,
"category": category,
"title": snapshot["title"],
"text_length": snapshot["text_length"],
"artifact": row["replay_artifact"],
}
)
rows.append(row)
gate_failures = sum(1 for row in rows if row["target"] == gate_target and not row.get("ok"))
summary: dict[str, Any] = {
"suite": "wild-web",
"runs": runs,
"seeds": list(selected),
"timeout_seconds": timeout_seconds,
"gate_target": gate_target,
"gate_failures": gate_failures,
"targets": {},
"total_failures": sum(1 for row in rows if not row.get("ok")),
"replay_capture": bool(capture_replay),
"replay_artifacts": len(replay_manifest),
}
for target in targets:
target_rows = [row for row in rows if row["target"] == target]
categories: dict[str, int] = {}
failure_kinds: dict[str, int] = {}
for row in target_rows:
categories[row["category"]] = categories.get(row["category"], 0) + 1
failure_kind = row.get("failure_kind")
if isinstance(failure_kind, str) and failure_kind:
failure_kinds[failure_kind] = failure_kinds.get(failure_kind, 0) + 1
summary["targets"][target] = {
**_wild_web_target_metadata(target),
"seeds": len(target_rows),
"runs": runs,
"passes": sum(1 for row in target_rows if row.get("ok")),
"failures": sum(1 for row in target_rows if not row.get("ok")),
"extraction_failures": sum(int(row.get("extraction_failure_count", 0) or 0) for row in target_rows),
"categories": categories,
"failure_kinds": failure_kinds,
"elapsed_ms": summarize(row["elapsed_ms"] for row in target_rows if row.get("elapsed_ms") is not None),
"peak_pss_bytes": summarize(row["peak_pss_bytes"] for row in target_rows if row.get("peak_pss_bytes") is not None),
}
write_csv(suite_dir / "raw-runs.csv", rows)
write_json(suite_dir / "runs.json", rows)
write_json(suite_dir / "summary.json", summary)
if capture_replay:
write_json(
suite_dir / "replay" / "manifest.json",
{
"schema_version": 1,
"note": "Captured only when --capture-replay or --wild-web-capture-replay is explicitly provided. Caller is responsible for robots/ToS review before publishing.",
"artifacts": replay_manifest,
},
)
return summary
moli-benchmark/moli_benchmark/wpt.py
+355
View File
@@ -0,0 +1,355 @@
from __future__ import annotations
import json
import os
import shutil
from pathlib import Path
from typing import Any
from .artifacts import write_csv, write_json, write_text
from .config import REPO_ROOT, clear_proxy_env
from .process import run_process
REPORT_PREFIX = "moli-wpt-compat-report-wpt-compat-"
WPT_RUNNER = "nextest"
def _target_dir(env: dict[str, str]) -> Path:
raw = env.get("CARGO_TARGET_DIR")
return Path(raw).expanduser().resolve() if raw else REPO_ROOT / "target"
def _report_paths(target_dir: Path) -> list[Path]:
return sorted(target_dir.glob(f"{REPORT_PREFIX}*.json"))
def _clean_reports(target_dir: Path) -> None:
for path in target_dir.glob(f"{REPORT_PREFIX}*.json"):
path.unlink(missing_ok=True)
for path in target_dir.glob(f"{REPORT_PREFIX}*.md"):
path.unlink(missing_ok=True)
def _normalize_runner(runner: str | None) -> str:
if runner in (None, WPT_RUNNER):
return WPT_RUNNER
raise RuntimeError("WPT compat benchmarks only support the release nextest runner")
def _run_command(runner: str | None = WPT_RUNNER) -> list[str]:
_normalize_runner(runner)
return [
"cargo",
"nextest",
"run",
"-p",
"moli-core",
"--test",
"wpt_compat",
"--release",
"--no-fail-fast",
]
def _empty_counts() -> dict[str, int]:
return {
"total": 0,
"pass": 0,
"known_fail": 0,
"unexpected_fail": 0,
"unexpected_pass": 0,
"skip": 0,
}
def _add_rates(counts: dict[str, Any]) -> None:
total = int(counts.get("total", 0) or 0)
if total <= 0:
counts["pass_rate_percent"] = None
counts["unexpected_fail_rate_percent"] = None
counts["skip_rate_percent"] = None
return
counts["pass_rate_percent"] = (int(counts.get("pass", 0) or 0) / total) * 100.0
counts["unexpected_fail_rate_percent"] = (int(counts.get("unexpected_fail", 0) or 0) / total) * 100.0
counts["skip_rate_percent"] = (int(counts.get("skip", 0) or 0) / total) * 100.0
def _record_category(summary: dict[str, Any], category: str, tags: list[str]) -> None:
key = category.replace("-", "_")
summary["total"] += 1
if key in summary:
summary[key] += 1
for tag in tags:
tag_summary = summary["by_tag"].setdefault(tag, _empty_counts())
tag_summary["total"] += 1
if key in tag_summary:
tag_summary[key] += 1
def _collect_reports(target_dir: Path, suite_dir: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]:
reports_dir = suite_dir / "reports"
reports_dir.mkdir(parents=True, exist_ok=True)
summary: dict[str, Any] = {**_empty_counts(), "by_tag": {}}
cases: list[dict[str, Any]] = []
for path in _report_paths(target_dir):
data = json.loads(path.read_text(encoding="utf-8"))
shutil.copy2(path, reports_dir / path.name)
md_path = path.with_suffix(".md")
if md_path.exists():
shutil.copy2(md_path, reports_dir / md_path.name)
for case in data.get("cases", []):
category = str(case.get("category", "unexpected-fail"))
tags = [str(tag) for tag in case.get("tags", [])]
_record_category(summary, category, tags)
cases.append(
{
"id": case.get("id"),
"expected": case.get("expected"),
"actual": case.get("actual"),
"category": category,
"tags": ",".join(tags),
"failures": " | ".join(str(item) for item in case.get("failures", [])),
}
)
_add_rates(summary)
for tag_summary in summary["by_tag"].values():
_add_rates(tag_summary)
return summary, cases
def _by_tag_rows(summary: dict[str, Any]) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for tag, counts in sorted(summary.get("by_tag", {}).items()):
rows.append({"tag": tag, **counts})
return rows
def _case_key(case: dict[str, Any]) -> str:
return str(case.get("id") or "")
def _case_tags(case: dict[str, Any]) -> str:
tags = case.get("tags", "")
if isinstance(tags, list):
return ",".join(str(tag) for tag in tags)
return str(tags or "")
def _case_index(cases: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
return {_case_key(case): case for case in cases if _case_key(case)}
def _load_baseline_cases(path: Path | None) -> list[dict[str, Any]]:
if path is None:
return []
data = json.loads(path.read_text(encoding="utf-8"))
cases = data.get("cases") if isinstance(data, dict) else data
if not isinstance(cases, list):
raise RuntimeError(f"WPT baseline does not contain a cases list: {path}")
return [case for case in cases if isinstance(case, dict)]
def _wpt_case_diff(current_cases: list[dict[str, Any]], baseline_cases: list[dict[str, Any]]) -> tuple[dict[str, int], list[dict[str, Any]]]:
current = _case_index(current_cases)
baseline = _case_index(baseline_cases)
rows: list[dict[str, Any]] = []
for case_id in sorted(current.keys() - baseline.keys()):
case = current[case_id]
rows.append(
{
"kind": "added",
"id": case_id,
"baseline_expected": None,
"current_expected": case.get("expected"),
"baseline_category": None,
"current_category": case.get("category"),
"baseline_actual": None,
"current_actual": case.get("actual"),
"tags": _case_tags(case),
}
)
for case_id in sorted(baseline.keys() - current.keys()):
case = baseline[case_id]
rows.append(
{
"kind": "removed",
"id": case_id,
"baseline_expected": case.get("expected"),
"current_expected": None,
"baseline_category": case.get("category"),
"current_category": None,
"baseline_actual": case.get("actual"),
"current_actual": None,
"tags": _case_tags(case),
}
)
for case_id in sorted(current.keys() & baseline.keys()):
current_case = current[case_id]
baseline_case = baseline[case_id]
if baseline_case.get("expected") != current_case.get("expected"):
rows.append(
{
"kind": "expectation-change",
"id": case_id,
"baseline_expected": baseline_case.get("expected"),
"current_expected": current_case.get("expected"),
"baseline_category": baseline_case.get("category"),
"current_category": current_case.get("category"),
"baseline_actual": baseline_case.get("actual"),
"current_actual": current_case.get("actual"),
"tags": _case_tags(current_case),
}
)
if baseline_case.get("category") != current_case.get("category"):
rows.append(
{
"kind": "category-change",
"id": case_id,
"baseline_expected": baseline_case.get("expected"),
"current_expected": current_case.get("expected"),
"baseline_category": baseline_case.get("category"),
"current_category": current_case.get("category"),
"baseline_actual": baseline_case.get("actual"),
"current_actual": current_case.get("actual"),
"tags": _case_tags(current_case),
}
)
summary = {
"added": sum(1 for row in rows if row["kind"] == "added"),
"removed": sum(1 for row in rows if row["kind"] == "removed"),
"expectation_changes": sum(1 for row in rows if row["kind"] == "expectation-change"),
"category_changes": sum(1 for row in rows if row["kind"] == "category-change"),
"total_changes": len(rows),
}
return summary, rows
def _rate(value: Any) -> str:
return "n/a" if value is None else f"{float(value):.2f}%"
def _markdown(summary: dict[str, Any], command_summary: dict[str, Any] | None, diff_summary: dict[str, Any] | None) -> str:
lines = [
"# WPT compat benchmark",
"",
f"- total: `{summary['total']}`",
f"- pass: `{summary['pass']}`",
f"- known-fail: `{summary['known_fail']}`",
f"- unexpected-fail: `{summary['unexpected_fail']}`",
f"- unexpected-pass: `{summary['unexpected_pass']}`",
f"- skip: `{summary['skip']}`",
f"- pass rate: `{_rate(summary.get('pass_rate_percent'))}`",
f"- unexpected-fail rate: `{_rate(summary.get('unexpected_fail_rate_percent'))}`",
f"- skip rate: `{_rate(summary.get('skip_rate_percent'))}`",
"",
]
if command_summary is not None:
lines.extend(
[
"Run command:",
"",
"```text",
" ".join(command_summary["command"]),
"```",
"",
f"- returncode: `{command_summary['returncode']}`",
f"- timed out: `{command_summary['timed_out']}`",
f"- elapsed ms: `{command_summary['elapsed_ms']:.3f}`",
"",
]
)
if summary["by_tag"]:
lines.append("By tag:")
for tag, counts in sorted(summary["by_tag"].items()):
lines.append(
f"- `{tag}`: total {counts['total']}, pass {counts['pass']}, "
f"unexpected-fail {counts['unexpected_fail']}, skip {counts['skip']}, "
f"pass rate {_rate(counts.get('pass_rate_percent'))}"
)
lines.append("")
if diff_summary is not None:
lines.extend(
[
"Baseline diff:",
"",
f"- added: `{diff_summary['added']}`",
f"- removed: `{diff_summary['removed']}`",
f"- expectation changes: `{diff_summary['expectation_changes']}`",
f"- category changes: `{diff_summary['category_changes']}`",
"",
]
)
return "\n".join(lines)
def run_wpt_suite(
*,
output_dir: Path,
timeout_seconds: float,
compat: str | None,
case_filter: str | None,
tag_filter: str | None,
no_run: bool,
runner: str | None = WPT_RUNNER,
baseline: Path | None = None,
) -> dict[str, Any]:
runner = _normalize_runner(runner)
suite_dir = output_dir / "wpt"
env = clear_proxy_env(os.environ)
if compat:
env["MOLI_WPT_COMPAT"] = compat
if case_filter:
env["MOLI_WPT_CASE"] = case_filter
if tag_filter:
env["MOLI_WPT_TAG"] = tag_filter
target_dir = _target_dir(env)
command_summary: dict[str, Any] | None = None
if not no_run:
_clean_reports(target_dir)
result = run_process(
_run_command(runner),
cwd=REPO_ROOT,
timeout_seconds=timeout_seconds,
env=env,
)
command_summary = result.json_summary(include_output=result.returncode != 0 or result.timed_out)
write_json(suite_dir / "process.json", command_summary)
compat_summary, cases = _collect_reports(target_dir, suite_dir)
diff_summary = None
if baseline is not None:
baseline_cases = _load_baseline_cases(baseline)
diff_summary, diff_rows = _wpt_case_diff(cases, baseline_cases)
write_json(suite_dir / "diff.json", {"baseline": str(baseline), "summary": diff_summary, "cases": diff_rows})
write_csv(suite_dir / "diff.csv", diff_rows)
total_failures = compat_summary["unexpected_fail"] + compat_summary["unexpected_pass"] + compat_summary["known_fail"] + compat_summary["skip"]
if command_summary is not None and (command_summary["returncode"] != 0 or command_summary["timed_out"]):
total_failures += 1
summary = {
"suite": "wpt",
"runner": runner,
"selection": {
"compat": compat,
"case": case_filter,
"tag": tag_filter,
"no_run": no_run,
},
"summary": compat_summary,
"cases": len(cases),
"total_failures": total_failures,
}
if baseline is not None:
summary["baseline"] = str(baseline)
summary["diff"] = diff_summary
write_json(suite_dir / "moli-wpt-compat-report.json", {"summary": compat_summary, "cases": cases})
write_csv(suite_dir / "raw-runs.csv", cases)
write_csv(suite_dir / "by-tag.csv", _by_tag_rows(compat_summary))
write_json(suite_dir / "summary.json", summary)
write_text(suite_dir / "summary.md", _markdown(compat_summary, command_summary, diff_summary))
return summary
moli-benchmark/moli_benchmark/wpt_cross/__init__.py
+23
View File
@@ -0,0 +1,23 @@
"""Cross-engine WPT runner.
Runs identical Web Platform Tests cases against multiple headless engines
(Moli, Lightpanda, Obscura) using the same fixture server, the same
upstream WPT case URLs, and the same CDP-based testharness completion bridge.
The goal is to produce an objective WebAPI coverage matrix: same case set,
same harness, same timeout, no per-engine source patches.
"""
from .engine import EngineDriver, ENGINES, build_driver
from .server import WptFixtureServer
from .runner import CaseResult, EngineRunResult, run_engine_on_cases
__all__ = [
"EngineDriver",
"ENGINES",
"build_driver",
"WptFixtureServer",
"CaseResult",
"EngineRunResult",
"run_engine_on_cases",
]
moli-benchmark/moli_benchmark/wpt_cross/__main__.py
+795
View File
@@ -0,0 +1,795 @@
"""CLI entry point for cross-engine WPT runs.
Usage:
python -m moli_benchmark.wpt_cross \\
--wpt-root ../wpt \\
--engine moli --engine lightpanda --engine obscura \\
--output-dir /tmp/moli-wpt-cross \\
--limit 20
This is the v1 driver verification harness. It:
1. Enumerates the default blacklist-filtered WPT case set from ``--wpt-root``.
2. Starts a single fixture server (loopback + optional global IPv6 for Obscura).
3. For each engine, launches it via :class:`EngineDriver`, runs every case
through CDP + the testharness completion bridge, and writes per-engine
JSON results.
4. Emits ``matrix.json`` with the cross-engine pass/fail/timeout/crash table.
"""
from __future__ import annotations
import argparse
import json
import sys
import time
from collections import Counter
from pathlib import Path
from typing import Any
from .case_set import ANY_JS_GLOBAL_CHOICES, WptCase, enumerate_cases, explicit_case
from .cli_runner import run_engine_on_cases_cli
from .engine import ENGINES, build_driver
from .audit import audit_matrix, load_known_failure_manifest
from .runner import MAX_RECORDED_FAILURES, engine_result_to_dict, run_engine_on_cases
from .scheduler import build_run_schedule, write_run_schedule
from ..config import clear_current_proxy_env
REPO_CASE_LIST_DIR = Path(__file__).resolve().parents[2] / "wpt-cross-current"
WPT_CROSS_CASE_TIMEOUT_SECONDS = 120.0
WPT_CROSS_PARALLELISM = 100
CASE_LIST_FILES = {
"pass": "passed-cases.txt",
"fail": "failed-cases.txt",
"timeout": "timeout-cases.txt",
"crash": "crash-cases.txt",
"harness-stalled": "harness-stalled-cases.txt",
"error": "error-cases.txt",
"missing": "missing-cases.txt",
"other": "other-cases.txt",
}
NON_TRUSTWORTHY_ORIGIN_CASES = frozenset(
{
"audio-output/secure-context.html",
"credential-management/require_securecontext.html",
"pointerevents/pointerevent_constructor.html",
}
)
def _case_requires_trustworthy_origin(case_path: str) -> bool:
file_part = case_path.split("?", 1)[0].split("#", 1)[0]
return ".https." in file_part or file_part.endswith(".https.html")
def _case_requires_non_trustworthy_origin(case_path: str) -> bool:
file_part = case_path.split("?", 1)[0].split("#", 1)[0]
return (
".http." in file_part
or file_part.endswith("_insecure_context.html")
or file_part.startswith("secure-contexts/")
or file_part in NON_TRUSTWORTHY_ORIGIN_CASES
)
def _url_for_case_origin(server: Any, case_path: str, *, external: bool) -> str:
if external:
return server.url_for_case(case_path, external=True)
if _case_requires_trustworthy_origin(case_path):
return server.url_for_case(case_path, external=False)
if _case_requires_non_trustworthy_origin(case_path) and server.external_base_url:
return server.url_for_case(case_path, external=True)
return server.url_for_case(case_path, external=False)
def _build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog="python -m moli_benchmark.wpt_cross",
description="Run a curated WPT case set across multiple headless engines.",
)
parser.add_argument(
"--wpt-root",
type=Path,
required=True,
help="Path to the upstream WPT checkout (e.g. ../wpt).",
)
parser.add_argument(
"--engine",
action="append",
required=True,
choices=sorted(ENGINES),
help="Engine to run (repeatable). Each engine is launched in turn.",
)
parser.add_argument(
"--output-dir",
type=Path,
required=True,
help="Directory to write per-engine results and matrix.json.",
)
parser.add_argument(
"--limit",
type=int,
default=None,
help="Max number of cases to run (after filter). Useful for smoke runs.",
)
parser.add_argument(
"--dir-prefix",
action="append",
default=None,
help=(
"Restrict enumeration to a WPT directory prefix (repeatable). "
"When omitted, the runner scans the whole WPT tree and applies "
"the default rendering/layout blacklist."
),
)
parser.add_argument(
"--include-tentative",
action="store_true",
help=(
"Include .tentative.* cases during --dir-prefix enumeration. "
"Default directory enumeration keeps tentative cases out of the "
"stable broad baseline."
),
)
parser.add_argument(
"--any-js-global",
choices=ANY_JS_GLOBAL_CHOICES,
default="none",
help=(
"Include .any.js cases during --dir-prefix enumeration for the "
"selected global: window, dedicatedworker, both, or none "
"(default). The broad baseline includes both supported globals "
"for Streams automatically. Explicit --case path.any.js still "
"runs the window variant by default."
),
)
parser.add_argument(
"--case",
action="append",
default=None,
help=(
"Run an explicit WPT case path (repeatable). This bypasses the "
"curated directory filters and is useful for tentative or "
"testdriver-backed investigations."
),
)
parser.add_argument(
"--launch-timeout",
type=float,
default=30.0,
help="Per-engine launch (CDP ready) timeout in seconds.",
)
parser.add_argument(
"--moli-bin",
type=str,
default=None,
help="Override MOLI_BIN.",
)
parser.add_argument(
"--lightpanda-bin",
type=str,
default=None,
help="Override LIGHTPANDA_BIN.",
)
parser.add_argument(
"--obscura-bin",
type=str,
default=None,
help="Override OBSCURA_BIN.",
)
parser.add_argument(
"--chrome-bin",
type=str,
default=None,
help="Override CHROME_BIN.",
)
parser.add_argument(
"--mode",
choices=("auto", "cli", "cdp"),
default="auto",
help=(
"Driver mode: 'cli' uses each engine's `fetch` CLI plus an HTTP "
"callback bridge (no CDP); 'cdp' uses the CDP runner; 'auto' "
"(default) picks CLI when the engine's cli_fetch_command is set "
"(currently moli, lightpanda) and falls back to CDP "
"otherwise (obscura, chrome)."
),
)
parser.add_argument(
"--known-failures",
type=Path,
default=None,
help=(
"Optional known-failure manifest to audit after writing matrix.json. "
"This does not rewrite case statuses; it only reports unexpected "
"failures, changed known-failure shapes, missing expected failures, "
"and resolved known failures. Any non-empty audit bucket except "
"known_failures makes the run fail."
),
)
parser.add_argument(
"--known-failures-engine",
choices=sorted(ENGINES),
default=None,
help=(
"Engine to audit with --known-failures. Default: manifest engine "
"field when present, otherwise the first --engine value."
),
)
parser.add_argument(
"--allow-missing-known-failures",
action="store_true",
help=(
"Allow --known-failures rules that are absent from this matrix. "
"Use only with focused --case / small-slice investigations; full "
"baseline audits should keep the default strict behavior."
),
)
return parser
def _binary_override_for(engine: str, args: argparse.Namespace) -> str | None:
return getattr(args, f"{engine}_bin", None)
def _build_matrix(
cases: list[WptCase],
engine_results: dict[str, list[dict[str, Any]]],
) -> list[dict[str, Any]]:
by_engine_case: dict[str, dict[str, dict[str, Any]]] = {
engine: {case_dict["case_path"]: case_dict for case_dict in engine_cases}
for engine, engine_cases in engine_results.items()
}
matrix = []
for case in cases:
row: dict[str, Any] = {
"case_path": case.case_path,
"results": {},
}
for engine in engine_results:
r = by_engine_case[engine].get(case.case_path)
if r is None:
row["results"][engine] = {"status": "missing"}
else:
row["results"][engine] = {
"status": r["status"],
"duration_ms": r["duration_ms"],
"subtests": r["subtests"],
"failures": r.get("failures", []),
"failure_names": r.get("failure_names", []),
"harness_status_name": r.get("harness_status_name"),
"harness_message": r.get("harness_message"),
"error": r.get("error"),
}
matrix.append(row)
return matrix
def _summarize(matrix: list[dict[str, Any]], engines: list[str]) -> dict[str, Any]:
summary: dict[str, Any] = {"total": len(matrix), "engines": {}}
for engine in engines:
counter: Counter[str] = Counter()
for row in matrix:
counter[row["results"][engine]["status"]] += 1
summary["engines"][engine] = dict(counter)
return summary
def _primary_case_list_engine(engines: list[str]) -> str:
return "moli" if "moli" in engines else engines[0]
def _write_case_list(path: Path, cases: list[str]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
content = "".join(f"{case}\n" for case in cases)
temp_path = path.with_name(f".{path.name}.tmp")
temp_path.write_text(content, encoding="utf-8")
temp_path.replace(path)
def _write_repo_case_lists(
matrix: list[dict[str, Any]],
engines: list[str],
*,
case_list_dir: Path | None = None,
) -> None:
if case_list_dir is None:
case_list_dir = REPO_CASE_LIST_DIR
engine = _primary_case_list_engine(engines)
cases_by_status: dict[str, list[str]] = {
status: [] for status in CASE_LIST_FILES
}
for row in matrix:
case_path = str(row.get("case_path", ""))
status = (
row.get("results", {})
.get(engine, {})
.get("status", "missing")
)
bucket = status if status in CASE_LIST_FILES else "other"
cases_by_status[bucket].append(case_path)
for status, file_name in CASE_LIST_FILES.items():
cases = cases_by_status[status]
cases.sort()
_write_case_list(case_list_dir / file_name, cases)
def _is_full_case_list_run(args: argparse.Namespace) -> bool:
return (
args.case is None
and args.dir_prefix is None
and args.limit is None
and not args.include_tentative
and args.any_js_global == "none"
)
def _recorded_failure_names(result: dict[str, Any]) -> dict[str, str]:
names: dict[str, str] = {}
for index, failure in enumerate(result.get("failures", [])):
if not isinstance(failure, dict):
continue
name = failure.get("name")
if not isinstance(name, str) or not name:
status = failure.get("status_name") or failure.get("status") or "failure"
name = f"<unnamed {status} #{index + 1}>"
message = failure.get("message")
names[name] = message if isinstance(message, str) else ""
return names
def _failure_name_set(result: dict[str, Any]) -> set[str]:
names = result.get("failure_names")
if isinstance(names, list):
return {name for name in names if isinstance(name, str)}
return set(_recorded_failure_names(result))
def _recorded_failure_drift(
matrix: list[dict[str, Any]],
engines: list[str],
*,
primary: str = "moli",
max_examples: int = 8,
) -> dict[str, Any]:
if primary not in engines:
primary = engines[0] if engines else "moli"
comparisons = []
for row in matrix:
results = row.get("results", {})
primary_result = results.get(primary, {})
if not isinstance(primary_result, dict):
continue
primary_failure_names = _failure_name_set(primary_result)
primary_failures = _recorded_failure_names(primary_result)
for peer in engines:
if peer == primary:
continue
peer_result = results.get(peer, {})
if not isinstance(peer_result, dict):
continue
peer_failure_names = _failure_name_set(peer_result)
peer_failures = _recorded_failure_names(peer_result)
primary_only = sorted(primary_failure_names - peer_failure_names)
peer_only = sorted(peer_failure_names - primary_failure_names)
message_diffs = sorted(
name
for name in set(primary_failures) & set(peer_failures)
if primary_failures[name] != peer_failures[name]
)
if not primary_only and not peer_only and not message_diffs:
continue
comparisons.append(
{
"case_path": row.get("case_path", ""),
"primary": primary,
"peer": peer,
"primary_only_count": len(primary_only),
"peer_only_count": len(peer_only),
"message_diff_count": len(message_diffs),
"primary_only_examples": primary_only[:max_examples],
"peer_only_examples": peer_only[:max_examples],
"message_diff_examples": message_diffs[:max_examples],
}
)
primary_only_count = sum(1 for row in comparisons if row["primary_only_count"] > 0)
peer_only_count = sum(1 for row in comparisons if row["peer_only_count"] > 0)
message_diff_count = sum(1 for row in comparisons if row["message_diff_count"] > 0)
return {
"primary": primary,
"failure_name_source": "full_failure_names",
"recorded_failure_limit_per_engine": MAX_RECORDED_FAILURES,
"comparison_count": len(comparisons),
"primary_only_comparison_count": primary_only_count,
"peer_only_comparison_count": peer_only_count,
"message_diff_comparison_count": message_diff_count,
"comparisons": comparisons,
}
def _case_timeout(base_seconds: float, case: WptCase) -> float:
del case
return base_seconds
STEP_TIMEOUT_SENSITIVE_CASES = frozenset(
{
"html/semantics/scripting-1/the-script-element/module/dynamic-import/delay-load-event.html",
}
)
def _harness_timeout_multiplier(
case_timeout_seconds: float,
default_harness_timeout_seconds: float,
case_path: str | None = None,
) -> float:
case_path = case_path.split("?", 1)[0].split("#", 1)[0] if case_path else None
if case_path in STEP_TIMEOUT_SENSITIVE_CASES:
return 1.0
return max(1.0, case_timeout_seconds / default_harness_timeout_seconds)
def main(argv: list[str] | None = None) -> int:
clear_current_proxy_env()
parser = _build_parser()
args = parser.parse_args(argv)
output_dir: Path = args.output_dir
output_dir.mkdir(parents=True, exist_ok=True)
known_failure_rules: list[dict[str, Any]] | None = None
known_failure_categories: dict[str, Any] | None = None
known_failure_engine: str | None = None
if args.known_failures is not None:
try:
known_failure_manifest = load_known_failure_manifest(args.known_failures)
except Exception as exc:
print(f"error: {exc}", file=sys.stderr)
return 2
known_failure_rules = known_failure_manifest["rules"]
known_failure_categories = known_failure_manifest.get("categories")
manifest_engine = known_failure_manifest.get("engine")
if args.known_failures_engine is not None:
known_failure_engine = args.known_failures_engine
elif isinstance(manifest_engine, str) and manifest_engine:
known_failure_engine = manifest_engine
else:
known_failure_engine = args.engine[0]
if known_failure_engine not in args.engine:
print(
"error: --known-failures targets engine "
f"{known_failure_engine!r}, but selected engines are {args.engine!r}",
file=sys.stderr,
)
return 2
if args.case:
cases = [explicit_case(args.wpt_root, case_path) for case_path in args.case]
if args.limit is not None:
cases = cases[: args.limit]
else:
prefixes = tuple(args.dir_prefix) if args.dir_prefix else None
cases = enumerate_cases(
args.wpt_root,
dir_prefixes=prefixes,
include_tentative=args.include_tentative,
any_js_global=args.any_js_global,
limit=args.limit,
)
if not cases:
print("error: no WPT cases selected; check --wpt-root and --dir-prefix", file=sys.stderr)
return 2
case_list_path = output_dir / "cases.txt"
case_list_path.write_text("\n".join(c.case_path for c in cases) + "\n", encoding="utf-8")
scheduled_cases, schedule_metadata = build_run_schedule(
cases,
case_path=lambda case: case.case_path,
)
write_run_schedule(
output_dir,
metadata=schedule_metadata,
scheduled_case_paths=[case.case_path for case in scheduled_cases],
)
# Fixture server is started lazily (we need it before ANY engine runs).
from .server import DEFAULT_TESTHARNESS_TIMEOUT_SECONDS, WptFixtureServer
started_at = time.perf_counter()
engine_results: dict[str, list[dict[str, Any]]] = {}
engine_metadata: dict[str, dict[str, Any]] = {}
with WptFixtureServer(args.wpt_root) as server:
meta_path = output_dir / "fixture-server.json"
meta_path.write_text(
json.dumps(
{
"wpt_root": str(server.wpt_root),
"loopback_base_url": server.base_url,
"loopback_alternate_base_url": server.alternate_base_url,
"external_base_url": server.external_base_url,
"external_alternate_base_url": server.external_alternate_base_url,
"external_remote_base_url": server.external_remote_base_url,
"external_host": server.external_host,
},
indent=2,
sort_keys=True,
),
encoding="utf-8",
)
for engine in args.engine:
driver = build_driver(engine)
external = engine == "obscura"
mode_for_check = args.mode
if mode_for_check == "auto":
mode_for_check = "cli" if driver.cli_fetch_command is not None else "cdp"
if external and mode_for_check == "cdp" and server.external_base_url is None:
print(
f"error: engine {engine} requires global IPv6 fixture host but none was detected",
file=sys.stderr,
)
return 3
engine_started = time.perf_counter()
engine_case_timeout = WPT_CROSS_CASE_TIMEOUT_SECONDS
server.set_harness_timeout_multipliers(
{
case.case_path: _harness_timeout_multiplier(
_case_timeout(engine_case_timeout, case),
DEFAULT_TESTHARNESS_TIMEOUT_SECONDS,
case.case_path,
)
for case in cases
},
default_multiplier=_harness_timeout_multiplier(
engine_case_timeout,
DEFAULT_TESTHARNESS_TIMEOUT_SECONDS,
),
)
cases_for_engine = [
(
case.case_path,
_url_for_case_origin(server, case.case_path, external=external),
_case_timeout(engine_case_timeout, case),
)
for case in cases
]
scheduled_cases_for_engine = [
(
case.case_path,
_url_for_case_origin(server, case.case_path, external=external),
_case_timeout(engine_case_timeout, case),
)
for case in scheduled_cases
]
mode = args.mode
if mode == "auto":
mode = "cli" if driver.cli_fetch_command is not None else "cdp"
elif mode == "cli" and driver.cli_fetch_command is None:
print(
f"error: engine {engine} has no cli_fetch_command; cannot use --mode cli",
file=sys.stderr,
)
return 4
if mode == "cli":
# CLI HTTP-callback runner: bridge POSTs results via fetch().
# Keep secure-context negative cases off localhost, which
# browsers treat as potentially trustworthy. Other cases stay
# on loopback because many WPTs build cross-origin URLs by
# string-editing location.host, which breaks on IPv6 literals.
cases_for_engine_cli = [
(
case.case_path,
_url_for_case_origin(server, case.case_path, external=external),
_case_timeout(engine_case_timeout, case),
_harness_timeout_multiplier(
_case_timeout(engine_case_timeout, case),
DEFAULT_TESTHARNESS_TIMEOUT_SECONDS,
case.case_path,
),
)
for case in cases
]
scheduled_cases_for_engine_cli = [
(
case.case_path,
_url_for_case_origin(server, case.case_path, external=external),
_case_timeout(engine_case_timeout, case),
_harness_timeout_multiplier(
_case_timeout(engine_case_timeout, case),
DEFAULT_TESTHARNESS_TIMEOUT_SECONDS,
case.case_path,
),
)
for case in scheduled_cases
]
print(
f"[wpt-cross] running {engine} (CLI mode) on {len(cases_for_engine_cli)} cases via fixture origins",
file=sys.stderr,
)
result = run_engine_on_cases_cli(
driver=driver,
fixture_server=server,
cases=cases_for_engine_cli,
execution_cases=scheduled_cases_for_engine_cli,
binary_override=_binary_override_for(engine, args),
case_timeout_seconds=engine_case_timeout,
parallelism=WPT_CROSS_PARALLELISM,
)
else:
print(
f"[wpt-cross] running {engine} (CDP mode) on {len(cases_for_engine)} cases via fixture origins",
file=sys.stderr,
)
cdp_par = WPT_CROSS_PARALLELISM
if cdp_par == 1 or len(cases_for_engine) <= 1:
result = run_engine_on_cases(
driver=driver,
cases=scheduled_cases_for_engine,
binary_override=_binary_override_for(engine, args),
case_timeout_seconds=engine_case_timeout,
launch_timeout_seconds=args.launch_timeout,
)
by_path = {case.case_path: case for case in result.cases}
result.cases = [
by_path[case[0]]
for case in cases_for_engine
if case[0] in by_path
]
else:
print(
f"[wpt-cross] {engine}: launching {cdp_par} parallel CDP workers",
file=sys.stderr,
)
chunks = [[] for _ in range(cdp_par)]
for i, case in enumerate(scheduled_cases_for_engine):
chunks[i % cdp_par].append(case)
chunks = [c for c in chunks if c]
from concurrent.futures import ThreadPoolExecutor as _TPE
from .runner import EngineRunResult as _EngineRunResult
with _TPE(max_workers=len(chunks)) as pool:
futures = [
pool.submit(
run_engine_on_cases,
driver=driver,
cases=chunk,
binary_override=_binary_override_for(engine, args),
case_timeout_seconds=engine_case_timeout,
launch_timeout_seconds=args.launch_timeout,
)
for chunk in chunks
]
partials = [f.result() for f in futures]
base = partials[0]
merged_cases = list(base.cases)
setup_errors = [p.setup_error for p in partials if p.setup_error]
shutdown_infos = [p.shutdown_info for p in partials]
for p in partials[1:]:
merged_cases.extend(p.cases)
by_path = {c.case_path: c for c in merged_cases}
merged_cases = [
by_path[case[0]]
for case in cases_for_engine
if case[0] in by_path
]
result = _EngineRunResult(
engine=base.engine,
binary=base.binary,
binary_sha256=base.binary_sha256,
binary_version=base.binary_version,
endpoint=base.endpoint,
ready_ms=base.ready_ms,
setup_error="; ".join(setup_errors) if setup_errors else None,
cases=merged_cases,
shutdown_info={"workers": shutdown_infos, "parallelism": len(chunks)},
)
engine_elapsed = time.perf_counter() - engine_started
if not hasattr(result, "shutdown_info") or result.shutdown_info is None:
result.shutdown_info = {}
result.shutdown_info["run_schedule"] = schedule_metadata
result_dict = engine_result_to_dict(result)
(output_dir / f"engine-{engine}.json").write_text(
json.dumps(result_dict, indent=2, sort_keys=True), encoding="utf-8"
)
engine_results[engine] = result_dict["cases"]
engine_metadata[engine] = {
"binary": result_dict.get("binary"),
"binary_sha256": result_dict.get("binary_sha256"),
"binary_version": result_dict.get("binary_version"),
"endpoint": result_dict.get("endpoint"),
"ready_ms": result_dict.get("ready_ms"),
"setup_error": result_dict.get("setup_error"),
"elapsed_seconds": engine_elapsed,
"run_schedule": schedule_metadata,
}
if result.setup_error:
print(
f"[wpt-cross] {engine} setup error: {result.setup_error}",
file=sys.stderr,
)
matrix = _build_matrix(cases, engine_results)
summary = _summarize(matrix, list(args.engine))
summary["recorded_failure_drift"] = _recorded_failure_drift(matrix, list(args.engine))
summary["total_elapsed_seconds"] = time.perf_counter() - started_at
summary["engine_metadata"] = engine_metadata
summary["run_schedule"] = schedule_metadata
if _is_full_case_list_run(args):
_write_repo_case_lists(matrix, list(args.engine))
else:
print("[wpt-cross] skipped repo case list refresh for non-full run")
known_failure_audit: dict[str, Any] | None = None
if known_failure_rules is not None and known_failure_engine is not None:
known_failure_audit = audit_matrix(
matrix,
known_failure_engine,
known_failure_rules,
categories=known_failure_categories,
allow_missing_known_failures=args.allow_missing_known_failures,
)
audit_name = f"known-failure-audit-{known_failure_engine}.json"
(output_dir / audit_name).write_text(
json.dumps(known_failure_audit, indent=2, sort_keys=True),
encoding="utf-8",
)
summary["known_failure_audits"] = {
known_failure_engine: {
"manifest": str(args.known_failures),
"output": audit_name,
"ok": known_failure_audit["ok"],
"allow_missing_known_failures": args.allow_missing_known_failures,
"categories": known_failure_audit["categories"],
"counts": known_failure_audit["counts"],
"category_counts": known_failure_audit["category_counts"],
}
}
(output_dir / "matrix.json").write_text(
json.dumps(matrix, indent=2, sort_keys=True), encoding="utf-8"
)
(output_dir / "summary.json").write_text(
json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8"
)
try:
from .render_html import render_html
html_path = render_html(output_dir)
print(f"[wpt-cross] wrote html report: {html_path}")
except Exception as exc:
print(f"[wpt-cross] html render failed: {exc}", file=sys.stderr)
# Compact text summary on stdout.
print(f"\n=== WPT cross-engine summary ({len(cases)} cases) ===")
for engine in args.engine:
counts = summary["engines"][engine]
ordered = ", ".join(f"{k}={v}" for k, v in sorted(counts.items()))
print(f" {engine}: {ordered}")
if known_failure_audit is not None and known_failure_engine is not None:
counts = known_failure_audit["counts"]
print(
f" {known_failure_engine} known-failure audit: "
"known={known_failures}, resolved={resolved_known_failures}, "
"mismatched={mismatched_known_failures}, "
"missing={missing_expected_failures}, "
"skipped={skipped_known_failures}, "
"unexpected={unexpected_failures}".format(**counts)
)
print(f"output: {output_dir}")
if known_failure_audit is not None and not known_failure_audit["ok"]:
print(
"[wpt-cross] known-failure audit found unexpected, changed, missing, or resolved failures",
file=sys.stderr,
)
return 5
return 0
if __name__ == "__main__":
sys.exit(main())
moli-benchmark/moli_benchmark/wpt_cross/any_js.py
+151
View File
@@ -0,0 +1,151 @@
"""Helpers for running WPT ``.any.js`` fixtures through a window wrapper."""
from __future__ import annotations
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
ANY_JS_WRAPPER_QUERY_KEY = "moli-wpt-any"
ANY_JS_WINDOW_GLOBAL = "window"
ANY_JS_DEDICATED_WORKER_GLOBAL = "dedicatedworker"
ANY_JS_WORKER_SCRIPT_SUFFIX = ".any.worker.js"
ANY_JS_SOURCE_SCRIPT_SUFFIX = ".any.js"
SCRIPT_JS_WRAPPER_QUERY_KEY = "moli-wpt-script"
SCRIPT_JS_WINDOW_GLOBAL = "window"
SCRIPT_JS_DEDICATED_WORKER_GLOBAL = "dedicatedworker"
WINDOW_JS_SOURCE_SCRIPT_SUFFIX = ".window.js"
WORKER_JS_SOURCE_SCRIPT_SUFFIX = ".worker.js"
def is_any_js_case_path(case_path: str) -> bool:
return urlsplit(case_path.lstrip("/")).path.endswith(ANY_JS_SOURCE_SCRIPT_SUFFIX)
def is_any_js_worker_script_path(case_path: str) -> bool:
return urlsplit(case_path.lstrip("/")).path.endswith(ANY_JS_WORKER_SCRIPT_SUFFIX)
def any_js_wrapper_global(query: str) -> str | None:
for key, value in parse_qsl(query, keep_blank_values=True):
if key == ANY_JS_WRAPPER_QUERY_KEY:
return value or ANY_JS_WINDOW_GLOBAL
return None
def is_window_js_case_path(case_path: str) -> bool:
return urlsplit(case_path.lstrip("/")).path.endswith(WINDOW_JS_SOURCE_SCRIPT_SUFFIX)
def is_worker_js_case_path(case_path: str) -> bool:
return urlsplit(case_path.lstrip("/")).path.endswith(WORKER_JS_SOURCE_SCRIPT_SUFFIX)
def is_script_js_case_path(case_path: str) -> bool:
return is_window_js_case_path(case_path) or is_worker_js_case_path(case_path)
def script_js_wrapper_global(query: str) -> str | None:
for key, value in parse_qsl(query, keep_blank_values=True):
if key == SCRIPT_JS_WRAPPER_QUERY_KEY:
return value or SCRIPT_JS_WINDOW_GLOBAL
return None
def normalize_any_js_window_case_path(case_path: str) -> str:
"""Return an explicit case path that asks the fixture server for a wrapper.
WPT ``.any.js`` files are script resources, not directly navigable test
pages. The benchmark keeps default enumeration HTML-only, but an explicit
``--case path.any.js`` should run the window-global variant because that is
the common WPT translation and keeps wasm/jsapi coverage measurable.
"""
case_path = case_path.lstrip("/")
parts = urlsplit(case_path)
if (
not parts.path.endswith(".any.js")
or any_js_wrapper_global(parts.query) is not None
):
return case_path
return any_js_case_path_for_global(case_path, ANY_JS_WINDOW_GLOBAL)
def normalize_script_js_case_path(case_path: str) -> str:
case_path = case_path.lstrip("/")
parts = urlsplit(case_path)
if script_js_wrapper_global(parts.query) is not None:
return case_path
if parts.path.endswith(WINDOW_JS_SOURCE_SCRIPT_SUFFIX):
return script_js_case_path_for_global(case_path, SCRIPT_JS_WINDOW_GLOBAL)
if parts.path.endswith(WORKER_JS_SOURCE_SCRIPT_SUFFIX):
return script_js_case_path_for_global(case_path, SCRIPT_JS_DEDICATED_WORKER_GLOBAL)
return case_path
def any_js_case_path_for_global(case_path: str, global_name: str) -> str:
"""Return a wrapper case path for a specific ``.any.js`` global."""
case_path = case_path.lstrip("/")
parts = urlsplit(case_path)
query_items = [
(key, value)
for key, value in parse_qsl(parts.query, keep_blank_values=True)
if key != ANY_JS_WRAPPER_QUERY_KEY
]
query_items.append((ANY_JS_WRAPPER_QUERY_KEY, global_name))
return urlunsplit(("", "", parts.path, urlencode(query_items), parts.fragment))
def script_js_case_path_for_global(case_path: str, global_name: str) -> str:
case_path = case_path.lstrip("/")
parts = urlsplit(case_path)
query_items = [
(key, value)
for key, value in parse_qsl(parts.query, keep_blank_values=True)
if key != SCRIPT_JS_WRAPPER_QUERY_KEY
]
query_items.append((SCRIPT_JS_WRAPPER_QUERY_KEY, global_name))
return urlunsplit(("", "", parts.path, urlencode(query_items), parts.fragment))
def any_js_worker_script_path(case_path: str) -> str:
"""Return the WPT-style dedicated worker wrapper path for a ``.any.js`` case."""
parts = urlsplit(case_path.lstrip("/"))
if not parts.path.endswith(ANY_JS_SOURCE_SCRIPT_SUFFIX):
return case_path.lstrip("/")
worker_path = (
parts.path.removesuffix(ANY_JS_SOURCE_SCRIPT_SUFFIX)
+ ANY_JS_WORKER_SCRIPT_SUFFIX
)
return urlunsplit(("", "", worker_path, parts.query, parts.fragment))
def any_js_source_script_path(worker_script_path: str) -> str:
"""Return the source ``.any.js`` path for a dedicated worker wrapper path."""
parts = urlsplit(worker_script_path.lstrip("/"))
if not parts.path.endswith(ANY_JS_WORKER_SCRIPT_SUFFIX):
return worker_script_path.lstrip("/")
source_path = (
parts.path.removesuffix(ANY_JS_WORKER_SCRIPT_SUFFIX)
+ ANY_JS_SOURCE_SCRIPT_SUFFIX
)
return urlunsplit(("", "", source_path, parts.query, parts.fragment))
def query_without_any_js_wrapper(query: str) -> str:
query_items = [
(key, value)
for key, value in parse_qsl(query, keep_blank_values=True)
if key != ANY_JS_WRAPPER_QUERY_KEY
]
return urlencode(query_items)
def query_without_script_js_wrapper(query: str) -> str:
query_items = [
(key, value)
for key, value in parse_qsl(query, keep_blank_values=True)
if key != SCRIPT_JS_WRAPPER_QUERY_KEY
]
return urlencode(query_items)
moli-benchmark/moli_benchmark/wpt_cross/audit.py
+635
View File
@@ -0,0 +1,635 @@
"""Audit a wpt_cross matrix against an explicit known-failure manifest."""
from __future__ import annotations
import argparse
import json
import re
import sys
from collections import Counter
from pathlib import Path
from typing import Any
from urllib.parse import unquote, urlsplit
PASS_STATUS = "pass"
ALLOWED_EVIDENCE_KINDS = {"chromium", "doc", "local-test", "wpt"}
ALLOWED_EXPECTED_FAILURE_STATUSES = {
"crash",
"error",
"fail",
"harness-stalled",
"missing",
"timeout",
}
def _load_json(path: Path) -> Any:
return json.loads(path.read_text(encoding="utf-8"))
def _resolve_existing_manifest_path(manifest_path: Path, relative: str) -> Path | None:
path_without_anchor = _split_manifest_reference(relative)[0]
if not path_without_anchor:
return None
relative_path = Path(path_without_anchor)
if relative_path.is_absolute():
return None
for base in (manifest_path.parent, *manifest_path.parents):
candidate = base / relative_path
if candidate.is_file():
return candidate
return None
def _is_absolute_manifest_path(path: str) -> bool:
path_without_anchor = _split_manifest_reference(path)[0]
return bool(path_without_anchor) and Path(path_without_anchor).is_absolute()
def _case_base(case_path: str) -> str:
return case_path.split("?", 1)[0]
def _validate_manifest_case_path(path: Path, case_path: str) -> None:
parsed = urlsplit(case_path)
path_parts = parsed.path.split("/")
invalid = (
bool(parsed.scheme)
or bool(parsed.netloc)
or bool(parsed.fragment)
or not parsed.path
or parsed.path.startswith("/")
or "\\" in parsed.path
or any(part in {"", ".", ".."} for part in path_parts)
)
if invalid:
raise ValueError(
f"{path}: rule case_path {case_path!r} must be a relative WPT case path"
)
def _split_manifest_reference(reference: str) -> tuple[str, str | None]:
path, separator, fragment = reference.partition("#")
return path, unquote(fragment) if separator else None
def _wpt_case_path_from_manifest_reference(reference: str) -> str | None:
path_without_anchor = _split_manifest_reference(reference)[0]
parts = Path(path_without_anchor).parts
if not parts:
return None
if "wpt" in parts:
root_index = parts.index("wpt") + 1
if root_index >= len(parts):
return None
return Path(*parts[root_index:]).as_posix()
return None
def _markdown_slug(heading: str) -> str:
heading = re.sub(r"<[^>]*>", "", heading)
heading = heading.replace("`", "")
chars: list[str] = []
for char in heading.strip().lower():
if char.isalnum() or char in {" ", "-", "_"}:
chars.append(char)
slug = re.sub(r"\s+", "-", "".join(chars).strip())
return re.sub(r"-+", "-", slug).strip("-")
def _markdown_heading_anchors(path: Path) -> set[str]:
anchors: set[str] = set()
seen: Counter[str] = Counter()
text = path.read_text(encoding="utf-8", errors="replace")
for line in text.splitlines():
match = re.match(r"^\s{0,3}#{1,6}\s+(.+?)\s*#*\s*$", line)
if not match:
continue
heading = match.group(1).strip()
candidates: list[str] = []
numbered = re.match(r"^(\d+)(?:[.\u3001\uff0e]|$)", heading)
if numbered:
candidates.append(numbered.group(1))
slug = _markdown_slug(heading)
if slug:
candidates.append(slug)
for candidate in candidates:
suffix = seen[candidate]
anchors.add(candidate if suffix == 0 else f"{candidate}-{suffix}")
seen[candidate] += 1
return anchors
def _validate_markdown_reference_anchor(
manifest_path: Path,
reference: str,
resolved_path: Path,
context: str,
) -> None:
_, fragment = _split_manifest_reference(reference)
if fragment is None:
return
anchor = fragment.strip()
if not anchor:
raise ValueError(f"{manifest_path}: {context} has an empty document anchor")
anchors = _markdown_heading_anchors(resolved_path)
if anchor not in anchors:
raise ValueError(
f"{manifest_path}: {context} document anchor does not exist: "
f"{reference}"
)
def _manifest_categories(path: Path, payload: dict[str, Any]) -> dict[str, Any]:
explicit = payload.get("categories")
if explicit is None:
return {}
if not isinstance(explicit, dict):
raise ValueError(f"{path}: top-level 'categories' must be an object")
categories: dict[str, Any] = {}
for category, metadata in explicit.items():
if not isinstance(category, str) or not category:
raise ValueError(f"{path}: category names must be non-empty strings")
if not isinstance(metadata, dict):
raise ValueError(f"{path}: category {category!r} metadata must be an object")
tracking_doc = metadata.get("tracking_doc")
if not isinstance(tracking_doc, str) or not tracking_doc:
raise ValueError(
f"{path}: category {category!r} tracking_doc must be a string"
)
if _is_absolute_manifest_path(tracking_doc):
raise ValueError(
f"{path}: category {category!r} tracking_doc must be relative"
)
resolved_tracking_doc = _resolve_existing_manifest_path(path, tracking_doc)
if resolved_tracking_doc is None:
raise ValueError(
f"{path}: category {category!r} tracking_doc does not exist: "
f"{tracking_doc}"
)
_validate_markdown_reference_anchor(
path,
tracking_doc,
resolved_tracking_doc,
f"category {category!r} tracking_doc",
)
scope = metadata.get("scope")
if not isinstance(scope, str) or not scope.strip():
raise ValueError(f"{path}: category {category!r} scope must be a string")
evidence = metadata.get("evidence")
if not isinstance(evidence, list) or not evidence:
raise ValueError(
f"{path}: category {category!r} evidence must be a non-empty list"
)
for idx, item in enumerate(evidence):
if not isinstance(item, dict):
raise ValueError(
f"{path}: category {category!r} evidence #{idx + 1} must be an object"
)
evidence_path = item.get("path")
if not isinstance(evidence_path, str) or not evidence_path:
raise ValueError(
f"{path}: category {category!r} evidence #{idx + 1} path must be a string"
)
if _is_absolute_manifest_path(evidence_path):
raise ValueError(
f"{path}: category {category!r} evidence #{idx + 1} path must be relative"
)
resolved_evidence_path = _resolve_existing_manifest_path(path, evidence_path)
if resolved_evidence_path is None:
raise ValueError(
f"{path}: category {category!r} evidence #{idx + 1} path does not exist: "
f"{evidence_path}"
)
kind = item.get("kind")
if not isinstance(kind, str) or kind not in ALLOWED_EVIDENCE_KINDS:
raise ValueError(
f"{path}: category {category!r} evidence #{idx + 1} kind must be one of "
f"{sorted(ALLOWED_EVIDENCE_KINDS)!r}"
)
if kind == "doc":
_validate_markdown_reference_anchor(
path,
evidence_path,
resolved_evidence_path,
f"category {category!r} evidence #{idx + 1}",
)
note = item.get("note")
if not isinstance(note, str) or not note.strip():
raise ValueError(
f"{path}: category {category!r} evidence #{idx + 1} note must be a string"
)
categories[category] = metadata
return categories
def load_known_failure_manifest(path: Path) -> dict[str, Any]:
payload = _load_json(path)
rules = payload.get("rules") if isinstance(payload, dict) else None
if not isinstance(rules, list):
raise ValueError(f"{path} must contain a top-level 'rules' list")
categories = _manifest_categories(path, payload)
seen_case_paths: set[str] = set()
for idx, rule in enumerate(rules):
if not isinstance(rule, dict):
raise ValueError(f"{path}: rule #{idx + 1} must be an object")
case_path = rule.get("case_path")
if not isinstance(case_path, str) or not case_path:
raise ValueError(f"{path}: rule #{idx + 1} must have case_path")
_validate_manifest_case_path(path, case_path)
if case_path in seen_case_paths:
raise ValueError(f"{path}: duplicate known-failure rule for {case_path!r}")
seen_case_paths.add(case_path)
category = rule.get("category")
if categories:
if not isinstance(category, str) or not category:
raise ValueError(
f"{path}: rule for {case_path!r} must reference a category"
)
if category not in categories:
raise ValueError(
f"{path}: rule for {case_path!r} references unknown category "
f"{category!r}"
)
_validate_categorized_rule_evidence(path, rule)
_validate_rule_has_wpt_source_evidence(path, rule, categories[category])
_expected_statuses(rule)
_message_needles(rule)
if categories:
referenced_categories = {
rule["category"]
for rule in rules
if isinstance(rule.get("category"), str) and rule["category"]
}
unused_categories = sorted(set(categories) - referenced_categories)
if unused_categories:
raise ValueError(
f"{path}: categories are not referenced by any rule: "
f"{unused_categories!r}"
)
return payload
def load_known_failures(path: Path) -> list[dict[str, Any]]:
return load_known_failure_manifest(path)["rules"]
def _expected_statuses(rule: dict[str, Any]) -> set[str]:
explicit = rule.get("expected_statuses")
if explicit is None:
explicit = rule.get("expected_status")
if explicit is None:
return set()
if isinstance(explicit, str) and explicit.strip():
statuses = {explicit}
_validate_expected_failure_statuses(rule, statuses)
return statuses
if (
isinstance(explicit, list)
and explicit
and all(isinstance(item, str) and item.strip() for item in explicit)
):
statuses = set(explicit)
_validate_expected_failure_statuses(rule, statuses)
return statuses
raise ValueError(
f"rule for {rule.get('case_path', '<unknown>')} has invalid expected_status"
)
def _validate_expected_failure_statuses(
rule: dict[str, Any],
statuses: set[str],
) -> None:
invalid = statuses - ALLOWED_EXPECTED_FAILURE_STATUSES
if invalid:
raise ValueError(
f"rule for {rule.get('case_path', '<unknown>')} has invalid expected_status "
f"{sorted(invalid)!r}; expected one of "
f"{sorted(ALLOWED_EXPECTED_FAILURE_STATUSES)!r}"
)
def _validate_categorized_rule_evidence(path: Path, rule: dict[str, Any]) -> None:
case_path = rule.get("case_path", "<unknown>")
reason = rule.get("reason")
if not isinstance(reason, str) or not reason.strip():
raise ValueError(f"{path}: categorized rule for {case_path!r} must have reason")
if not _expected_statuses(rule):
raise ValueError(
f"{path}: categorized rule for {case_path!r} must have expected_status"
)
if not _message_needles(rule):
raise ValueError(
f"{path}: categorized rule for {case_path!r} must have message_contains"
)
def _validate_rule_has_wpt_source_evidence(
path: Path,
rule: dict[str, Any],
category_metadata: dict[str, Any],
) -> None:
case_path = str(rule.get("case_path", "<unknown>"))
case_base = _case_base(case_path)
evidence = category_metadata.get("evidence")
wpt_evidence_paths = {
normalized
for item in evidence
if isinstance(item, dict) and item.get("kind") == "wpt"
for normalized in [_wpt_case_path_from_manifest_reference(str(item.get("path", "")))]
if normalized is not None
}
if case_base not in wpt_evidence_paths:
raise ValueError(
f"{path}: categorized rule for {case_path!r} must have matching "
f"wpt evidence for {case_base!r}"
)
def _message_needles(rule: dict[str, Any]) -> list[str]:
explicit = rule.get("message_contains")
if explicit is None:
return []
if isinstance(explicit, str) and explicit.strip():
return [explicit]
if (
isinstance(explicit, list)
and all(isinstance(item, str) and item.strip() for item in explicit)
):
return explicit
raise ValueError(
f"rule for {rule.get('case_path', '<unknown>')} has invalid message_contains"
)
def _failure_text(result: dict[str, Any]) -> str:
parts: list[str] = []
error = result.get("error")
if error:
parts.append(str(error))
harness = result.get("harness_status_name")
if harness:
parts.append(str(harness))
harness_message = result.get("harness_message")
if harness_message:
parts.append(str(harness_message))
for failure in result.get("failures") or []:
if isinstance(failure, dict):
for key in ("name", "message", "status_name"):
value = failure.get(key)
if value:
parts.append(str(value))
elif failure:
parts.append(str(failure))
return "\n".join(parts)
def _audit_entry(
case_path: str,
result: dict[str, Any] | None,
rule: dict[str, Any] | None,
*,
note: str | None = None,
) -> dict[str, Any]:
entry: dict[str, Any] = {"case_path": case_path}
if result is not None:
entry["status"] = result.get("status", "missing")
for key in ("error", "harness_status_name", "harness_message"):
value = result.get(key)
if value:
entry[key] = value
failures = result.get("failures") or []
if failures:
entry["failures"] = failures
if rule is not None:
for key in (
"category",
"reason",
"expected_status",
"expected_statuses",
"message_contains",
):
if key in rule:
entry[key] = rule[key]
if note:
entry["note"] = note
return entry
def _category_counts(entries: list[dict[str, Any]]) -> dict[str, int]:
counter: Counter[str] = Counter()
for entry in entries:
category = entry.get("category")
counter[str(category) if category else "uncategorized"] += 1
return dict(sorted(counter.items()))
def _matrix_rows_by_case_path(matrix: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
rows: dict[str, dict[str, Any]] = {}
for idx, row in enumerate(matrix):
case_path = row.get("case_path")
if not isinstance(case_path, str) or not case_path:
raise ValueError(f"matrix row #{idx + 1} must have case_path")
if case_path in rows:
raise ValueError(f"matrix contains duplicate case_path {case_path!r}")
rows[case_path] = row
return rows
def audit_matrix(
matrix: list[dict[str, Any]],
engine: str,
rules: list[dict[str, Any]],
*,
categories: dict[str, Any] | None = None,
allow_missing_known_failures: bool = False,
) -> dict[str, Any]:
rows_by_case = _matrix_rows_by_case_path(matrix)
rules_by_case = {str(rule["case_path"]): rule for rule in rules}
known_failures: list[dict[str, Any]] = []
resolved_known_failures: list[dict[str, Any]] = []
mismatched_known_failures: list[dict[str, Any]] = []
missing_expected_failures: list[dict[str, Any]] = []
skipped_known_failures: list[dict[str, Any]] = []
unexpected_failures: list[dict[str, Any]] = []
for case_path, rule in rules_by_case.items():
row = rows_by_case.get(case_path)
result = None
if row is not None:
raw_result = (row.get("results") or {}).get(engine)
if isinstance(raw_result, dict):
result = raw_result
if result is None:
entry = _audit_entry(
case_path,
None,
rule,
note=f"matrix has no result for engine {engine}",
)
if allow_missing_known_failures:
skipped_known_failures.append(entry)
else:
missing_expected_failures.append(entry)
continue
status = str(result.get("status", "missing"))
if status == PASS_STATUS:
resolved_known_failures.append(_audit_entry(case_path, result, rule))
continue
expected_statuses = _expected_statuses(rule)
if expected_statuses and status not in expected_statuses:
mismatched_known_failures.append(
_audit_entry(
case_path,
result,
rule,
note=f"status {status!r} did not match {sorted(expected_statuses)!r}",
)
)
continue
text = _failure_text(result)
missing_needles = [needle for needle in _message_needles(rule) if needle not in text]
if missing_needles:
mismatched_known_failures.append(
_audit_entry(
case_path,
result,
rule,
note=f"missing expected failure text: {missing_needles!r}",
)
)
continue
known_failures.append(_audit_entry(case_path, result, rule))
for row in matrix:
case_path = str(row.get("case_path", ""))
if case_path in rules_by_case:
continue
result = (row.get("results") or {}).get(engine)
if not isinstance(result, dict):
continue
if str(result.get("status", "missing")) != PASS_STATUS:
unexpected_failures.append(_audit_entry(case_path, result, None))
counts = {
"known_failures": len(known_failures),
"resolved_known_failures": len(resolved_known_failures),
"mismatched_known_failures": len(mismatched_known_failures),
"missing_expected_failures": len(missing_expected_failures),
"skipped_known_failures": len(skipped_known_failures),
"unexpected_failures": len(unexpected_failures),
}
category_counts = {
"known_failures": _category_counts(known_failures),
"resolved_known_failures": _category_counts(resolved_known_failures),
"mismatched_known_failures": _category_counts(mismatched_known_failures),
"missing_expected_failures": _category_counts(missing_expected_failures),
"skipped_known_failures": _category_counts(skipped_known_failures),
"unexpected_failures": _category_counts(unexpected_failures),
}
return {
"engine": engine,
"allow_missing_known_failures": allow_missing_known_failures,
"categories": categories or {},
"counts": counts,
"category_counts": category_counts,
"known_failures": known_failures,
"resolved_known_failures": resolved_known_failures,
"mismatched_known_failures": mismatched_known_failures,
"missing_expected_failures": missing_expected_failures,
"skipped_known_failures": skipped_known_failures,
"unexpected_failures": unexpected_failures,
"ok": (
counts["mismatched_known_failures"] == 0
and counts["missing_expected_failures"] == 0
and counts["resolved_known_failures"] == 0
and counts["unexpected_failures"] == 0
),
}
def _build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog="python -m moli_benchmark.wpt_cross.audit",
description="Audit a wpt_cross matrix against an explicit known-failure manifest.",
)
parser.add_argument("--matrix", type=Path, required=True, help="Path to matrix.json")
parser.add_argument("--engine", required=True, help="Engine key to audit, e.g. moli")
parser.add_argument(
"--known-failures",
type=Path,
required=True,
help="JSON manifest containing known failure rules",
)
parser.add_argument("--output", type=Path, default=None, help="Optional JSON output path")
parser.add_argument(
"--allow-missing-known-failures",
action="store_true",
help=(
"Allow manifest rules that are absent from the matrix. Use only "
"for focused single-case or small-slice investigations; full "
"baseline audits should keep the default strict behavior."
),
)
return parser
def main(argv: list[str] | None = None) -> int:
parser = _build_parser()
args = parser.parse_args(argv)
try:
matrix = _load_json(args.matrix)
if not isinstance(matrix, list):
raise ValueError(f"{args.matrix} must contain a matrix row list")
manifest = load_known_failure_manifest(args.known_failures)
rules = manifest["rules"]
audit = audit_matrix(
matrix,
args.engine,
rules,
categories=manifest.get("categories"),
allow_missing_known_failures=args.allow_missing_known_failures,
)
except Exception as exc:
print(f"error: {exc}", file=sys.stderr)
return 2
if args.output:
args.output.write_text(json.dumps(audit, indent=2, sort_keys=True), encoding="utf-8")
counts = audit["counts"]
print(
"known={known_failures} resolved={resolved_known_failures} "
"mismatched={mismatched_known_failures} missing={missing_expected_failures} "
"skipped={skipped_known_failures} unexpected={unexpected_failures}".format(
**counts
)
)
for section in (
"unexpected_failures",
"mismatched_known_failures",
"missing_expected_failures",
"skipped_known_failures",
"resolved_known_failures",
):
entries = audit[section]
if entries:
print(f"{section}:")
for entry in entries:
print(f" - {entry['case_path']}: {entry.get('status', 'missing')}")
note = entry.get("note")
if note:
print(f" {note}")
return 0 if audit["ok"] else 1
if __name__ == "__main__":
sys.exit(main())
moli-benchmark/moli_benchmark/wpt_cross/build_partial.py
+80
View File
@@ -0,0 +1,80 @@
"""Synthesize a partial matrix.json + summary.json from per-engine JSON files.
Useful when the wpt_cross run is still in progress and you want a snapshot
report of the engines that already finished.
"""
from __future__ import annotations
import argparse
import json
import sys
from collections import Counter
from pathlib import Path
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser()
parser.add_argument("output_dir", type=Path, help="wpt_cross output directory")
parser.add_argument("--engine", action="append", required=True,
help="Engine name to include (must have engine-<NAME>.json present). Repeatable.")
args = parser.parse_args(argv)
out_dir: Path = args.output_dir
engine_data: dict[str, dict] = {}
for e in args.engine:
p = out_dir / f"engine-{e}.json"
if not p.exists():
print(f"error: {p} not found", file=sys.stderr)
return 2
engine_data[e] = json.loads(p.read_text(encoding="utf-8"))
# Union of case_paths across engines (preserves first engine's order then appends new ones).
seen: set[str] = set()
ordered_cases: list[str] = []
for e in args.engine:
for c in engine_data[e]["cases"]:
cp = c["case_path"]
if cp not in seen:
seen.add(cp)
ordered_cases.append(cp)
by_engine_case = {
e: {c["case_path"]: c for c in engine_data[e]["cases"]} for e in args.engine
}
matrix = []
for cp in ordered_cases:
row = {"case_path": cp, "results": {}}
for e in args.engine:
r = by_engine_case[e].get(cp)
if r is None:
row["results"][e] = {"status": "missing"}
else:
row["results"][e] = {
"status": r["status"],
"duration_ms": r["duration_ms"],
"subtests": r["subtests"],
"harness_status_name": r.get("harness_status_name"),
"harness_message": r.get("harness_message"),
"error": r.get("error"),
}
matrix.append(row)
summary = {"total": len(matrix), "engines": {}, "partial": True}
for e in args.engine:
c = Counter()
for row in matrix:
c[row["results"][e]["status"]] += 1
summary["engines"][e] = dict(c)
matrix_path = out_dir / f"matrix.partial.{'-'.join(args.engine)}.json"
summary_path = out_dir / f"summary.partial.{'-'.join(args.engine)}.json"
matrix_path.write_text(json.dumps(matrix, indent=2, sort_keys=True), encoding="utf-8")
summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8")
print(f"wrote {matrix_path}")
print(f"wrote {summary_path}")
return 0
if __name__ == "__main__":
sys.exit(main())
moli-benchmark/moli_benchmark/wpt_cross/case_set.py
+1054
View File
File diff suppressed because it is too large Load Diff
moli-benchmark/moli_benchmark/wpt_cross/cli_runner.py
+483
View File
@@ -0,0 +1,483 @@
"""Per-engine WPT case runner using each engine's ``fetch`` CLI plus the
in-page report bridge.
The fixture server's ``/resources/testharnessreport.js`` stores the final
testharness payload in a hidden DOM node and POSTs bounded payloads back to the
fixture server as a fallback. This runner reads the serialized DOM from a
successful CLI process first, then consults the fixture's
:class:`ResultsStore` when stdout has no final payload.
Compared to the CDP runner this is:
* Process-per-case — no cross-case state leakage, no relaunch logic.
* No CDP keepalive / awaitPromise / per-engine websocket quirks.
* The fairness cost is real but small: each engine uses its own native
fetch CLI, which is the most "natural" entry-point each project ships.
Engines whose CLI does not actually execute JavaScript (currently obscura)
expose ``cli_fetch_command = None`` and must use the CDP runner instead.
"""
from __future__ import annotations
import json
import multiprocessing
import os
import subprocess
import sys
import time
from concurrent.futures import Future, ProcessPoolExecutor, as_completed
from dataclasses import dataclass
from html.parser import HTMLParser
from pathlib import Path
from ..config import clear_proxy_env
from ..versions import sha256_file
from .engine import EngineDriver, build_driver
from .runner import CaseResult, EngineRunResult, classify_payload
from .runner import FINAL_PAYLOAD_SOURCES
from .server import DEFAULT_TESTHARNESS_TIMEOUT_SECONDS, WptFixtureServer
CaseRun = tuple[str, str] | tuple[str, str, float] | tuple[str, str, float, float]
MAX_RECORDED_STDERR_CHARS = 2000
BENCH_WPT_PAYLOAD_ELEMENT_ID = "__bench_wpt_payload"
WPT_TEMPLATE_HOSTS = (
"localhost",
"www.localhost",
"www1.localhost",
"www2.localhost",
"alt.localhost",
"www.alt.localhost",
)
MOLI_WPT_USER_AGENT = (
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"
)
@dataclass(frozen=True)
class _CliSubprocessResult:
duration_ms: float
proc_error: str | None
proc_returncode: int | None
proc_stderr: str
proc_stdout: bytes
wait_script_timeout: bool
@dataclass(frozen=True)
class _CliCaseWorkerInput:
engine: str
binary: str
wpt_root: str
case_path: str
external: bool
timeout_seconds: float
harness_timeout_multiplier: float
env: dict[str, str]
process_timeout_margin_seconds: float
payload_grace_seconds: float
successful_process_payload_grace_seconds: float
def _run_cli_subprocess(
argv: list[str],
env: dict[str, str],
proc_timeout: float,
) -> _CliSubprocessResult:
started = time.perf_counter()
proc_error: str | None = None
proc_returncode: int | None = None
proc_stderr = ""
proc_stdout = b""
wait_script_timeout = False
try:
cp = subprocess.run(
argv,
env=env,
capture_output=True,
timeout=proc_timeout,
check=False,
)
proc_returncode = cp.returncode
proc_stdout = cp.stdout or b""
proc_stderr = cp.stderr.decode("utf-8", errors="replace")
wait_script_timeout = "timed out waiting for script to become truthy" in proc_stderr
except subprocess.TimeoutExpired:
proc_error = f"engine subprocess wall timeout after {proc_timeout:.1f}s"
except OSError as error:
proc_error = f"engine subprocess failed: {error}"
return _CliSubprocessResult(
duration_ms=(time.perf_counter() - started) * 1000.0,
proc_error=proc_error,
proc_returncode=proc_returncode,
proc_stderr=proc_stderr,
proc_stdout=proc_stdout,
wait_script_timeout=wait_script_timeout,
)
def _bridge_key_for_case(case_path: str) -> str:
return case_path if case_path.startswith("/") else "/" + case_path
def _default_harness_timeout_multiplier(timeout_seconds: float) -> float:
return max(1.0, timeout_seconds / DEFAULT_TESTHARNESS_TIMEOUT_SECONDS)
class _BenchPayloadParser(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self._depth = 0
self.chunks: list[str] = []
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
attrs_dict = dict(attrs)
if attrs_dict.get("id") == BENCH_WPT_PAYLOAD_ELEMENT_ID:
self._depth += 1
elif self._depth:
self._depth += 1
def handle_endtag(self, tag: str) -> None:
if self._depth:
self._depth -= 1
def handle_data(self, data: str) -> None:
if self._depth:
self.chunks.append(data)
def _payload_from_stdout_html(stdout: bytes) -> dict | None:
if not stdout:
return None
text = stdout.decode("utf-8", errors="replace")
parser = _BenchPayloadParser()
try:
parser.feed(text)
except Exception:
return None
if not parser.chunks:
return None
raw = "".join(parser.chunks).strip()
if not raw:
return None
try:
payload = json.loads(raw)
except json.JSONDecodeError:
return None
return payload if isinstance(payload, dict) else None
def _stderr_tail(stderr: str) -> str:
stderr = stderr.strip()
if len(stderr) <= MAX_RECORDED_STDERR_CHARS:
return stderr
return stderr[-MAX_RECORDED_STDERR_CHARS:]
def _nonzero_exit_status(returncode: int, stderr: str) -> str:
if returncode < 0:
return "crash"
if "panicked at" in stderr or ("thread '" in stderr and " panicked at " in stderr):
return "crash"
return "error"
def _payload_grace_for_process_result(
*,
proc_error: str | None,
proc_returncode: int | None,
payload_grace_seconds: float,
successful_process_payload_grace_seconds: float,
) -> float:
if proc_error is None and proc_returncode == 0:
return successful_process_payload_grace_seconds
return payload_grace_seconds
def _curl_resolve_address(address: str) -> str:
if ":" in address and not (address.startswith("[") and address.endswith("]")):
return f"[{address}]"
return address
def _moli_fixture_host_resolve_args(fixture_server: WptFixtureServer) -> list[str]:
"""Return curl resolve overrides for WPT template hostnames.
Non-secure Moli CLI WPT cases normally run against the fixture
server's external IPv6 address so localhost is not treated as a trustworthy
origin. WPT templates still contain origin-distinguishing hostnames such as
alt.localhost and www.localhost; those names must stay in the URL for
browser-origin semantics, but libcurl needs to connect them back to the
external fixture listener.
"""
external_host = getattr(fixture_server, "external_host", None)
if not external_host:
return []
ports = [
getattr(fixture_server, "external_port", None),
getattr(fixture_server, "external_alternate_port", None),
getattr(fixture_server, "external_remote_port", None),
]
address = _curl_resolve_address(str(external_host))
args: list[str] = []
seen: set[tuple[str, int]] = set()
for port in ports:
if port is None:
continue
port = int(port)
for host in WPT_TEMPLATE_HOSTS:
key = (host, port)
if key in seen:
continue
seen.add(key)
args.extend(["--http-host-resolve", f"{host}:{port}:{address}"])
return args
def _classify_cli_case_result(
*,
case_path: str,
url: str,
bridge_key: str,
fixture_server: WptFixtureServer,
subprocess_result: _CliSubprocessResult,
payload_grace_seconds: float,
successful_process_payload_grace_seconds: float,
) -> CaseResult:
payload_grace = _payload_grace_for_process_result(
proc_error=subprocess_result.proc_error,
proc_returncode=subprocess_result.proc_returncode,
payload_grace_seconds=payload_grace_seconds,
successful_process_payload_grace_seconds=successful_process_payload_grace_seconds,
)
payload = _payload_from_stdout_html(subprocess_result.proc_stdout)
if payload is None:
payload = fixture_server.results.wait_for_final(bridge_key, timeout=payload_grace)
if payload is None:
payload = fixture_server.results.get(bridge_key)
bridge_installed = payload is not None
case_status_error = subprocess_result.proc_error
if isinstance(payload, dict) and payload.get("source") in FINAL_PAYLOAD_SOURCES:
case_status_error = None
if payload is None and subprocess_result.wait_script_timeout:
case_status_error = "testharness did not complete within timeout"
elif (
payload is None
and subprocess_result.proc_error is None
and subprocess_result.proc_returncode != 0
):
case_status_error = (
f"engine exited code={subprocess_result.proc_returncode} with no harness payload"
)
if subprocess_result.proc_stderr.strip():
case_status_error += f"; stderr tail: {_stderr_tail(subprocess_result.proc_stderr)}"
case_result = classify_payload(
payload=payload,
case_path=case_path,
url=url,
duration_ms=subprocess_result.duration_ms,
bridge_installed=bridge_installed,
error=case_status_error,
)
if (
payload is None
and subprocess_result.proc_returncode is not None
and subprocess_result.proc_returncode != 0
and subprocess_result.proc_error is None
and not subprocess_result.wait_script_timeout
):
case_result.status = _nonzero_exit_status(
subprocess_result.proc_returncode,
subprocess_result.proc_stderr,
)
return case_result
def _run_cli_case_worker(job: _CliCaseWorkerInput) -> CaseResult:
driver = build_driver(job.engine)
if driver.cli_fetch_command is None:
return CaseResult(
case_path=job.case_path,
url="",
status="error",
duration_ms=0.0,
error=f"engine `{driver.name}` has no cli_fetch_command; use the CDP runner",
)
binary = Path(job.binary)
with WptFixtureServer(Path(job.wpt_root)) as server:
server.set_harness_timeout_multipliers(
{job.case_path: job.harness_timeout_multiplier},
default_multiplier=job.harness_timeout_multiplier,
)
url = server.url_for_case(job.case_path, external=job.external)
bridge_key = _bridge_key_for_case(job.case_path)
server.results.clear(bridge_key)
argv = driver.cli_fetch_command(binary, url, job.timeout_seconds)
if driver.name == "moli":
argv.extend(["--user-agent", MOLI_WPT_USER_AGENT])
argv.extend(_moli_fixture_host_resolve_args(server))
subprocess_result = _run_cli_subprocess(
argv,
job.env,
job.timeout_seconds + job.process_timeout_margin_seconds,
)
return _classify_cli_case_result(
case_path=job.case_path,
url=url,
bridge_key=bridge_key,
fixture_server=server,
subprocess_result=subprocess_result,
payload_grace_seconds=job.payload_grace_seconds,
successful_process_payload_grace_seconds=job.successful_process_payload_grace_seconds,
)
def run_engine_on_cases_cli(
*,
driver: EngineDriver,
fixture_server: WptFixtureServer,
cases: list[CaseRun],
execution_cases: list[CaseRun] | None = None,
binary_override: str | None = None,
case_timeout_seconds: float = 8.0,
process_timeout_margin_seconds: float = 4.0,
payload_grace_seconds: float = 2.0,
successful_process_payload_grace_seconds: float = 8.0,
parallelism: int = 1,
progress_every: int = 25,
) -> EngineRunResult:
"""Run all ``cases`` through ``driver`` in CLI report-bridge mode.
``cases`` is ``[(case_path, url)]`` or
``[(case_path, url, timeout_seconds)]`` where ``case_path`` matches the
bridge payload's ``case_path`` (i.e. ``location.pathname`` = leading-slash
WPT-relative path).
``case_timeout_seconds`` is passed to the engine's CLI as its in-page wait
deadline; the runner adds ``process_timeout_margin_seconds`` on top for
the subprocess wall-clock. ``execution_cases`` may provide a different
submission order, while ``cases`` remains the canonical result order. Each
case runs entirely inside a worker process, including its own fixture server
and harness result collection. This keeps the parent process out of
per-case execution and avoids Python threads in the CLI runner.
"""
if driver.cli_fetch_command is None:
raise RuntimeError(
f"engine `{driver.name}` has no cli_fetch_command; use the CDP runner"
)
binary = driver.resolve_binary(binary_override)
binary_version = None
try:
completed = subprocess.run(
[str(binary), *driver.version_args],
capture_output=True, text=True, timeout=5, check=False,
)
out = (completed.stdout or completed.stderr).strip()
binary_version = out.splitlines()[0] if out else None
except (OSError, subprocess.SubprocessError):
pass
result = EngineRunResult(
engine=driver.name,
binary=str(binary),
binary_sha256=sha256_file(binary),
binary_version=binary_version,
endpoint=f"cli:{binary}",
ready_ms=None,
)
result.shutdown_info = {"mode": "cli", "cases_run": 0, "scheduler": "process-pool"}
env = clear_proxy_env(os.environ)
env.update(driver.extra_env)
# Belt-and-braces NO_PROXY for the local fixture endpoints created by each
# worker process.
no_proxy_extra = "127.0.0.1,localhost,::1"
if fixture_server.external_host:
no_proxy_extra += "," + fixture_server.external_host
env["NO_PROXY"] = no_proxy_extra
env["no_proxy"] = no_proxy_extra
def case_parts(case: CaseRun) -> tuple[str, bool, float, float]:
external_base_url = getattr(fixture_server, "external_base_url", None)
external = bool(external_base_url and case[1].startswith(external_base_url))
if len(case) == 4:
return case[0], external, case[2], case[3]
if len(case) == 3:
timeout_seconds = case[2]
return (
case[0],
external,
timeout_seconds,
_default_harness_timeout_multiplier(timeout_seconds),
)
return (
case[0],
external,
case_timeout_seconds,
_default_harness_timeout_multiplier(case_timeout_seconds),
)
results_by_path: dict[str, CaseResult] = {}
completed_count = 0
submission_cases = execution_cases if execution_cases is not None else cases
total = len(cases)
max_workers = max(1, parallelism)
# The parent may already have fixture-server threads; spawn avoids forking
# that multi-threaded state into workers.
process_context = multiprocessing.get_context("spawn")
with ProcessPoolExecutor(max_workers=max_workers, mp_context=process_context) as pool:
future_to_path: dict[Future[CaseResult], str] = {}
for case in submission_cases:
case_path, external, timeout_seconds, harness_timeout_multiplier = case_parts(case)
job = _CliCaseWorkerInput(
engine=driver.name,
binary=str(binary),
wpt_root=str(fixture_server.wpt_root),
case_path=case_path,
external=external,
timeout_seconds=timeout_seconds,
harness_timeout_multiplier=harness_timeout_multiplier,
env=env,
process_timeout_margin_seconds=process_timeout_margin_seconds,
payload_grace_seconds=payload_grace_seconds,
successful_process_payload_grace_seconds=successful_process_payload_grace_seconds,
)
future_to_path[pool.submit(_run_cli_case_worker, job)] = case_path
for fut in as_completed(future_to_path):
cp = future_to_path[fut]
try:
results_by_path[cp] = fut.result()
except Exception as exc: # pragma: no cover - defensive
results_by_path[cp] = CaseResult(
case_path=cp,
url="",
status="error",
duration_ms=0.0,
error=f"runner exception: {exc!r}",
)
completed_count += 1
if progress_every and completed_count % progress_every == 0:
print(
f"[wpt-cross] {driver.name}: {completed_count}/{total}",
file=sys.stderr,
flush=True,
)
# Preserve original case order in result.cases.
for case in cases:
case_path, _, _, _ = case_parts(case)
if case_path in results_by_path:
result.cases.append(results_by_path[case_path])
result.shutdown_info["cases_run"] = len(result.cases)
result.shutdown_info["parallelism"] = parallelism
return result
moli-benchmark/moli_benchmark/wpt_cross/coverage.py
+167
View File
@@ -0,0 +1,167 @@
"""Audit focused WPT-cross discovery coverage for wasm cases."""
from __future__ import annotations
import argparse
import json
from collections import Counter
from pathlib import Path
from typing import Any
from .audit import load_known_failure_manifest
from .case_set import enumerate_cases
WASM_CASE_SUFFIXES = (".html", ".any.js", ".window.js", ".worker.js")
def _is_wasm_source_case(path: Path) -> bool:
return path.name.endswith(WASM_CASE_SUFFIXES)
def _is_support_resource_case(rel: str) -> bool:
return "resources" in rel.split("/")[:-1]
def _case_base(case_path: str) -> str:
return case_path.split("?", 1)[0]
def _rule_category(rule: dict[str, Any]) -> str:
category = rule.get("category")
return category if isinstance(category, str) and category else "uncategorized"
def wasm_source_case_paths(wpt_root: Path) -> list[str]:
root = wpt_root.resolve()
wasm_root = root / "wasm"
if not wasm_root.exists():
raise RuntimeError(f"WPT wasm directory does not exist: {wasm_root}")
cases = [
path.relative_to(root).as_posix()
for path in sorted(wasm_root.rglob("*"))
if path.is_file() and _is_wasm_source_case(path)
]
return cases
def audit_wasm_focused_coverage(
wpt_root: Path,
*,
known_failures: Path | None = None,
) -> dict[str, Any]:
source_cases = wasm_source_case_paths(wpt_root)
enumerated_cases = enumerate_cases(
wpt_root,
dir_prefixes=("wasm",),
include_tentative=True,
any_js_global="both",
)
enumerated_case_paths = sorted(case.case_path for case in enumerated_cases)
duplicate_enumerated_case_paths = [
case_path
for case_path, count in sorted(Counter(enumerated_case_paths).items())
if count > 1
]
enumerated_case_set = set(enumerated_case_paths)
enumerated_base_paths = sorted(
{_case_base(case.case_path) for case in enumerated_cases}
)
enumerated_base_set = set(enumerated_base_paths)
ignored_support_resources = [
case for case in source_cases if _is_support_resource_case(case)
]
top_level_cases = [
case for case in source_cases if not _is_support_resource_case(case)
]
missing_top_level_cases = [
case for case in top_level_cases if case not in enumerated_base_set
]
enumerated_support_resource_paths = [
case for case in enumerated_base_paths if _is_support_resource_case(case)
]
extra_enumerated_base_paths = [
case for case in enumerated_base_paths if case not in source_cases
]
missing_known_failure_rules: list[str] = []
missing_known_failure_rule_details: list[dict[str, str]] = []
known_failure_rule_count = 0
known_failure_category_counts: dict[str, int] = {}
missing_known_failure_category_counts: dict[str, int] = {}
if known_failures is not None:
rules = load_known_failure_manifest(known_failures)["rules"]
known_failure_rule_count = len(rules)
known_failure_category_counts = dict(
sorted(Counter(_rule_category(rule) for rule in rules).items())
)
missing_rules = [
rule for rule in rules if rule["case_path"] not in enumerated_case_set
]
missing_known_failure_rules = [rule["case_path"] for rule in missing_rules]
missing_known_failure_rule_details = [
{
"case_path": rule["case_path"],
"category": _rule_category(rule),
}
for rule in missing_rules
]
missing_known_failure_category_counts = dict(
sorted(Counter(_rule_category(rule) for rule in missing_rules).items())
)
return {
"ok": (
not missing_top_level_cases
and not missing_known_failure_rules
and not duplicate_enumerated_case_paths
and not enumerated_support_resource_paths
and not extra_enumerated_base_paths
),
"wpt_root": str(wpt_root.resolve()),
"source_case_count": len(source_cases),
"top_level_case_count": len(top_level_cases),
"ignored_support_resource_count": len(ignored_support_resources),
"enumerated_case_count": len(enumerated_cases),
"enumerated_base_count": len(enumerated_base_paths),
"known_failure_rule_count": known_failure_rule_count,
"known_failure_category_counts": known_failure_category_counts,
"missing_known_failure_category_counts": missing_known_failure_category_counts,
"duplicate_enumerated_case_paths": duplicate_enumerated_case_paths,
"enumerated_support_resource_paths": enumerated_support_resource_paths,
"missing_top_level_cases": missing_top_level_cases,
"missing_known_failure_rules": missing_known_failure_rules,
"missing_known_failure_rule_details": missing_known_failure_rule_details,
"ignored_support_resources": ignored_support_resources,
"extra_enumerated_base_paths": extra_enumerated_base_paths,
}
def _build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog="python -m moli_benchmark.wpt_cross.coverage",
description="Audit focused WPT-cross discovery coverage for wasm cases.",
)
parser.add_argument("--wpt-root", required=True, type=Path)
parser.add_argument(
"--known-failures",
type=Path,
help="optional wasm known-failure manifest; every rule must be in the focused case set",
)
parser.add_argument("--json-output", type=Path)
return parser
def main(argv: list[str] | None = None) -> int:
args = _build_parser().parse_args(argv)
audit = audit_wasm_focused_coverage(
args.wpt_root,
known_failures=args.known_failures,
)
payload = json.dumps(audit, indent=2, sort_keys=True)
if args.json_output:
args.json_output.write_text(payload, encoding="utf-8")
print(payload)
return 0 if audit["ok"] else 1
if __name__ == "__main__":
raise SystemExit(main())
moli-benchmark/moli_benchmark/wpt_cross/engine.py
+325
View File
@@ -0,0 +1,325 @@
"""Engine drivers for cross-engine WPT runs.
An :class:`EngineDriver` knows how to launch a single headless browser engine
in CDP mode, expose its CDP HTTP endpoint (e.g. ``http://127.0.0.1:9222``),
and report enough metadata to be recorded in ``environment.json``.
The drivers are intentionally thin wrappers around the existing
``target_serve.start_target_serve`` machinery. They do **not** know anything
about WPT cases or the testharness — that lives in :mod:`runner`.
"""
from __future__ import annotations
import os
import shutil
import subprocess
import tempfile
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Callable
from ..config import REPO_ROOT, ReservedPort, clear_proxy_env, reserve_port
from ..sampling import ResourceSampler
from ..serve import probe_url
from ..versions import sha256_file
@dataclass
class EngineDriverHandle:
"""Live state of a launched engine driver."""
engine: str
process: subprocess.Popen[bytes]
endpoint: str
sampler: ResourceSampler
port_lease: ReservedPort
logs: list[str] = field(default_factory=list)
temp_dir: Path | None = None
ready_ms: float | None = None
binary: Path | None = None
binary_sha256: str | None = None
binary_version: str | None = None
def _binary_version(binary: Path, version_args: tuple[str, ...]) -> str | None:
try:
completed = subprocess.run(
[str(binary), *version_args],
capture_output=True,
text=True,
timeout=5,
check=False,
)
except (OSError, subprocess.TimeoutExpired):
return None
output = (completed.stdout or completed.stderr).strip()
return output.splitlines()[0] if output else None
def _drain_pipes(handle: EngineDriverHandle) -> None:
for stream, label in ((handle.process.stdout, "stdout"), (handle.process.stderr, "stderr")):
if stream is None:
continue
try:
while True:
line = stream.readline()
if not line:
break
handle.logs.append(f"{label}: {line.decode('utf-8', errors='replace').rstrip()}")
except OSError:
continue
def _terminate(process: subprocess.Popen[bytes]) -> None:
if process.poll() is not None:
return
try:
process.terminate()
process.wait(timeout=2)
return
except subprocess.TimeoutExpired:
pass
except OSError:
return
try:
process.kill()
process.wait(timeout=2)
except (OSError, subprocess.TimeoutExpired):
return
@dataclass
class EngineDriver:
"""Definition of how to launch a single engine.
``build_command`` returns the argv to launch the engine in CDP serve mode
on the given port. ``needs_profile_dir`` indicates whether a temp profile
directory should be provisioned before launch.
"""
name: str
binary_env_var: str
default_binary_names: tuple[str, ...]
build_command: Callable[[Path, int, Path | None], list[str]]
version_args: tuple[str, ...]
needs_profile_dir: bool = False
extra_env: dict[str, str] = field(default_factory=dict)
# Optional CLI-mode fetch invocation: (binary, url, timeout_seconds) -> argv.
# When set, the runner prefers HTTP-callback CLI mode and skips CDP launch
# entirely. Engines whose CLI does not execute JavaScript (e.g. obscura)
# leave this as None and fall back to CDP.
cli_fetch_command: Callable[[Path, str, float], list[str]] | None = None
def resolve_binary(self, override: str | None = None) -> Path:
raw = override or os.environ.get(self.binary_env_var)
if raw:
path = Path(raw).expanduser().resolve()
if not path.exists():
raise RuntimeError(f"{self.binary_env_var} does not exist: {path}")
return path
for name in self.default_binary_names:
located = shutil.which(name)
if located:
return Path(located).resolve()
raise RuntimeError(
f"missing {self.name} binary; set {self.binary_env_var} or install one of {self.default_binary_names}"
)
def launch(self, *, binary_override: str | None = None, ready_timeout_seconds: float = 30.0) -> EngineDriverHandle:
binary = self.resolve_binary(binary_override)
last_error: RuntimeError | None = None
for attempt in range(1, 4):
temp_dir: Path | None = None
if self.needs_profile_dir:
temp_dir = Path(tempfile.mkdtemp(prefix=f"moli-bench-wpt-{self.name}-"))
reserved = reserve_port()
try:
port = reserved.port
command = self.build_command(binary, port, temp_dir)
env = clear_proxy_env(os.environ)
env.update(self.extra_env)
reserved.release_socket()
process = subprocess.Popen(
command,
cwd=REPO_ROOT,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
start_new_session=True,
)
except BaseException:
reserved.close()
if temp_dir is not None:
shutil.rmtree(temp_dir, ignore_errors=True)
raise
endpoint = f"http://127.0.0.1:{port}"
sampler = ResourceSampler(process.pid)
sampler.start()
handle = EngineDriverHandle(
engine=self.name,
process=process,
endpoint=endpoint,
sampler=sampler,
port_lease=reserved,
temp_dir=temp_dir,
binary=binary,
binary_sha256=sha256_file(binary),
binary_version=_binary_version(binary, self.version_args),
)
started = time.perf_counter()
deadline = started + ready_timeout_seconds
version_url = endpoint + "/json/version"
while time.perf_counter() < deadline:
if process.poll() is not None:
_drain_pipes(handle)
log_tail = "; ".join(handle.logs[-20:])
self.shutdown(handle)
last_error = RuntimeError(
f"{self.name} CDP server exited early with code {process.returncode}"
f" on launch attempt {attempt}: {log_tail}"
)
if "Address already in use" in log_tail and attempt < 3:
time.sleep(0.05 * attempt)
break
raise last_error
if probe_url(version_url):
handle.ready_ms = (time.perf_counter() - started) * 1000.0
return handle
time.sleep(0.025)
else:
self.shutdown(handle)
last_error = RuntimeError(
f"timed out waiting for {self.name} CDP server at {version_url}: "
+ "; ".join(handle.logs[-20:])
)
raise last_error
# Early exit due to a retryable port race.
continue
if last_error is not None:
raise last_error
raise RuntimeError(f"failed to launch {self.name} CDP server")
def shutdown(self, handle: EngineDriverHandle | None) -> dict[str, Any]:
if handle is None:
return {}
try:
_terminate(handle.process)
_drain_pipes(handle)
resources = handle.sampler.stop()
finally:
handle.port_lease.close()
if handle.temp_dir is not None:
shutil.rmtree(handle.temp_dir, ignore_errors=True)
return {
"returncode": handle.process.returncode,
"ready_ms": handle.ready_ms,
"resources": resources,
"log_tail": handle.logs[-40:],
}
def _moli_command(binary: Path, port: int, _tmp: Path | None) -> list[str]:
return [
str(binary),
"serve",
"--layout",
"--resource",
"--host",
"127.0.0.1",
"--port",
str(port),
]
def _moli_fetch(binary: Path, url: str, timeout_seconds: float) -> list[str]:
timeout_ms = max(1000, int(timeout_seconds * 1000))
return [
str(binary), "fetch", "--layout", "--resource", url,
"--wait-until", "done",
"--wait-script",
"globalThis.__bench_wpt__ && globalThis.__bench_wpt__.source !== 'incremental'",
"--timeout", str(timeout_ms),
"--log-level", "error",
]
def _lightpanda_command(binary: Path, port: int, _tmp: Path | None) -> list[str]:
return [str(binary), "serve", "--host", "127.0.0.1", "--port", str(port)]
def _lightpanda_fetch(binary: Path, url: str, timeout_seconds: float) -> list[str]:
# lightpanda's --wait-until done often never fires (no DOMContentLoaded
# equivalent in the testharness completion path), so wait-ms acts as the
# real in-page deadline. Keep it aligned with the configured case timeout.
wait_ms = max(1000, int(timeout_seconds * 1000))
return [
str(binary), "fetch", url,
"--dump", "html",
"--wait-until", "done",
"--wait-ms", str(wait_ms),
"--http-timeout", str(wait_ms),
"--terminate-ms", str(wait_ms),
]
def _obscura_command(binary: Path, port: int, _tmp: Path | None) -> list[str]:
return [str(binary), "serve", "--port", str(port)]
def _chrome_command(binary: Path, port: int, tmp: Path | None) -> list[str]:
if tmp is None:
raise RuntimeError("chrome driver requires a profile directory")
return [
str(binary),
"--headless=new",
"--no-sandbox",
"--disable-gpu",
"--disable-dev-shm-usage",
"--no-first-run",
f"--user-data-dir={tmp}",
f"--remote-debugging-port={port}",
"about:blank",
]
ENGINES: dict[str, EngineDriver] = {
"moli": EngineDriver(
name="moli",
binary_env_var="MOLI_BIN",
default_binary_names=("moli",),
build_command=_moli_command,
cli_fetch_command=_moli_fetch,
version_args=("version",),
),
"lightpanda": EngineDriver(
name="lightpanda",
binary_env_var="LIGHTPANDA_BIN",
default_binary_names=("lightpanda",),
build_command=_lightpanda_command,
cli_fetch_command=_lightpanda_fetch,
version_args=("version",),
),
"obscura": EngineDriver(
name="obscura",
binary_env_var="OBSCURA_BIN",
default_binary_names=("obscura",),
build_command=_obscura_command,
version_args=("--help",),
),
"chrome": EngineDriver(
name="chrome",
binary_env_var="CHROME_BIN",
default_binary_names=("google-chrome", "google-chrome-stable", "chrome", "chromium", "chromium-browser"),
build_command=_chrome_command,
version_args=("--version",),
needs_profile_dir=True,
),
}
def build_driver(engine: str) -> EngineDriver:
if engine not in ENGINES:
raise RuntimeError(f"unknown engine `{engine}`; known: {sorted(ENGINES)}")
return ENGINES[engine]
moli-benchmark/moli_benchmark/wpt_cross/render_html.py
+1007
View File
File diff suppressed because it is too large Load Diff
moli-benchmark/moli_benchmark/wpt_cross/runner.py
+757
View File
@@ -0,0 +1,757 @@
"""Per-engine WPT case runner.
Connects to a launched :class:`EngineDriver`, navigates the engine to each
WPT case URL, and polls ``window.__bench_wpt__`` (installed by the fixture
server's testharnessreport.js bridge) for the testharness completion result.
Classifies each result as ``pass`` / ``fail`` / ``timeout`` / ``crash`` /
``error`` so that the cross-engine matrix can be built without the runner
making subjective judgments about WHY a case failed.
"""
from __future__ import annotations
import asyncio
import json
import time
from dataclasses import dataclass, field
from typing import Any
from ..raw_cdp import RawCdpClient, RawCdpError, connect_raw_cdp
from .engine import EngineDriver, EngineDriverHandle
CaseRun = tuple[str, str] | tuple[str, str, float]
# WPT testharness.js status constants (testharness.js: TestsStatus)
HARNESS_STATUS_OK = 0
HARNESS_STATUS_ERROR = 1
HARNESS_STATUS_TIMEOUT = 2
HARNESS_STATUS_PRECONDITION_FAILED = 3
HARNESS_STATUS_NAMES = {
HARNESS_STATUS_OK: "OK",
HARNESS_STATUS_ERROR: "ERROR",
HARNESS_STATUS_TIMEOUT: "TIMEOUT",
HARNESS_STATUS_PRECONDITION_FAILED: "PRECONDITION_FAILED",
}
# WPT testharness.js per-test status constants (testharness.js: Test.statuses)
TEST_STATUS_PASS = 0
TEST_STATUS_FAIL = 1
TEST_STATUS_TIMEOUT = 2
TEST_STATUS_NOTRUN = 3
TEST_STATUS_PRECONDITION_FAILED = 4
TEST_STATUS_NAMES = {
TEST_STATUS_PASS: "PASS",
TEST_STATUS_FAIL: "FAIL",
TEST_STATUS_TIMEOUT: "TIMEOUT",
TEST_STATUS_NOTRUN: "NOTRUN",
TEST_STATUS_PRECONDITION_FAILED: "PRECONDITION_FAILED",
}
MAX_RECORDED_FAILURES = 40
MAX_FAILURE_MESSAGE_CHARS = 500
FINAL_PAYLOAD_SOURCES = {"completion-callback", "done-hook", "done-hook-late"}
@dataclass
class CaseResult:
case_path: str
url: str
status: str # "pass" | "fail" | "timeout" | "crash" | "error"
duration_ms: float | None
harness_status: int | None = None
harness_message: str | None = None
subtests_total: int = 0
subtests_pass: int = 0
subtests_fail: int = 0
subtests_timeout: int = 0
subtests_notrun: int = 0
error: str | None = None
console_errors: int = 0
js_exceptions: int = 0
payload_source: str | None = None
failures: list[dict[str, Any]] = field(default_factory=list)
failure_names: list[str] = field(default_factory=list)
@dataclass
class EngineRunResult:
engine: str
binary: str | None
binary_sha256: str | None
binary_version: str | None
endpoint: str
ready_ms: float | None
cases: list[CaseResult] = field(default_factory=list)
shutdown_info: dict[str, Any] = field(default_factory=dict)
setup_error: str | None = None
_TRACE_METHODS = {"Runtime.consoleAPICalled", "Runtime.exceptionThrown", "Log.entryAdded"}
def _count_traces(messages: list[dict[str, Any]]) -> tuple[int, int]:
console_errors = 0
js_exceptions = 0
for message in messages:
method = message.get("method")
if method == "Runtime.exceptionThrown":
js_exceptions += 1
elif method == "Runtime.consoleAPICalled":
params = message.get("params") or {}
if params.get("type") in {"error", "assert"}:
console_errors += 1
elif method == "Log.entryAdded":
entry = (message.get("params") or {}).get("entry") or {}
if entry.get("level") == "error":
console_errors += 1
return console_errors, js_exceptions
def _recorded_failures(tests: Any) -> list[dict[str, Any]]:
if not isinstance(tests, list):
return []
failures: list[dict[str, Any]] = []
for entry in tests:
if not isinstance(entry, dict):
continue
status = entry.get("status")
if status == TEST_STATUS_PASS:
continue
failure: dict[str, Any] = {
"status": status,
"status_name": TEST_STATUS_NAMES.get(status) if isinstance(status, int) else None,
}
name = entry.get("name")
if isinstance(name, str):
failure["name"] = name
message = entry.get("message")
if isinstance(message, str) and message:
failure["message"] = message[:MAX_FAILURE_MESSAGE_CHARS]
if len(message) > MAX_FAILURE_MESSAGE_CHARS:
failure["message_truncated"] = True
failures.append(failure)
if len(failures) >= MAX_RECORDED_FAILURES:
break
return failures
def _failure_name(entry: dict[str, Any], index: int) -> str:
name = entry.get("name")
if isinstance(name, str) and name:
return name
status = entry.get("status_name") or entry.get("status") or "failure"
return f"<unnamed {status} #{index + 1}>"
def _failure_names(tests: Any) -> list[str]:
if not isinstance(tests, list):
return []
names: list[str] = []
for index, entry in enumerate(tests):
if not isinstance(entry, dict):
names.append(f"<malformed failure #{index + 1}>")
continue
status = entry.get("status")
if status == TEST_STATUS_PASS:
continue
names.append(_failure_name(entry, index))
return names
async def _attach_page(client: RawCdpClient) -> str:
"""Create a fresh BrowserContext + Target and return its sessionId.
Falls back to the default target if Target.createBrowserContext is not
supported. Enables Runtime/Page/Log so we can collect harness results
and console errors.
"""
browser_context_id: str | None = None
try:
ctx_id = await client.send("Target.createBrowserContext")
ctx_resp, _ = await client.recv_until_id(ctx_id, timeout=5)
value = (ctx_resp.get("result") or {}).get("browserContextId")
if isinstance(value, str) and value:
browser_context_id = value
except RawCdpError:
browser_context_id = None
except asyncio.TimeoutError:
browser_context_id = None
target_params: dict[str, Any] = {"url": "about:blank"}
if browser_context_id:
target_params["browserContextId"] = browser_context_id
target_id = await client.send("Target.createTarget", target_params)
target_resp, _ = await client.recv_until_id(target_id, timeout=10)
target = (target_resp.get("result") or {}).get("targetId")
if not isinstance(target, str) or not target:
raise RuntimeError(f"missing targetId in createTarget response: {target_resp}")
attach_id = await client.send("Target.attachToTarget", {"targetId": target, "flatten": True})
attach_resp, _ = await client.recv_until_id(attach_id, timeout=5)
session_id = (attach_resp.get("result") or {}).get("sessionId")
if not isinstance(session_id, str) or not session_id:
raise RuntimeError(f"missing sessionId in attachToTarget response: {attach_resp}")
for method in ("Runtime.enable", "Page.enable"):
cmd_id = await client.send(method, session_id=session_id)
await client.recv_until_id(cmd_id, timeout=5)
for method in ("Log.enable",):
try:
cmd_id = await client.send(method, session_id=session_id)
await client.recv_until_id(cmd_id, timeout=3)
except (RawCdpError, asyncio.TimeoutError):
pass
return session_id
_HARNESS_PROBE_EXPRESSION = """
(function() {
if (typeof window === 'undefined') return null;
if (typeof window.__bench_wpt__ === 'undefined') return null;
return window.__bench_wpt__;
})()
"""
_BRIDGE_INSTALLED_EXPRESSION = """
(function() {
var t = (typeof window !== 'undefined') ? window.__bench_wpt_trace__ : null;
if (!Array.isArray(t)) return false;
for (var i = 0; i < t.length; i++) {
if (t[i] && t[i].installing === true) return true;
}
return false;
})()
"""
async def _bridge_installed(client: RawCdpClient, session_id: str) -> bool:
try:
eval_id = await client.send(
"Runtime.evaluate",
{"expression": _BRIDGE_INSTALLED_EXPRESSION, "returnByValue": True},
session_id=session_id,
)
response, _ = await client.recv_until_id(eval_id, timeout=5)
except (RawCdpError, asyncio.TimeoutError):
return False
return bool(((response.get("result") or {}).get("result") or {}).get("value"))
async def _run_one_case(
*,
client: RawCdpClient,
session_id: str,
case_path: str,
url: str,
timeout_seconds: float,
) -> CaseResult:
started = time.perf_counter()
seen_messages: list[dict[str, Any]] = []
try:
nav_id = await client.send("Page.navigate", {"url": url}, session_id=session_id)
_, nav_seen = await client.recv_until_id(nav_id, timeout=timeout_seconds)
seen_messages.extend(nav_seen)
except (RawCdpError, asyncio.TimeoutError) as error:
return CaseResult(
case_path=case_path,
url=url,
status="error",
duration_ms=(time.perf_counter() - started) * 1000.0,
error=f"navigate failed: {error}",
)
deadline = time.perf_counter() + timeout_seconds
payload: Any = None
while time.perf_counter() < deadline:
try:
eval_id = await client.send(
"Runtime.evaluate",
{
"expression": _HARNESS_PROBE_EXPRESSION,
"returnByValue": True,
"awaitPromise": False,
},
session_id=session_id,
)
response, eval_seen = await client.recv_until_id(eval_id, timeout=5)
seen_messages.extend(eval_seen)
except (RawCdpError, asyncio.TimeoutError) as error:
return CaseResult(
case_path=case_path,
url=url,
status="error",
duration_ms=(time.perf_counter() - started) * 1000.0,
error=f"evaluate failed: {error}",
)
result = ((response.get("result") or {}).get("result") or {})
value = result.get("value")
if isinstance(value, dict):
source = value.get("source") if isinstance(value.get("source"), str) else None
payload = value
if source in FINAL_PAYLOAD_SOURCES:
break
await asyncio.sleep(0.05)
duration_ms = (time.perf_counter() - started) * 1000.0
console_errors, js_exceptions = _count_traces(seen_messages)
# Distinguish "bridge never installed" (engine couldn't load
# testharness.js at all) from "bridge installed but testharness never
# produced results" (engine-side completion bug). A non-final payload also
# proves the bridge was installed, but it is not enough to pass the case.
bridge_installed = payload is not None or await _bridge_installed(client, session_id)
return classify_payload(
payload=payload if isinstance(payload, dict) else None,
case_path=case_path,
url=url,
duration_ms=duration_ms,
bridge_installed=bridge_installed,
console_errors=console_errors,
js_exceptions=js_exceptions,
)
def classify_payload(
*,
payload: dict | None,
case_path: str,
url: str,
duration_ms: float | None,
bridge_installed: bool,
console_errors: int = 0,
js_exceptions: int = 0,
error: str | None = None,
) -> CaseResult:
"""Map a bridge payload (or its absence) to a CaseResult.
Used by both the CDP runner and the CLI HTTP-callback runner so the
pass/fail/timeout/harness-stalled classification stays uniform.
"""
if payload is None or not isinstance(payload, dict):
return CaseResult(
case_path=case_path,
url=url,
status="harness-stalled" if bridge_installed else "timeout",
duration_ms=duration_ms,
console_errors=console_errors,
js_exceptions=js_exceptions,
error=error or (
"bridge installed but testharness produced no result/completion callbacks"
if bridge_installed
else "testharness did not complete within timeout"
),
)
payload_source = payload.get("source")
harness = payload.get("harness")
tests = payload.get("tests")
harness_status = harness.get("status") if isinstance(harness, dict) else None
harness_message = harness.get("message") if isinstance(harness, dict) else None
counts = {"pass": 0, "fail": 0, "timeout": 0, "notrun": 0, "other": 0}
if isinstance(tests, list):
for entry in tests:
if not isinstance(entry, dict):
counts["other"] += 1
continue
status = entry.get("status")
if status == TEST_STATUS_PASS:
counts["pass"] += 1
elif status == TEST_STATUS_FAIL or status == TEST_STATUS_PRECONDITION_FAILED:
counts["fail"] += 1
elif status == TEST_STATUS_TIMEOUT:
counts["timeout"] += 1
elif status == TEST_STATUS_NOTRUN:
counts["notrun"] += 1
else:
counts["other"] += 1
total = sum(counts.values())
overall = "pass"
has_observed_failure = bool(counts["fail"] or counts["timeout"])
if harness_status == HARNESS_STATUS_TIMEOUT:
overall = "timeout"
has_observed_failure = True
elif harness_status == HARNESS_STATUS_ERROR:
overall = "fail"
has_observed_failure = True
elif harness_status == HARNESS_STATUS_PRECONDITION_FAILED:
overall = "fail"
has_observed_failure = True
elif counts["fail"] or counts["timeout"]:
overall = "fail"
elif total == 0:
overall = "fail"
if payload_source not in FINAL_PAYLOAD_SOURCES and not has_observed_failure:
overall = "harness-stalled" if bridge_installed else "timeout"
if error is None:
source_label = (
payload_source if isinstance(payload_source, str) else "non-final"
)
error = (
f"testharness produced only {source_label} payload "
"without final completion"
)
elif total == 0 and error is None:
error = "testharness completed without reporting any subtests"
if isinstance(harness_message, str) and harness_message:
error = f"{error}: {harness_message}"
return CaseResult(
case_path=case_path,
url=url,
status=overall,
duration_ms=duration_ms,
harness_status=harness_status if isinstance(harness_status, int) else None,
harness_message=harness_message if isinstance(harness_message, str) else None,
subtests_total=total,
subtests_pass=counts["pass"],
subtests_fail=counts["fail"],
subtests_timeout=counts["timeout"],
subtests_notrun=counts["notrun"],
console_errors=console_errors,
js_exceptions=js_exceptions,
payload_source=payload_source if isinstance(payload_source, str) else None,
error=error,
failures=_recorded_failures(tests),
failure_names=_failure_names(tests),
)
async def _run_async(
*,
driver: EngineDriver,
binary_override: str | None,
cases: list[CaseRun],
case_timeout_seconds: float,
launch_timeout_seconds: float,
) -> EngineRunResult:
handle: EngineDriverHandle | None = None
result = EngineRunResult(
engine=driver.name,
binary=None,
binary_sha256=None,
binary_version=None,
endpoint="",
ready_ms=None,
)
try:
handle = driver.launch(binary_override=binary_override, ready_timeout_seconds=launch_timeout_seconds)
except Exception as error:
result.setup_error = f"launch failed: {error}"
return result
result.binary = str(handle.binary) if handle.binary else None
result.binary_sha256 = handle.binary_sha256
result.binary_version = handle.binary_version
result.endpoint = handle.endpoint
result.ready_ms = handle.ready_ms
try:
client = await connect_raw_cdp(handle.endpoint)
except Exception as error:
result.setup_error = f"cdp connect failed: {error}"
result.shutdown_info = driver.shutdown(handle)
return result
try:
session_id = await _attach_page(client)
except Exception as error:
result.setup_error = f"attach failed: {error}"
try:
await client.websocket.close()
except Exception:
pass
result.shutdown_info = driver.shutdown(handle)
return result
relaunch_count = 0
max_relaunches = 10
consecutive_relaunch_failures = 0
try:
for case_index, case in enumerate(cases):
case_path, url, timeout_seconds = _case_parts(case, case_timeout_seconds)
# Engine died between cases?
if handle.process.poll() is not None:
exit_code = handle.process.returncode
result.cases.append(
CaseResult(
case_path=case_path,
url=url,
status="crash",
duration_ms=None,
error=f"engine process exited with code {exit_code} (pre-case)",
)
)
relaunched = await _try_relaunch(
driver=driver,
binary_override=binary_override,
launch_timeout_seconds=launch_timeout_seconds,
)
if relaunched is None:
consecutive_relaunch_failures += 1
if consecutive_relaunch_failures >= 3:
for remaining in cases[case_index + 1:]:
rp, ru, _ = _case_parts(remaining, case_timeout_seconds)
result.cases.append(
CaseResult(
case_path=rp, url=ru, status="crash",
duration_ms=None,
error="engine relaunch failed 3x; aborting",
)
)
break
continue
consecutive_relaunch_failures = 0
relaunch_count += 1
# swap in new handle/client/session
try:
await client.websocket.close()
except Exception:
pass
driver.shutdown(handle) # ensure old handle fully reaped
handle, client, session_id = relaunched
continue
try:
case_result = await _run_one_case(
client=client,
session_id=session_id,
case_path=case_path,
url=url,
timeout_seconds=timeout_seconds,
)
except Exception as error:
# CDP / websocket / asyncio explosion mid-case.
proc_alive = handle.process.poll() is None
case_result = CaseResult(
case_path=case_path,
url=url,
status="crash" if not proc_alive else "error",
duration_ms=None,
error=f"runner exception: {type(error).__name__}: {error}",
)
result.cases.append(case_result)
if relaunch_count >= max_relaunches:
for remaining in cases[case_index + 1:]:
rp, ru, _ = _case_parts(remaining, case_timeout_seconds)
result.cases.append(
CaseResult(
case_path=rp, url=ru, status="crash",
duration_ms=None,
error=f"exceeded max relaunches ({max_relaunches})",
)
)
break
# Tear down current connection + engine, relaunch fresh.
try:
await client.websocket.close()
except Exception:
pass
driver.shutdown(handle)
relaunched = await _try_relaunch(
driver=driver,
binary_override=binary_override,
launch_timeout_seconds=launch_timeout_seconds,
)
if relaunched is None:
consecutive_relaunch_failures += 1
if consecutive_relaunch_failures >= 3:
for remaining in cases[case_index + 1:]:
rp, ru, _ = _case_parts(remaining, case_timeout_seconds)
result.cases.append(
CaseResult(
case_path=rp, url=ru, status="crash",
duration_ms=None,
error="engine relaunch failed 3x after crash; aborting",
)
)
break
# client/handle are dead; fabricate placeholders that will fail
# the poll() check next iteration -> retry relaunch path.
handle, client, session_id = await _wait_then_retry_launch(
driver,
binary_override,
launch_timeout_seconds,
result,
cases,
case_index,
case_timeout_seconds,
)
if handle is None:
break
else:
consecutive_relaunch_failures = 0
relaunch_count += 1
if relaunched is not None:
handle, client, session_id = relaunched
continue
consecutive_relaunch_failures = 0
result.cases.append(case_result)
finally:
try:
await client.websocket.close()
except Exception:
pass
try:
result.shutdown_info = driver.shutdown(handle)
except Exception:
pass
return result
async def _try_relaunch(
*,
driver: EngineDriver,
binary_override: str | None,
launch_timeout_seconds: float,
) -> tuple[EngineDriverHandle, RawCdpClient, str] | None:
"""Relaunch engine + reconnect CDP + reattach. Returns None on failure."""
try:
new_handle = driver.launch(
binary_override=binary_override,
ready_timeout_seconds=launch_timeout_seconds,
)
except Exception:
return None
try:
new_client = await connect_raw_cdp(new_handle.endpoint)
except Exception:
try:
driver.shutdown(new_handle)
except Exception:
pass
return None
try:
new_session_id = await _attach_page(new_client)
except Exception:
try:
await new_client.websocket.close()
except Exception:
pass
try:
driver.shutdown(new_handle)
except Exception:
pass
return None
return new_handle, new_client, new_session_id
async def _wait_then_retry_launch(
driver: EngineDriver,
binary_override: str | None,
launch_timeout_seconds: float,
result: EngineRunResult,
cases: list[CaseRun],
case_index: int,
case_timeout_seconds: float,
) -> tuple[EngineDriverHandle | None, RawCdpClient | None, str | None]:
await asyncio.sleep(1.0)
relaunched = await _try_relaunch(
driver=driver,
binary_override=binary_override,
launch_timeout_seconds=launch_timeout_seconds,
)
if relaunched is None:
for remaining in cases[case_index + 1:]:
rp, ru, _ = _case_parts(remaining, case_timeout_seconds)
result.cases.append(
CaseResult(
case_path=rp, url=ru, status="crash",
duration_ms=None,
error="engine relaunch failed after backoff; aborting",
)
)
return None, None, None
return relaunched
def _case_parts(case: CaseRun, default_timeout: float) -> tuple[str, str, float]:
if len(case) == 3:
return case[0], case[1], case[2]
return case[0], case[1], default_timeout
def run_engine_on_cases(
*,
driver: EngineDriver,
cases: list[CaseRun],
binary_override: str | None = None,
case_timeout_seconds: float = 30.0,
launch_timeout_seconds: float = 30.0,
) -> EngineRunResult:
"""Synchronous wrapper around the async runner.
``cases`` is a list of ``(case_path, url)`` or
``(case_path, url, timeout_seconds)`` tuples where ``case_path`` is the
WPT-relative path (used as case identity) and ``url`` is what the engine
actually navigates to (loopback or external IPv6).
"""
return asyncio.run(
_run_async(
driver=driver,
binary_override=binary_override,
cases=cases,
case_timeout_seconds=case_timeout_seconds,
launch_timeout_seconds=launch_timeout_seconds,
)
)
def case_result_to_dict(case: CaseResult) -> dict[str, Any]:
return {
"case_path": case.case_path,
"url": case.url,
"status": case.status,
"duration_ms": case.duration_ms,
"harness_status": case.harness_status,
"harness_status_name": HARNESS_STATUS_NAMES.get(case.harness_status, None) if case.harness_status is not None else None,
"harness_message": case.harness_message,
"subtests": {
"total": case.subtests_total,
"pass": case.subtests_pass,
"fail": case.subtests_fail,
"timeout": case.subtests_timeout,
"notrun": case.subtests_notrun,
},
"console_errors": case.console_errors,
"js_exceptions": case.js_exceptions,
"payload_source": case.payload_source,
"error": case.error,
"failures": case.failures,
"failure_names": case.failure_names,
}
def engine_result_to_dict(result: EngineRunResult) -> dict[str, Any]:
return {
"engine": result.engine,
"binary": result.binary,
"binary_sha256": result.binary_sha256,
"binary_version": result.binary_version,
"endpoint": result.endpoint,
"ready_ms": result.ready_ms,
"setup_error": result.setup_error,
"shutdown": result.shutdown_info,
"cases": [case_result_to_dict(c) for c in result.cases],
}
def write_engine_result(path, result: EngineRunResult) -> None:
"""Write ``result`` to ``path`` as JSON. Convenience helper for CLI."""
from pathlib import Path as _Path
out = _Path(path)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(engine_result_to_dict(result), indent=2, sort_keys=True), encoding="utf-8")
moli-benchmark/moli_benchmark/wpt_cross/scheduler.py
+123
View File
@@ -0,0 +1,123 @@
"""Execution-order scheduling for wpt_cross case runs."""
from __future__ import annotations
import hashlib
import json
from collections.abc import Callable, Sequence
from pathlib import Path
from typing import Any, TypeVar
T = TypeVar("T")
FIXED_RUN_SHUFFLE_SEED = "moli-wpt-cross-run-order-v1"
BUCKET_DEPTH = 1
def build_run_schedule(
cases: Sequence[T],
*,
case_path: Callable[[T], str],
) -> tuple[list[T], dict[str, Any]]:
"""Return execution-ordered cases plus metadata.
The input order remains the canonical result order. The returned list is a
fixed pseudo-random order used only to submit work to the runner.
"""
canonical_paths = [case_path(case) for case in cases]
scheduled_cases, bucket_count = _prefix_balanced_shuffle(cases, case_path)
scheduled_paths = [case_path(case) for case in scheduled_cases]
metadata: dict[str, Any] = {
"mode": "fixed-prefix-balanced-shuffle",
"seed": FIXED_RUN_SHUFFLE_SEED,
"bucket_depth": BUCKET_DEPTH,
"bucket_count": bucket_count,
"case_count": len(canonical_paths),
"canonical_cases_sha256": _sha256_lines(canonical_paths),
"scheduled_cases_sha256": _sha256_lines(scheduled_paths),
"output_order": "canonical-case-path",
"execution_order": "fixed-prefix-balanced-shuffle",
}
return scheduled_cases, metadata
def write_run_schedule(
output_dir: Path,
*,
metadata: dict[str, Any],
scheduled_case_paths: Sequence[str],
) -> None:
"""Write reproducibility artifacts for the execution schedule."""
(output_dir / "schedule.txt").write_text(
"\n".join(scheduled_case_paths) + "\n",
encoding="utf-8",
)
(output_dir / "schedule.json").write_text(
json.dumps(metadata, indent=2, sort_keys=True),
encoding="utf-8",
)
def _shuffle_key(case_path: str) -> str:
payload = f"{FIXED_RUN_SHUFFLE_SEED}\0{case_path}".encode("utf-8")
return hashlib.sha256(payload).hexdigest()
def _prefix_balanced_shuffle(
cases: Sequence[T],
case_path: Callable[[T], str],
) -> tuple[list[T], int]:
groups: dict[str, list[T]] = {}
for case in cases:
groups.setdefault(_case_bucket(case_path(case)), []).append(case)
states: list[dict[str, Any]] = []
for bucket in sorted(groups):
group = groups[bucket]
group.sort(key=lambda case: (_shuffle_key(case_path(case)), case_path(case)))
group.reverse()
states.append(
{
"bucket": bucket,
"items": group,
"weight": len(group),
"current": 0,
"tie": _shuffle_key(bucket),
}
)
scheduled: list[T] = []
total_weight = sum(state["weight"] for state in states)
while states:
for state in states:
state["current"] += state["weight"]
best_index = max(
range(len(states)),
key=lambda index: (states[index]["current"], states[index]["tie"]),
)
best = states[best_index]
scheduled.append(best["items"].pop())
best["current"] -= total_weight
if not best["items"]:
total_weight -= best["weight"]
states.pop(best_index)
return scheduled, len(groups)
def _case_bucket(case_path: str) -> str:
path_only = case_path.split("?", 1)[0].split("#", 1)[0]
parts = [part for part in path_only.split("/") if part]
if len(parts) >= BUCKET_DEPTH:
return "/".join(parts[:BUCKET_DEPTH])
if parts:
return parts[0]
return "<root>"
def _sha256_lines(paths: Sequence[str]) -> str:
payload = "".join(f"{path}\n" for path in paths).encode("utf-8")
return hashlib.sha256(payload).hexdigest()
moli-benchmark/moli_benchmark/wpt_cross/server.py
+2749
View File
File diff suppressed because it is too large Load Diff
moli-benchmark/pyproject.toml
+15
View File
@@ -0,0 +1,15 @@
[project]
name = "moli-benchmark"
version = "0.1.0"
license = "MIT OR Apache-2.0"
requires-python = ">=3.11"
dependencies = [
"websockets>=13,<16",
]
[project.scripts]
moli-benchmark = "moli_benchmark.cli:main"
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
moli-benchmark/scripts/heaptrack-moli-wrapper.sh
Executable
+10
View File
@@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -euo pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
repo_root="$(cd "$script_dir/../.." && pwd)"
real_binary="${MOLI_REAL_BINARY:-$repo_root/target/release/moli}"
heaptrack_output="${HEAPTRACK_OUT:-/tmp/heaptrack-moli.%p.gz}"
exec heaptrack --output "$heaptrack_output" -- "$real_binary" "$@"
moli-benchmark/scripts/massif-moli-wrapper.sh
Executable
+18
View File
@@ -0,0 +1,18 @@
#!/usr/bin/env bash
set -euo pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
repo_root="$(cd "$script_dir/../.." && pwd)"
real_binary="${MOLI_REAL_BINARY:-$repo_root/target/release/moli}"
massif_output="${MASSIF_OUT:-/tmp/massif-moli.%p.out}"
exec valgrind \
--tool=massif \
--pages-as-heap=yes \
--time-unit=B \
--threshold=0.2 \
--detailed-freq=1 \
--max-snapshots=200 \
--massif-out-file="$massif_output" \
"$real_binary" "$@"
moli-benchmark/scripts/probe-cdp-memory-ablation.py
+2210
View File
File diff suppressed because it is too large Load Diff
moli-benchmark/scripts/run-webfetch-mix-benchmark.sh
Executable
+218
View File
@@ -0,0 +1,218 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BENCH_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
REPO_ROOT="$(cd "${BENCH_DIR}/.." && pwd)"
BUILD="${BUILD:-1}"
SOURCE="${SOURCE:-webfetch-mix}"
PROFILE="${PROFILE:-webfetch}"
RUNS="${RUNS:-1}"
TIMEOUT="${TIMEOUT:-30}"
PARALLELISM="${PARALLELISM:-20}"
CHROME_PARALLELISM="${CHROME_PARALLELISM:-2}"
LIMIT="${LIMIT:-}"
TARGETS="${TARGETS:-moli moli-cdp moli-full moli-full-cdp}"
OUTPUT_DIR="${OUTPUT_DIR:-}"
MOLI_BIN="${MOLI_BIN:-${REPO_ROOT}/target/release/moli}"
LIGHTPANDA_BIN="${LIGHTPANDA_BIN:-}"
OBSCURA_BIN="${OBSCURA_BIN:-}"
CHROME_BIN="${CHROME_BIN:-}"
detect_command() {
local name="$1"
if command -v "${name}" >/dev/null 2>&1; then
command -v "${name}"
fi
}
if [[ -z "${LIGHTPANDA_BIN}" ]]; then
LIGHTPANDA_BIN="$(detect_command lightpanda || true)"
fi
if [[ -z "${OBSCURA_BIN}" ]]; then
OBSCURA_BIN="$(detect_command obscura || true)"
fi
if [[ -z "${CHROME_BIN}" ]]; then
if [[ -x "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" ]]; then
CHROME_BIN="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
elif command -v google-chrome >/dev/null 2>&1; then
CHROME_BIN="$(command -v google-chrome)"
elif command -v chromium >/dev/null 2>&1; then
CHROME_BIN="$(command -v chromium)"
elif command -v chromium-browser >/dev/null 2>&1; then
CHROME_BIN="$(command -v chromium-browser)"
fi
fi
if [[ -z "${OUTPUT_DIR}" ]]; then
OUTPUT_DIR="${BENCH_DIR}/results/webfetch-mix-benchmark-$(date +%Y%m%d-%H%M%S)"
elif [[ "${OUTPUT_DIR}" != /* ]]; then
OUTPUT_DIR="${REPO_ROOT}/${OUTPUT_DIR}"
fi
if [[ "${BUILD}" == "1" ]]; then
(cd "${REPO_ROOT}" && cargo build --release -p moli)
fi
if [[ ! -x "${MOLI_BIN}" ]]; then
echo "missing executable MOLI_BIN=${MOLI_BIN}" >&2
exit 2
fi
cmd=(
uv run moli-benchmark top-sites
--source "${SOURCE}"
--profile "${PROFILE}"
--runs "${RUNS}"
--timeout "${TIMEOUT}"
--parallelism "${PARALLELISM}"
--chrome-parallelism "${CHROME_PARALLELISM}"
--output-dir "${OUTPUT_DIR}"
--moli-bin "${MOLI_BIN}"
)
if [[ -n "${CHROME_BIN}" ]]; then
cmd+=(--chrome-bin "${CHROME_BIN}")
fi
if [[ -n "${LIGHTPANDA_BIN}" ]]; then
cmd+=(--lightpanda-bin "${LIGHTPANDA_BIN}")
fi
if [[ -n "${OBSCURA_BIN}" ]]; then
cmd+=(--obscura-bin "${OBSCURA_BIN}")
fi
if [[ -n "${LIMIT}" ]]; then
cmd+=(--limit "${LIMIT}")
fi
for target in ${TARGETS}; do
cmd+=(--target "${target}")
done
echo "repo: ${REPO_ROOT}"
echo "benchmark: ${BENCH_DIR}"
echo "output: ${OUTPUT_DIR}"
echo "targets: ${TARGETS}"
echo "source=${SOURCE} profile=${PROFILE} runs=${RUNS} timeout=${TIMEOUT}s parallelism=${PARALLELISM} chrome_parallelism=${CHROME_PARALLELISM}"
echo "moli=${MOLI_BIN}"
echo "lightpanda=${LIGHTPANDA_BIN:-auto/default}"
echo "chrome=${CHROME_BIN:-auto/default}"
echo "obscura=${OBSCURA_BIN:-auto/default}"
echo
printf '+'
printf ' %q' "${cmd[@]}"
printf '\n\n'
set +e
run_output="$(cd "${BENCH_DIR}" && "${cmd[@]}" 2>&1)"
status=$?
set -e
echo "${run_output}"
if [[ ! -f "${OUTPUT_DIR}/top-sites/summary.json" ]]; then
echo "benchmark exited ${status}; summary not found at ${OUTPUT_DIR}/top-sites/summary.json" >&2
exit "${status}"
fi
python3 - "${OUTPUT_DIR}" <<'PY'
import collections
import json
import pathlib
import sys
result_dir = pathlib.Path(sys.argv[1])
summary = json.loads((result_dir / "top-sites" / "summary.json").read_text())
runs = json.loads((result_dir / "top-sites" / "runs.json").read_text())
def fmt_ms(value):
return "-" if value is None else f"{value / 1000:.2f}s"
def fmt_bytes(value):
return "-" if value is None else f"{value / 1024 / 1024:.0f}MiB"
lines = []
lines.append("# WebFetch Mix Benchmark Summary")
lines.append("")
lines.append(f"Result dir: `{result_dir}`")
lines.append("")
lines.append(
"Config: "
f"source={summary['source']}, profile={summary['profile']}, runs={summary['runs']}, "
f"timeout={summary['timeout_seconds']}s, parallelism={summary['parallelism']}, "
f"chrome_parallelism={summary['chrome_parallelism']}, counted={summary['counted_site_count']}, "
f"excluded={summary['excluded_site_count']}"
)
lines.append("")
lines.append(
"| target | pass | fail | timed_out flag | failure timeout | median | p95 | "
"PSS median/p95 | RSS median/p95 | failure kinds |"
)
lines.append("|---|---:|---:|---:|---:|---:|---:|---:|---:|---|")
for name, row in summary["targets"].items():
sites = row["sites"]
passes = row["passes"]
rate = passes / sites * 100 if sites else 0
elapsed = row.get("elapsed_ms") or {}
pss = row.get("peak_pss_bytes") or {}
rss = row.get("peak_rss_bytes") or {}
timed_out = sum(
1
for run in runs
if run.get("target") == name and not run.get("excluded") and run.get("timed_out")
)
failure_timeout = (row.get("failure_kinds") or {}).get("timeout", 0)
failure_kinds = ", ".join(
f"{key}:{value}" for key, value in sorted((row.get("failure_kinds") or {}).items())
)
lines.append(
f"| {name} | {passes}/{sites} ({rate:.1f}%) | {row['failures']} | {timed_out} | "
f"{failure_timeout} | {fmt_ms(elapsed.get('median'))} | {fmt_ms(elapsed.get('p95'))} | "
f"{fmt_bytes(pss.get('median'))}/{fmt_bytes(pss.get('p95'))} | "
f"{fmt_bytes(rss.get('median'))}/{fmt_bytes(rss.get('p95'))} | {failure_kinds} |"
)
lines.append("")
lines.append("## Overlaps")
by_site = collections.defaultdict(dict)
for run in runs:
if run.get("excluded"):
continue
by_site[run["domain"]][run["target"]] = run
for left, right in (
("moli", "moli-full"),
("moli-cdp", "moli-full-cdp"),
("moli", "moli-cdp"),
("moli-full", "moli-full-cdp"),
("lightpanda", "lightpanda-cdp"),
("obscura", "obscura-cdp"),
):
paired = [site_runs for site_runs in by_site.values() if left in site_runs and right in site_runs]
if not paired:
continue
both = sum(bool(site_runs[left].get("ok")) and bool(site_runs[right].get("ok")) for site_runs in paired)
left_only = sum(bool(site_runs[left].get("ok")) and not bool(site_runs[right].get("ok")) for site_runs in paired)
right_only = sum((not bool(site_runs[left].get("ok"))) and bool(site_runs[right].get("ok")) for site_runs in paired)
neither = sum(
(not bool(site_runs[left].get("ok"))) and (not bool(site_runs[right].get("ok")))
for site_runs in paired
)
lines.append(
f"- `{left}` vs `{right}`: both={both}, `{left}` only={left_only}, "
f"`{right}` only={right_only}, neither={neither}"
)
text = "\n".join(lines) + "\n"
(result_dir / "run-summary.md").write_text(text, encoding="utf-8")
print(text)
print(f"summary written: {result_dir / 'run-summary.md'}")
PY
exit "${status}"
moli-benchmark/tests/test_agent_episode.py
+270
View File
@@ -0,0 +1,270 @@
from __future__ import annotations
import http.client
import json
import tempfile
import unittest
import urllib.error
import urllib.request
from pathlib import Path
from unittest import mock
from moli_benchmark.agent_episode import (
AgentEpisodeError,
AgentEpisodeFixtureServer,
DEFAULT_MANIFEST_PATH,
_aggregate_resource_samples,
_known_error_counts,
_navigate,
_step_assertion_count,
load_agent_episode_manifest,
run_agent_episode_suite,
)
from moli_benchmark.agent_episode_fixture import response_for_agent_path
from moli_benchmark.agent_episode_report import write_agent_episode_report
class AgentEpisodeManifestTests(unittest.TestCase):
def test_checked_in_manifest_has_required_contract_cases(self) -> None:
manifest = load_agent_episode_manifest()
episode_ids = {str(episode["id"]) for episode in manifest.episodes}
self.assertEqual(manifest.fixture_version, "agent-episode-fixture-v1")
self.assertEqual(len(manifest.sha256), 64)
self.assertTrue(
{
"observe-static",
"fill-reactive-form",
"click-same-document",
"click-cross-document",
"failed-navigation",
"dynamic-controls",
"idle-resume",
"episode-isolation",
}.issubset(episode_ids)
)
def test_manifest_rejects_duplicate_episode_ids(self) -> None:
payload = json.loads(DEFAULT_MANIFEST_PATH.read_text(encoding="utf-8"))
payload["episodes"].append(payload["episodes"][0])
with tempfile.TemporaryDirectory() as temporary:
path = Path(temporary) / "manifest.json"
path.write_text(json.dumps(payload), encoding="utf-8")
with self.assertRaisesRegex(AgentEpisodeError, "duplicate episode id"):
load_agent_episode_manifest(path)
def test_manifest_rejects_invalid_observation_contract(self) -> None:
payload = json.loads(DEFAULT_MANIFEST_PATH.read_text(encoding="utf-8"))
payload["episodes"][0]["steps"][0]["expect"]["text_contains"] = "not-an-array"
with tempfile.TemporaryDirectory() as temporary:
path = Path(temporary) / "manifest.json"
path.write_text(json.dumps(payload), encoding="utf-8")
with self.assertRaisesRegex(AgentEpisodeError, "text_contains"):
load_agent_episode_manifest(path)
class AgentEpisodeFixtureTests(unittest.TestCase):
def test_fixture_serves_reactive_page_and_records_request(self) -> None:
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
with AgentEpisodeFixtureServer() as fixture:
with opener.open(fixture.url("/agent/fill-reactive-form"), timeout=2) as response:
body = response.read().decode("utf-8")
requests = fixture.requests
self.assertIn("Reactive form ready", body)
self.assertIn("queueMicrotask", body)
self.assertEqual(requests[-1]["path"], "/agent/fill-reactive-form")
def test_isolation_fixture_escapes_inline_script_end_tags(self) -> None:
token = "</ScRiPt><script>globalThis.injected=true</script>"
response = response_for_agent_path("/agent/isolation", {"token": [token]})
self.assertIsNotNone(response)
body = response.decode("utf-8") if response is not None else ""
self.assertNotIn("</ScRiPt><script>", body)
self.assertIn(
r'globalThis.__agentIsolationToken = "<\/ScRiPt><script>globalThis.injected=true<\/script>";',
body,
)
def test_failure_fixture_resets_connection_before_response(self) -> None:
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
with AgentEpisodeFixtureServer() as fixture:
with self.assertRaises(
(ConnectionResetError, http.client.RemoteDisconnected, urllib.error.URLError)
):
opener.open(fixture.url("/agent/reset-before-response"), timeout=2).read()
class AgentEpisodeNavigationTests(unittest.IsolatedAsyncioTestCase):
async def test_document_events_share_timeout_budget(self) -> None:
class Clock:
def __init__(self) -> None:
self._times = iter((100.0, 100.2, 100.9))
def time(self) -> float:
return next(self._times)
cases = (
(None, {}),
(
"net::ERR_CONNECTION_RESET",
{
"expect_error_text_contains": "ERR_CONNECTION_RESET",
"expect_error_document": True,
},
),
)
for error_text, expectation in cases:
with self.subTest(error_text=error_text):
client = mock.Mock(current_sequence=7)
client.command = mock.AsyncMock(
return_value=mock.Mock(
response={"result": {"errorText": error_text}}
)
)
client.wait_for_event = mock.AsyncMock()
with mock.patch(
"moli_benchmark.agent_episode.asyncio.get_running_loop",
return_value=Clock(),
):
await _navigate(
client, # type: ignore[arg-type]
"session-1",
"http://fixture.test/next",
1.0,
**expectation,
)
waits = client.wait_for_event.await_args_list
self.assertEqual(
[call.args[0] for call in waits],
["Page.frameNavigated", "Page.loadEventFired"],
)
self.assertAlmostEqual(waits[0].kwargs["timeout"], 0.8)
self.assertAlmostEqual(waits[1].kwargs["timeout"], 0.1)
class AgentEpisodeAggregationTests(unittest.TestCase):
def test_resource_aggregation_requires_all_workers_for_summary(self) -> None:
samples = _aggregate_resource_samples(
{
"worker-1": [
{"timestamp": 10.0, "rss_bytes": 10, "pss_bytes": 8, "cpu_percent": 20},
{"timestamp": 11.0, "rss_bytes": 12, "pss_bytes": 9, "cpu_percent": 25},
],
"worker-2": [
{"timestamp": 10.5, "rss_bytes": 20, "pss_bytes": 15, "cpu_percent": 30},
],
}
)
self.assertEqual(samples[0]["observed_worker_count"], 1)
self.assertEqual(samples[1]["rss_bytes"], 30)
self.assertEqual(samples[1]["cpu_percent"], 50)
def test_known_errors_distinguish_exact_protocol_messages_and_timeouts(self) -> None:
counts = _known_error_counts(
[
{"status": "protocol_error", "cdp_error_message": "Promise was collected"},
{"status": "timeout_page_alive", "cdp_error_message": None},
],
[
{
"status": "protocol_error",
"failure_step": 0,
"cdp_error_message": "NoDocumentLoaded",
}
],
)
self.assertEqual(counts["Promise was collected"], 1)
self.assertEqual(counts["NoDocumentLoaded"], 1)
self.assertEqual(counts["command timeout"], 1)
def test_known_errors_do_not_double_count_step_failure_episode_rollup(self) -> None:
counts = _known_error_counts(
[{"status": "timeout_page_alive", "cdp_error_message": None}],
[
{
"status": "timeout_page_alive",
"failure_step": 1,
"cdp_error_message": None,
}
],
)
self.assertEqual(counts["command timeout"], 1)
def test_assertion_count_comes_from_manifest_contract(self) -> None:
self.assertEqual(
_step_assertion_count(
{
"operation": "observe",
"expect": {
"url_path": "/agent/example",
"text_contains": ["one", "two"],
"text_not_contains": ["three"],
"min_controls": 1,
},
}
),
5,
)
def test_unavailable_target_still_writes_authoritative_report(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
output_dir = Path(temporary)
summary = run_agent_episode_suite(
output_dir=output_dir,
target_matrix={"moli": {"available": False, "path": None}},
targets=("moli-cdp",),
runs=1,
workers=1,
parallelism=1,
step_dwell_ms=0,
sample_interval_ms=500,
timeout_seconds=1,
)
report = json.loads(
(output_dir / "agent-episode" / "report-data.json").read_text(
encoding="utf-8"
)
)
self.assertEqual(summary["total_failures"], len(summary["cases"]))
self.assertEqual(report["schema"], "moli.agent-episode.report.v1")
self.assertTrue(all(row["status"] == "target_unavailable" for row in report["episodes"]))
def test_report_is_self_contained_and_json_is_authoritative(self) -> None:
summary = {
"episodes_total": 1,
"episodes_passed": 1,
"total_failures": 0,
"steps_total": 1,
"workers": 1,
"parallelism": 1,
"step_dwell_ms": 14_000,
"targets": {},
}
with tempfile.TemporaryDirectory() as temporary:
suite_dir = Path(temporary)
payload = write_agent_episode_report(
suite_dir=suite_dir,
summary=summary,
episode_rows=[],
step_rows=[],
resources={},
markers=[],
config={"fixture_version": "agent-episode-fixture-v1"},
)
serialized = json.loads((suite_dir / "report-data.json").read_text())
html = (suite_dir / "index.html").read_text(encoding="utf-8")
self.assertEqual(serialized, payload)
self.assertIn('id="report-data"', html)
self.assertNotIn("cdn.jsdelivr.net", html)
if __name__ == "__main__":
unittest.main()
moli-benchmark/tests/test_amiibo_crawler.py
+309
View File
@@ -0,0 +1,309 @@
from __future__ import annotations
import asyncio
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from moli_benchmark.amiibo_crawler import (
AMIIBO_CONCURRENCY_MATRIX,
AMIIBO_MODES,
PageSession,
_classify_amiibo_error,
_collect_page_assertion_failures,
_crawl_with_endpoint,
_run_process_mode,
_row_failure_kind,
_summarize_rows,
_summarize_serve_resources,
run_amiibo_crawler_suite,
)
class AmiiboCrawlerTests(unittest.TestCase):
class _FakeWebSocket:
def __init__(self) -> None:
self.close_count = 0
async def close(self) -> None:
self.close_count += 1
class _FakeClient:
def __init__(self, websocket: "AmiiboCrawlerTests._FakeWebSocket") -> None:
self.websocket = websocket
def test_crawl_returns_when_all_page_sessions_fail(self) -> None:
async def fail_create_session(endpoint: str, timeout_seconds: float) -> object:
raise RuntimeError("session failed")
async def run() -> dict[str, object]:
with patch("moli_benchmark.amiibo_crawler._create_page_session", fail_create_session):
return await asyncio.wait_for(
_crawl_with_endpoint(
endpoint="http://127.0.0.1:1",
pool=2,
limit=1,
url="https://demo-browser.lightpanda.io/amiibo/",
timeout_seconds=0.1,
),
timeout=1.0,
)
result = asyncio.run(run())
self.assertEqual(result["known_count"], 1)
self.assertEqual(result["expected_pages"], 1)
errors = result["errors"]
self.assertIsInstance(errors, list)
self.assertEqual(len(errors), 2)
self.assertEqual({error["stage"] for error in errors}, {"create-session"})
def test_crawl_closes_created_sessions_when_creation_is_cancelled(self) -> None:
websocket = self._FakeWebSocket()
calls = 0
async def create_session(endpoint: str, timeout_seconds: float) -> PageSession:
nonlocal calls
calls += 1
if calls == 1:
return PageSession(client=self._FakeClient(websocket), session_id="first")
await asyncio.sleep(10)
raise AssertionError("unreachable")
async def run() -> None:
with patch("moli_benchmark.amiibo_crawler._create_page_session", create_session):
await asyncio.wait_for(
_crawl_with_endpoint(
endpoint="http://127.0.0.1:1",
pool=2,
limit=1,
url="https://demo-browser.lightpanda.io/amiibo/",
timeout_seconds=30.0,
),
timeout=0.01,
)
with self.assertRaises(TimeoutError):
asyncio.run(run())
self.assertGreaterEqual(websocket.close_count, 1)
def test_process_mode_closes_created_sessions_when_creation_times_out(self) -> None:
websocket = self._FakeWebSocket()
calls = 0
class Serve:
def __init__(self, worker_id: int) -> None:
self.endpoint = f"http://127.0.0.1:{worker_id}"
def start_serve(target: str, binary: Path, timeout_seconds: float) -> Serve:
return Serve(calls + 1)
def stop_serve(serve: Serve) -> dict[str, object]:
return {"endpoint": serve.endpoint, "resources": {"peak_pss_bytes": 0, "peak_cpu_percent": 0.0}}
async def create_session(endpoint: str, timeout_seconds: float) -> PageSession:
nonlocal calls
calls += 1
if calls == 1:
return PageSession(client=self._FakeClient(websocket), session_id="first")
await asyncio.sleep(10)
raise AssertionError("unreachable")
with (
patch("moli_benchmark.amiibo_crawler.start_target_serve", start_serve),
patch("moli_benchmark.amiibo_crawler.stop_target_serve", stop_serve),
patch("moli_benchmark.amiibo_crawler._create_page_session", create_session),
):
row, detail = _run_process_mode(
target="moli",
binary=Path("/tmp/moli"),
pool=2,
limit=1,
url="https://demo-browser.lightpanda.io/amiibo/",
timeout_seconds=0.01,
)
self.assertFalse(row["ok"])
self.assertEqual(row["failure_kind"], "timeout")
self.assertEqual(len(detail["serve"]["workers"]), 2)
self.assertGreaterEqual(websocket.close_count, 1)
def test_page_assertions_require_extracted_fields(self) -> None:
failures = _collect_page_assertion_failures(
[
{
"url": "https://demo-browser.lightpanda.io/amiibo/",
"ready_state": "complete",
"title": "Sandy",
"text_length": 222,
"link_count": 12,
"fields": {
"name": "Sandy",
"game": "Animal Crossing",
"serie": "Animal Crossing",
"imageSrc": "https://example.test/sandy.png",
"altCount": 10,
},
},
{"url": "", "title": "", "text_length": 0, "link_count": -1, "worker": 3},
{
"url": "https://demo-browser.lightpanda.io/amiibo/stale.html",
"ready_state": "complete",
"title": "Amiibo Character",
"text_length": 80,
"link_count": 0,
"worker": 4,
"fields": {
"name": "Amiibo Character",
"game": "Amiibo Game",
"serie": "Amiibo Serie",
"imageSrc": "",
"altCount": 0,
},
},
]
)
self.assertEqual(
failures,
[
{
"url": "",
"worker": 3,
"failures": ["missing-url", "missing-title", "document-not-complete", "missing-body-text", "missing-link-count", "missing-fields"],
},
{
"url": "https://demo-browser.lightpanda.io/amiibo/stale.html",
"worker": 4,
"failures": [
"missing-amiibo-name",
"missing-amiibo-series",
"missing-game-series",
"missing-image-src",
],
},
],
)
def test_failure_classification_prefers_specific_causes(self) -> None:
self.assertEqual(_classify_amiibo_error("crawler run exceeded 30.0s wall timeout"), "timeout")
self.assertEqual(_classify_amiibo_error("CDP command id=1 failed"), "protocol-error")
self.assertEqual(_classify_amiibo_error("Page.navigate failed"), "navigation-error")
self.assertEqual(_classify_amiibo_error("Runtime.evaluate returned an unexpected result"), "script-error")
self.assertEqual(
_row_failure_kind(
ok=False,
pages=49,
expected_pages=50,
crawler_errors=0,
assertion_failures=0,
),
"page-count-mismatch",
)
self.assertEqual(
_row_failure_kind(
ok=False,
pages=50,
expected_pages=50,
crawler_errors=0,
assertion_failures=1,
),
"assertion-failure",
)
self.assertEqual(
_row_failure_kind(
ok=False,
pages=None,
expected_pages=50,
crawler_errors=0,
assertion_failures=0,
),
"error",
)
def test_process_resource_summary_aggregates_worker_processes(self) -> None:
summary = _summarize_serve_resources(
[
{"resources": {"peak_pss_bytes": 10, "peak_cpu_percent": 1.5, "peak_process_count": 1}},
{"resources": {"peak_pss_bytes": 20, "peak_cpu_percent": 2.5, "peak_process_count": 1}},
]
)
self.assertEqual(summary["peak_pss_bytes"], 30)
self.assertEqual(summary["peak_cpu_percent"], 4.0)
self.assertEqual(summary["peak_process_count"], 2)
self.assertEqual(summary["max_worker_peak_pss_bytes"], 20)
self.assertEqual(summary["worker_count"], 2)
def test_row_summary_keeps_failure_kind_counts(self) -> None:
summary = _summarize_rows(
[
{"ok": True, "elapsed_ms": 10.0, "browser_peak_pss_bytes": 100, "assertion_failures": 0},
{
"ok": False,
"elapsed_ms": 20.0,
"browser_peak_pss_bytes": 200,
"assertion_failures": 2,
"failure_kind": "assertion-failure",
},
]
)
self.assertEqual(summary["runs"], 2)
self.assertEqual(summary["passes"], 1)
self.assertEqual(summary["failures"], 1)
self.assertEqual(summary["assertion_failures"], 2)
self.assertEqual(summary["failure_kinds"], {"assertion-failure": 1})
self.assertEqual(summary["elapsed_ms"]["p50"], 10.0)
def test_formal_profile_requirements_are_reported_without_browser(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
summary = run_amiibo_crawler_suite(
output_dir=Path(temp_dir),
target_matrix={"moli": {"available": False}},
profile="formal",
targets=("moli",),
pools=AMIIBO_CONCURRENCY_MATRIX,
modes=AMIIBO_MODES,
runs=1,
limit=0,
timeout_seconds=0.1,
gate_target="moli",
)
self.assertEqual(summary["profile"], "formal")
self.assertEqual(summary["profile_failures"], 0)
self.assertEqual(summary["formal_requirements"]["limit"]["ok"], True)
self.assertEqual(summary["formal_requirements"]["pool"]["ok"], True)
self.assertEqual(summary["formal_requirements"]["modes"]["ok"], True)
self.assertEqual(summary["targets"]["moli-cdp"]["failure_kinds"], {"target-unavailable": 12})
self.assertEqual(summary["targets"]["moli-cdp"]["engine"], "moli")
self.assertEqual(summary["targets"]["moli-cdp"]["driver"], "cdp")
def test_smoke_profile_does_not_gate_on_formal_requirements(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
summary = run_amiibo_crawler_suite(
output_dir=Path(temp_dir),
target_matrix={"moli": {"available": False}},
profile="smoke",
targets=("moli",),
pools=(1,),
modes=("session",),
runs=1,
limit=5,
timeout_seconds=0.1,
gate_target="chrome",
)
self.assertEqual(summary["profile"], "smoke")
self.assertEqual(summary["profile_failures"], 0)
self.assertEqual(summary["formal_requirements"]["limit"]["ok"], False)
self.assertEqual(summary["formal_requirements"]["pool"]["ok"], False)
self.assertEqual(summary["formal_requirements"]["modes"]["ok"], False)
self.assertEqual(summary["gate_failures"], 0)
if __name__ == "__main__":
unittest.main()
moli-benchmark/tests/test_cdp_memory_ablation_probe.py
+212
View File
@@ -0,0 +1,212 @@
from __future__ import annotations
import importlib.util
import sys
import types
import unittest
import urllib.parse
from pathlib import Path
SCRIPT_PATH = (
Path(__file__).resolve().parents[1] / "scripts" / "probe-cdp-memory-ablation.py"
)
def load_probe_module():
sys.modules.setdefault("websockets", types.ModuleType("websockets"))
sys.modules.setdefault("websockets.asyncio", types.ModuleType("websockets.asyncio"))
websockets_client = types.ModuleType("websockets.asyncio.client")
websockets_client.ClientConnection = object
sys.modules.setdefault("websockets.asyncio.client", websockets_client)
spec = importlib.util.spec_from_file_location("probe_cdp_memory_ablation", SCRIPT_PATH)
if spec is None or spec.loader is None:
raise RuntimeError(f"could not load {SCRIPT_PATH}")
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
class CdpMemoryAblationProbeTests(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
cls.probe = load_probe_module()
def test_fixed_probe_data_page_url_has_no_shared_worker_constructor(self) -> None:
url = self.probe.fixed_probe_page_url(
self.probe.FIXED_PROBE_DATA_PAGES,
index=2,
payload_kib=1,
)
html = urllib.parse.unquote(url.removeprefix("data:text/html,"))
self.assertIn('data-fixed-probe="data-pages"', html)
self.assertIn('data-index="2"', html)
self.assertNotIn("new SharedWorker", html)
def test_fixed_probe_shared_worker_key_modes_are_explicit(self) -> None:
same_key = urllib.parse.unquote(
self.probe.fixed_probe_page_url(
self.probe.FIXED_PROBE_SHARED_WORKER_SAME_KEY,
index=1,
payload_kib=0,
).removeprefix("data:text/html,")
)
distinct_key = urllib.parse.unquote(
self.probe.fixed_probe_page_url(
self.probe.FIXED_PROBE_SHARED_WORKER_DISTINCT_KEY,
index=3,
payload_kib=0,
).removeprefix("data:text/html,")
)
self.assertIn('"fixed-shared-worker"', same_key)
self.assertIn('"fixed-shared-worker-3"', distinct_key)
self.assertIn("new SharedWorker", same_key)
self.assertIn("new SharedWorker", distinct_key)
def test_fixed_probe_shared_worker_http_fixture_uses_same_origin_worker_url(self) -> None:
url = self.probe.fixed_probe_page_url(
self.probe.FIXED_PROBE_SHARED_WORKER_SAME_KEY,
index=2,
payload_kib=0,
base_url="http://127.0.0.1:43210",
)
parsed = urllib.parse.urlparse(url)
query = urllib.parse.parse_qs(parsed.query)
self.assertEqual(parsed.scheme, "http")
self.assertEqual(parsed.netloc, "127.0.0.1:43210")
self.assertEqual(parsed.path, "/fixed-page")
self.assertEqual(query["probe"], ["shared-worker-same-key"])
self.assertEqual(query["index"], ["2"])
html = self.probe.fixed_probe_page_html(
self.probe.FIXED_PROBE_SHARED_WORKER_SAME_KEY,
index=2,
payload_kib=0,
worker_script_url="/fixed-shared-worker.js",
)
self.assertIn('new SharedWorker(url, "fixed-shared-worker")', html)
self.assertIn('const url = "/fixed-shared-worker.js";', html)
def test_fixed_probe_popup_and_background_modes_do_not_install_shared_worker(self) -> None:
self.assertIn(
self.probe.FIXED_PROBE_BACKGROUND_TARGETS,
self.probe.FIXED_PROBE_CHOICES,
)
self.assertIn(
self.probe.FIXED_PROBE_POPUP_TARGETS,
self.probe.FIXED_PROBE_CHOICES,
)
popup = urllib.parse.unquote(
self.probe.fixed_probe_page_url(
self.probe.FIXED_PROBE_POPUP_TARGETS,
index=4,
payload_kib=0,
).removeprefix("data:text/html,")
)
background = urllib.parse.unquote(
self.probe.fixed_probe_page_url(
self.probe.FIXED_PROBE_BACKGROUND_TARGETS,
index=5,
payload_kib=0,
).removeprefix("data:text/html,")
)
self.assertIn('data-fixed-probe="popup-targets"', popup)
self.assertIn('data-index="4"', popup)
self.assertNotIn("new SharedWorker", popup)
self.assertIn('data-fixed-probe="background-targets"', background)
self.assertIn('data-index="5"', background)
self.assertNotIn("new SharedWorker", background)
def test_fixed_probe_dedicated_worker_mode_is_explicit(self) -> None:
self.assertIn(
self.probe.FIXED_PROBE_DEDICATED_WORKER,
self.probe.FIXED_PROBE_CHOICES,
)
self.assertIn(
self.probe.FIXED_PROBE_DEDICATED_WORKER,
self.probe.FIXED_PROBE_WORKER_MODES,
)
html = urllib.parse.unquote(
self.probe.fixed_probe_page_url(
self.probe.FIXED_PROBE_DEDICATED_WORKER,
index=6,
payload_kib=0,
).removeprefix("data:text/html,")
)
self.assertIn('data-fixed-probe="dedicated-worker"', html)
self.assertIn("new Worker", html)
self.assertIn("setInterval", html)
self.assertNotIn("new SharedWorker", html)
def test_fixed_probe_different_browser_context_mode_has_plain_page_payload(self) -> None:
self.assertIn(
self.probe.FIXED_PROBE_DIFFERENT_BROWSER_CONTEXTS,
self.probe.FIXED_PROBE_CHOICES,
)
html = urllib.parse.unquote(
self.probe.fixed_probe_page_url(
self.probe.FIXED_PROBE_DIFFERENT_BROWSER_CONTEXTS,
index=7,
payload_kib=0,
).removeprefix("data:text/html,")
)
self.assertIn('data-fixed-probe="different-browser-contexts"', html)
self.assertIn('data-index="7"', html)
self.assertNotIn("new Worker", html)
self.assertNotIn("new SharedWorker", html)
def test_compact_snapshot_includes_per_target_heap(self) -> None:
snapshot = {
"label": "fixed_before_close_targets",
"resources": {"rss_bytes": 1024 * 1024, "pss_bytes": 512 * 1024, "thread_count": 7},
"smaps": {
"categories": {},
"top_mappings": [],
"anonymous_histogram_by_vma_size": {},
},
"extra": {
"perTargetHeap": [
{
"index": 1,
"targetId": "TID-1",
"sessionId": "SID-1",
"browserContextId": "BID-1",
"heap": {
"elapsed_ms": 1.0,
"seen_count": 2,
"response": {
"usedSize": 2 * 1024 * 1024,
"totalSize": 4 * 1024 * 1024,
"totalPhysicalSize": 3 * 1024 * 1024,
"mallocedMemory": 0,
"externalMemory": 0,
"numberOfNativeContexts": 1,
"numberOfDetachedContexts": 0,
},
},
}
]
},
}
compact = self.probe.compact_snapshot(snapshot)
self.assertEqual(compact["per_target_heap"][0]["index"], 1)
self.assertEqual(compact["per_target_heap"][0]["targetId"], "TID-1")
self.assertEqual(compact["per_target_heap"][0]["browserContextId"], "BID-1")
self.assertEqual(compact["per_target_heap"][0]["heap"]["used_mib"], 2.0)
self.assertEqual(compact["per_target_heap"][0]["heap"]["native_contexts"], 1)
if __name__ == "__main__":
unittest.main()

Some files were not shown because too many files have changed in this diff Show More