diff --git a/moli-dom/src/accessibility/ax_node.rs b/moli-dom/src/accessibility/ax_node.rs index 28490b603c..78a64b697e 100644 --- a/moli-dom/src/accessibility/ax_node.rs +++ b/moli-dom/src/accessibility/ax_node.rs @@ -1,4 +1,5 @@ use std::fmt; +use std::num::NonZeroU32; use serde::{Serialize, Serializer, ser::SerializeMap}; use serde_json::Value; @@ -6,8 +7,8 @@ use serde_json::Value; /// Keep references numeric until a protocol or dump actually serializes them. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum AccessibilityNodeId { - Dom(u32), - ListMarker(u32), + Dom(NonZeroU32), + ListMarker(NonZeroU32), } impl fmt::Display for AccessibilityNodeId { @@ -37,15 +38,79 @@ pub struct AccessibilityValue { #[derive(Debug)] pub struct AccessibilityNode { pub node_id: AccessibilityNodeId, - pub backend_node_id: Option, + pub backend_node_id: Option, pub ignored: bool, - pub ignored_reasons: Option, - pub role: AccessibilityValue<&'static str>, - pub name: Option>, - pub value: Option, - pub properties: Option>, + pub role: &'static str, + pub name: Option, pub parent_id: Option, pub child_ids: Option>, + properties_present: bool, + rare_data: Option>, +} + +/// Common nodes have neither a value nor an ignored reason, and most have no +/// properties. Keep their empty JSON containers out of the common record. +#[derive(Debug, Default)] +struct AccessibilityNodeRareData { + ignored_reasons: Option, + value: Option, + properties: Vec, +} + +impl AccessibilityNode { + /// An ordinary AX node includes an empty properties array even when it + /// needs no rare allocation. Synthetic nodes can omit it via set_rare_data. + pub fn new(node_id: AccessibilityNodeId, role: &'static str) -> Self { + Self { + node_id, + backend_node_id: match node_id { + AccessibilityNodeId::Dom(id) => Some(id), + AccessibilityNodeId::ListMarker(_) => None, + }, + ignored: false, + role, + name: None, + parent_id: None, + child_ids: None, + properties_present: true, + rare_data: None, + } + } + + pub fn set_rare_data( + &mut self, + ignored_reasons: Option, + value: Option, + properties: Option>, + ) { + self.properties_present = properties.is_some(); + let properties = properties.unwrap_or_default(); + self.rare_data = if ignored_reasons.is_none() && value.is_none() && properties.is_empty() { + None + } else { + Some(Box::new(AccessibilityNodeRareData { + ignored_reasons, + value, + properties, + })) + }; + } + + pub fn ignored_reasons(&self) -> Option<&Value> { + self.rare_data.as_deref()?.ignored_reasons.as_ref() + } + + pub fn value(&self) -> Option<&Value> { + self.rare_data.as_deref()?.value.as_ref() + } + + pub fn properties(&self) -> Option<&[Value]> { + self.properties_present.then(|| { + self.rare_data + .as_deref() + .map_or(&[][..], |data| data.properties.as_slice()) + }) + } } impl Serialize for AccessibilityNode { @@ -58,27 +123,37 @@ impl Serialize for AccessibilityNode { payload.serialize_entry("ignored", &self.ignored)?; // DOM nodes without an AX object have a smaller synthetic payload, // whose role historically precedes its ignored reason. - if self.properties.is_none() { - payload.serialize_entry("role", &self.role)?; + let role = AccessibilityValue { + kind: "role", + value: self.role, + }; + if !self.properties_present { + payload.serialize_entry("role", &role)?; } - if let Some(reasons) = &self.ignored_reasons { + if let Some(reasons) = self.ignored_reasons() { payload.serialize_entry("ignoredReasons", reasons)?; } - if self.properties.is_some() { - payload.serialize_entry("role", &self.role)?; + if self.properties_present { + payload.serialize_entry("role", &role)?; } if let Some(name) = &self.name { - payload.serialize_entry("name", name)?; + payload.serialize_entry( + "name", + &AccessibilityValue { + kind: "computedString", + value: name, + }, + )?; } - if let Some(value) = &self.value { + if let Some(value) = self.value() { payload.serialize_entry("value", value)?; } - if let Some(properties) = &self.properties { + if let Some(properties) = self.properties() { payload.serialize_entry("properties", properties)?; } // Joining a child frame appends parentId to its existing root payload. let mut parent_id = self.parent_id; - if self.role.value != "RootWebArea" + if self.role != "RootWebArea" && let Some(parent) = parent_id.take() { payload.serialize_entry("parentId", &parent)?; @@ -98,18 +173,22 @@ impl From for Value { let mut payload = serde_json::Map::new(); payload.insert("nodeId".to_owned(), Value::String(node.node_id.to_string())); if let Some(id) = node.backend_node_id { - payload.insert("backendDOMNodeId".to_owned(), id.into()); + payload.insert("backendDOMNodeId".to_owned(), id.get().into()); } payload.insert("ignored".to_owned(), node.ignored.into()); - let mut ignored_reasons = node.ignored_reasons; - if node.properties.is_some() + let AccessibilityNodeRareData { + mut ignored_reasons, + value, + properties, + } = node.rare_data.map(|data| *data).unwrap_or_default(); + if node.properties_present && let Some(reasons) = ignored_reasons.take() { payload.insert("ignoredReasons".to_owned(), reasons); } payload.insert( "role".to_owned(), - serde_json::json!({"type":node.role.kind, "value":node.role.value}), + serde_json::json!({"type":"role", "value":node.role}), ); if let Some(reasons) = ignored_reasons { payload.insert("ignoredReasons".to_owned(), reasons); @@ -117,17 +196,17 @@ impl From for Value { if let Some(name) = node.name { payload.insert( "name".to_owned(), - serde_json::json!({"type":name.kind, "value":name.value}), + serde_json::json!({"type":"computedString", "value":name}), ); } - if let Some(value) = node.value { + if let Some(value) = value { payload.insert("value".to_owned(), value); } - if let Some(properties) = node.properties { + if node.properties_present { payload.insert("properties".to_owned(), Value::Array(properties)); } let mut parent_id = node.parent_id; - if node.role.value != "RootWebArea" + if node.role != "RootWebArea" && let Some(parent) = parent_id.take() { payload.insert("parentId".to_owned(), Value::String(parent.to_string())); @@ -149,3 +228,133 @@ impl From for Value { Self::Object(payload) } } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn id(value: u32) -> NonZeroU32 { + NonZeroU32::new(value).expect("nonzero fixture ID") + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn common_ax_node_layout_stays_compact() { + assert_eq!(size_of::>(), size_of::()); + assert_eq!(size_of::(), 8); + assert_eq!(size_of::>(), 8); + assert_eq!(size_of::>>(), 8); + assert!( + size_of::() <= 96, + "AccessibilityNode grew to {} bytes", + size_of::() + ); + } + + #[test] + fn empty_properties_do_not_allocate_rare_data() { + let mut node = AccessibilityNode::new(AccessibilityNodeId::Dom(id(7)), "generic"); + node.set_rare_data(None, None, Some(Vec::new())); + assert!(node.rare_data.is_none()); + assert_eq!(node.properties(), Some(&[][..])); + assert_eq!( + serde_json::to_string(&node).expect("AX JSON"), + r#"{"nodeId":"AX-7","backendDOMNodeId":7,"ignored":false,"role":{"type":"role","value":"generic"},"properties":[]}"# + ); + + node.set_rare_data(None, Some(json!({"type":"string", "value":"text"})), None); + assert!(node.rare_data.is_some()); + assert!(node.properties().is_none()); + node.set_rare_data(None, None, None); + assert!(node.rare_data.is_none()); + assert!(node.properties().is_none()); + } + + #[test] + fn cold_fields_preserve_protocol_values_and_field_order() { + for properties in [ + None, + Some(Vec::new()), + Some(vec![ + json!({"name":"focusable", "value":{"type":"boolean", "value":true}}), + ]), + ] { + let mut node = AccessibilityNode::new(AccessibilityNodeId::Dom(id(7)), "textbox"); + node.ignored = true; + node.name = Some("A name".to_owned()); + node.parent_id = Some(AccessibilityNodeId::Dom(id(1))); + node.child_ids = Some(Vec::new()); + let reasons = json!([{"name":"notRendered", "value":{"type":"boolean", "value":true}}]); + let value = json!({"type":"string", "value":"A value"}); + node.set_rare_data( + Some(reasons.clone()), + Some(value.clone()), + properties.clone(), + ); + + let serialized = serde_json::to_string(&node).expect("AX JSON"); + let mut expected = json!({ + "nodeId":"AX-7", "backendDOMNodeId":7, "ignored":true, + "ignoredReasons":reasons, "role":{"type":"role", "value":"textbox"}, + "name":{"type":"computedString", "value":"A name"}, "value":value, + "parentId":"AX-1", "childIds":[], + }); + if let Some(properties) = &properties { + expected["properties"] = json!(properties); + } + assert_eq!( + serde_json::from_str::(&serialized).expect("AX JSON"), + expected + ); + assert_eq!(Value::from(node), expected); + + let role_position = serialized.find("\"role\"").expect("role"); + let reason_position = serialized.find("\"ignoredReasons\"").expect("reasons"); + assert_eq!(role_position < reason_position, properties.is_none()); + assert!(serialized.find("\"parentId\"") < serialized.find("\"childIds\"")); + } + } + + #[test] + fn list_markers_and_child_frame_roots_keep_their_distinct_payloads() { + let mut marker = + AccessibilityNode::new(AccessibilityNodeId::ListMarker(id(7)), "ListMarker"); + marker.name = Some("• ".to_owned()); + marker.parent_id = Some(AccessibilityNodeId::Dom(id(7))); + marker.child_ids = Some(Vec::new()); + let expected = json!({ + "nodeId":"AX-LM-7", "ignored":false, + "role":{"type":"role", "value":"ListMarker"}, + "name":{"type":"computedString", "value":"• "}, + "properties":[], "parentId":"AX-7", "childIds":[], + }); + assert!(marker.rare_data.is_none()); + assert_eq!( + serde_json::to_value(&marker).expect("marker JSON"), + expected + ); + assert_eq!(Value::from(marker), expected); + + let mut root = AccessibilityNode::new(AccessibilityNodeId::Dom(id(8)), "RootWebArea"); + root.parent_id = Some(AccessibilityNodeId::Dom(id(7))); + root.child_ids = Some(vec![AccessibilityNodeId::Dom(id(9))]); + let serialized = serde_json::to_string(&root).expect("root JSON"); + assert!(serialized.find("\"childIds\"") < serialized.find("\"parentId\"")); + assert_eq!( + serde_json::from_str::(&serialized).expect("root JSON"), + Value::from(root) + ); + } + + #[test] + fn ids_preserve_the_full_nonzero_u32_range_and_marker_namespace() { + for value in [1, 2_000_000_000, u32::MAX] { + let dom = AccessibilityNodeId::Dom(id(value)); + let marker = AccessibilityNodeId::ListMarker(id(value)); + assert_ne!(dom, marker); + assert_eq!(dom.to_string(), format!("AX-{value}")); + assert_eq!(marker.to_string(), format!("AX-LM-{value}")); + } + } +} diff --git a/moli-dom/src/accessibility/ax_tree.rs b/moli-dom/src/accessibility/ax_tree.rs index 5b573f43aa..cd443339b0 100644 --- a/moli-dom/src/accessibility/ax_tree.rs +++ b/moli-dom/src/accessibility/ax_tree.rs @@ -4,11 +4,12 @@ use crate::{ }; use serde_json::{Value, json}; use std::collections::VecDeque; +use std::num::NonZeroU32; use super::ax_projection::{AxIgnoredReason, AxTreeProjection}; use super::ax_properties::{ax_name, ax_properties, ax_value}; use super::ax_roles::{ax_role, ordered_list_item_index}; -use super::{AccessibilityInput, AccessibilityNode, AccessibilityNodeId, AccessibilityValue}; +use super::{AccessibilityInput, AccessibilityNode, AccessibilityNodeId}; /// The semantic traversal requested by one synchronous AX command. #[derive(Clone, Copy, Debug)] @@ -189,24 +190,15 @@ fn ax_dom_node_without_object_payload( node_id: NodeId, backend_node_id_for_node: &mut dyn FnMut(NodeId) -> Option, ) -> Option { - let backend_node_id = backend_node_id_for_node(node_id)?; - Some(AccessibilityNode { - node_id: AccessibilityNodeId::Dom(backend_node_id), - backend_node_id: Some(backend_node_id), - ignored: true, - role: AccessibilityValue { - kind: "role", - value: "none", - }, - ignored_reasons: Some( - json!([{"name": "notRendered", "value": {"type": "boolean", "value": true}}]), - ), - name: None, - value: None, - properties: None, - parent_id: None, - child_ids: None, - }) + let backend_node_id = NonZeroU32::new(backend_node_id_for_node(node_id)?)?; + let mut node = AccessibilityNode::new(AccessibilityNodeId::Dom(backend_node_id), "none"); + node.ignored = true; + node.set_rare_data( + Some(json!([{"name": "notRendered", "value": {"type": "boolean", "value": true}}])), + None, + None, + ); + Some(node) } fn add_ax_children>( @@ -259,7 +251,7 @@ fn ax_node_payload( let document = projection.document; let node = document.node(node_id)?; let projected = projection.node(node_id).ok()??; - let backend_node_id = backend_node_id_for_node(node_id)?; + let backend_node_id = NonZeroU32::new(backend_node_id_for_node(node_id)?)?; let ignored = projected.ignored_reason.is_some(); let ignored_reasons = projected.ignored_reason.map(|reason| { ax_ignored_reasons_payload(document, node_id, reason, backend_node_id_for_node) @@ -280,7 +272,9 @@ fn ax_node_payload( ax_properties(document, node_id, node) }; let parent_id = match projected.parent { - Some(parent) => Some(AccessibilityNodeId::Dom(backend_node_id_for_node(parent)?)), + Some(parent) => Some(AccessibilityNodeId::Dom(NonZeroU32::new( + backend_node_id_for_node(parent)?, + )?)), None => None, }; let children = projection.children(node_id).ok()?; @@ -290,31 +284,23 @@ fn ax_node_payload( let mut child_ids = Vec::with_capacity(children.len()); for child_id in children { if projection.is_rendered(child_id).ok()? { - child_ids.push(AccessibilityNodeId::Dom(backend_node_id_for_node( - child_id, + child_ids.push(AccessibilityNodeId::Dom(NonZeroU32::new( + backend_node_id_for_node(child_id)?, )?)); } } Some(child_ids) }; - Some(AccessibilityNode { - node_id: AccessibilityNodeId::Dom(backend_node_id), - backend_node_id: Some(backend_node_id), - ignored, - ignored_reasons, - role: AccessibilityValue { - kind: "role", - value: if ignored { "none" } else { ax_role(node) }, - }, - name: (!name.is_empty()).then_some(AccessibilityValue { - kind: "computedString", - value: name, - }), - value, - properties: Some(properties), - parent_id, - child_ids, - }) + let mut payload = AccessibilityNode::new( + AccessibilityNodeId::Dom(backend_node_id), + if ignored { "none" } else { ax_role(node) }, + ); + payload.ignored = ignored; + payload.name = (!name.is_empty()).then_some(name); + payload.parent_id = parent_id; + payload.child_ids = child_ids; + payload.set_rare_data(ignored_reasons, value, Some(properties)); + Some(payload) } fn ax_ignored_reasons_payload( @@ -392,23 +378,13 @@ fn ax_list_marker_payload( _ => return None, }; - let parent_backend_node_id = backend_node_id_for_node(node_id)?; - Some(AccessibilityNode { - node_id: AccessibilityNodeId::ListMarker(parent_backend_node_id), - backend_node_id: None, - ignored: false, - ignored_reasons: None, - role: AccessibilityValue { - kind: "role", - value: "ListMarker", - }, - name: Some(AccessibilityValue { - kind: "computedString", - value: marker_text, - }), - value: None, - properties: Some(Vec::new()), - parent_id: Some(AccessibilityNodeId::Dom(parent_backend_node_id)), - child_ids: Some(Vec::new()), - }) + let parent_backend_node_id = NonZeroU32::new(backend_node_id_for_node(node_id)?)?; + let mut payload = AccessibilityNode::new( + AccessibilityNodeId::ListMarker(parent_backend_node_id), + "ListMarker", + ); + payload.name = Some(marker_text); + payload.parent_id = Some(AccessibilityNodeId::Dom(parent_backend_node_id)); + payload.child_ids = Some(Vec::new()); + Some(payload) } diff --git a/moli/src/fetch_dump/semantic.rs b/moli/src/fetch_dump/semantic.rs index d9eab948a1..4994e5cb90 100644 --- a/moli/src/fetch_dump/semantic.rs +++ b/moli/src/fetch_dump/semantic.rs @@ -58,10 +58,10 @@ fn attach_child_frame_accessibility_tree( let Some(child_root) = child_payloads.first_mut() else { return; }; - let Some(owner) = payloads - .iter_mut() - .find(|node| node.backend_node_id == Some(owner_backend_node_id)) - else { + let Some(owner) = payloads.iter_mut().find(|node| { + node.backend_node_id + .is_some_and(|id| id.get() == owner_backend_node_id) + }) else { return; }; child_root.parent_id = Some(owner.node_id); @@ -88,23 +88,20 @@ fn render_payloads_text(payloads: &[AccessibilityNode]) -> String { out.push_str(" "); } out.push_str("- "); - out.push_str(node.role.value); + out.push_str(node.role); if let Some(name) = &node.name - && !name.value.is_empty() + && !name.is_empty() { out.push_str(": "); - out.push_str(&name.value); + out.push_str(name); } - if let Some(value) = node - .value - .as_ref() - .and_then(|value| value["value"].as_str()) + if let Some(value) = node.value().and_then(|value| value["value"].as_str()) && !value.is_empty() { out.push_str(" = "); out.push_str(value); } - if let Some(backend) = node.backend_node_id.filter(|id| *id != 0) { + if let Some(backend) = node.backend_node_id { write!(&mut out, " [backendNodeId={backend}]").expect("writing to a String"); } out.push('\n'); @@ -122,8 +119,14 @@ fn render_payloads_text(payloads: &[AccessibilityNode]) -> String { #[cfg(test)] mod tests { + use std::num::NonZeroU32; + use super::*; + fn dom_id(id: u32) -> AccessibilityNodeId { + AccessibilityNodeId::Dom(NonZeroU32::new(id).expect("fixture backend ID")) + } + fn frame(frame_id: &str, child_frames: Vec) -> ChildFrameTreeSnapshot { ChildFrameTreeSnapshot { frame_id: frame_id.to_owned(), @@ -157,27 +160,10 @@ mod tests { parent: Option, children: &[u32], ) -> AccessibilityNode { - AccessibilityNode { - node_id: AccessibilityNodeId::Dom(id), - backend_node_id: Some(id), - ignored: false, - ignored_reasons: None, - role: moli_core::page::AccessibilityValue { - kind: "role", - value: role, - }, - name: None, - value: None, - properties: Some(Vec::new()), - parent_id: parent.map(AccessibilityNodeId::Dom), - child_ids: Some( - children - .iter() - .copied() - .map(AccessibilityNodeId::Dom) - .collect(), - ), - } + let mut node = AccessibilityNode::new(dom_id(id), role); + node.parent_id = parent.map(dom_id); + node.child_ids = Some(children.iter().copied().map(dom_id).collect()); + node } #[test] @@ -187,19 +173,13 @@ mod tests { node(2, "Iframe", Some(1), &[]), ]; let mut child_button = node(4, "button", Some(3), &[]); - child_button.name = Some(moli_core::page::AccessibilityValue { - kind: "computedString", - value: "Child action".to_owned(), - }); + child_button.name = Some("Child action".to_owned()); let child_payloads = vec![node(3, "RootWebArea", None, &[4]), child_button]; attach_child_frame_accessibility_tree(&mut payloads, 2, child_payloads); - assert_eq!( - payloads[1].child_ids, - Some(vec![AccessibilityNodeId::Dom(3)]) - ); - assert_eq!(payloads[2].parent_id, Some(AccessibilityNodeId::Dom(2))); + assert_eq!(payloads[1].child_ids, Some(vec![dom_id(3)])); + assert_eq!(payloads[2].parent_id, Some(dom_id(2))); assert!(render_payloads_text(&payloads).contains("button: Child action")); } @@ -232,11 +212,12 @@ mod tests { )); } let mut last = node(DEPTH + 1, "textbox", Some(1), &[]); - last.name = Some(moli_core::page::AccessibilityValue { - kind: "computedString", - value: "Name".to_owned(), - }); - last.value = Some(serde_json::json!({"type":"string", "value":"value"})); + last.name = Some("Name".to_owned()); + last.set_rare_data( + None, + Some(serde_json::json!({"type":"string", "value":"value"})), + Some(Vec::new()), + ); nodes.push(last); let text = render_payloads_text(&nodes); assert_eq!(text.lines().count(), (DEPTH + 1) as usize); diff --git a/scripts/benchmark-ax-compact.py b/scripts/benchmark-ax-compact.py new file mode 100644 index 0000000000..f185317ae7 --- /dev/null +++ b/scripts/benchmark-ax-compact.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +"""Compare semantic dumps with alternating before/after release binaries. + +Uses generated local fixtures, Linux wait4 peak RSS, three measured repetitions +by default, and SHA-256 checks for every output. Run without concurrent builds. +Example: + python3 scripts/benchmark-ax-compact.py --before /path/before/moli \ + --after /path/after/moli --output target/ax-compact-benchmark/results +""" + +import argparse +import functools +import hashlib +import html +import http.server +import json +import multiprocessing +import os +from pathlib import Path +import platform +import statistics +import subprocess +import threading +import time + + +def fixtures(root): + root.mkdir(parents=True, exist_ok=True) + + def page(name, body): + (root / f"{name}.html").write_text( + f"{name}" + f"{body}", encoding="utf-8" + ) + + for count in (500, 40000, 200000): + page(f"rows-{count}", "".join( + f'
Item {i}' + f' description {i}
' for i in range(count) + )) + page("table-10000", '' + '' + "".join( + f'' + for i in range(10000) + ) + '
Results
ItemCountStatus
Item {i}{i}Ready
') + page("forms-300", "".join( + f'' + f'' + f'' + f'' + f'' + for i in range(300) + )) + page("ignored-10000", "".join( + f'
Text {i}
' + for i in range(10000) + for attribute in [('aria-hidden="true"', 'inert', + 'style="visibility:hidden"')[i % 3]] + ) + '

Visible sibling

') + page("markers-10000", '
    ' + "".join( + f'
  • Bullet {i}
    • ' for i in range(5000) + ) + '
    ' + "".join( + f'
  1. Number {i}
    2. ' for i in range(5000) + ) + '
') + child = '

Frame heading

' + "".join( + f'

Frame row {i}

' for i in range(200) + ) + '' + page("frames", '

Main heading

' + "".join( + f'' + for i in range(4) + )) + page("mixed", '
  • Bullet
  1. Number
' + '' + '' + '
Inert
Invisible
' + '' + '
Locked
' + '' + '') + + +def run(binary, url, dump, frames, destination, timeout, env): + args = ["moli", "fetch", "--timeout", "120000", "--dump", dump] + if frames: + args.append("--with-frames") + args.append(url) + started = time.monotonic() + with destination.open("wb") as out, destination.with_suffix(".stderr").open("wb") as err: + process = subprocess.Popen(args, executable=binary, stdout=out, stderr=err, env=env) + timed_out = threading.Event() + + def terminate(): + timed_out.set() + process.kill() + + watchdog = threading.Timer(timeout, terminate) + watchdog.daemon = True + watchdog.start() + try: + _, status, usage = os.wait4(process.pid, 0) + process.returncode = os.waitstatus_to_exitcode(status) + finally: + watchdog.cancel() + elapsed = time.monotonic() - started + if timed_out.is_set(): + raise TimeoutError(f"{binary}: {url} {dump}") + if process.returncode: + raise RuntimeError(destination.with_suffix(".stderr").read_text()) + with destination.open("rb") as output: + digest = hashlib.file_digest(output, "sha256").hexdigest() + return {"maxrss_bytes": usage.ru_maxrss * 1024, "wall_seconds": elapsed, + "output_bytes": destination.stat().st_size, "sha256": digest} + + +def distribution(path): + with path.open() as file: + nodes = json.load(file) + return { + "nodes": len(nodes), + "ignored_reasons": sum("ignoredReasons" in node for node in nodes), + "values": sum("value" in node for node in nodes), + "nonempty_properties": sum(bool(node.get("properties")) for node in nodes), + "rare_payloads": sum("ignoredReasons" in node or "value" in node + or bool(node.get("properties")) for node in nodes), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--before", type=Path, required=True) + parser.add_argument("--after", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--repeats", type=int, default=3) + parser.add_argument("--timeout", type=float, default=180) + parser.add_argument("--cases", nargs="+") + parser.add_argument("--resume", action="store_true", + help="reuse complete samples after verifying binary hashes") + args = parser.parse_args() + if platform.system() != "Linux": + parser.error("peak RSS reporting currently requires Linux") + if args.repeats < 1: + parser.error("--repeats must be positive") + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=True) + root = output / "fixtures" + # A large fixture's temporary strings must not inflate the RSS inherited + # by subsequently forked benchmark processes. + generator = multiprocessing.get_context("spawn").Process(target=fixtures, args=(root,)) + generator.start() + generator.join() + if generator.exitcode: + raise RuntimeError("fixture generation failed") + binaries = {"before": str(args.before.resolve()), "after": str(args.after.resolve())} + manifest = {"binaries": {}, "platform": platform.platform(), "repeats": args.repeats} + for label, binary in binaries.items(): + with open(binary, "rb") as file: + manifest["binaries"][label] = { + "path": binary, "sha256": hashlib.file_digest(file, "sha256").hexdigest() + } + if args.resume: + previous_manifest = json.loads((output / "manifest.json").read_text()) + if previous_manifest != manifest: + parser.error("resume requires the same binaries, platform and repeat count") + (output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + env = {key: value for key, value in os.environ.items() + if key.lower() not in ("http_proxy", "https_proxy", "all_proxy")} + + class Handler(http.server.SimpleHTTPRequestHandler): + def log_message(self, *unused): + pass + + server = http.server.ThreadingHTTPServer( + ("127.0.0.1", 0), functools.partial(Handler, directory=str(root)) + ) + threading.Thread(target=server.serve_forever, daemon=True).start() + base = f"http://127.0.0.1:{server.server_port}" + cases = ["rows-500", "rows-40000", "table-10000", "forms-300", + "ignored-10000", "markers-10000", "frames", "mixed", "rows-200000"] + if args.cases: + unknown = set(args.cases) - set(cases) + if unknown: + parser.error(f"unknown cases: {sorted(unknown)}") + cases = args.cases + records = [] + completed = set() + if args.resume: + previous = [json.loads(line) for line in (output / "results.jsonl").read_text().splitlines()] + expected_trials = {(label, repeat) for label in binaries for repeat in range(args.repeats)} + for case, dump in dict.fromkeys((row["case"], row["dump"]) for row in previous): + group = [row for row in previous if (row["case"], row["dump"]) == (case, dump)] + trials = {(row["binary"], row["repeat"]) for row in group} + if trials == expected_trials and len(group) == len(expected_trials): + if len({row["sha256"] for row in group}) != 1: + raise AssertionError(f"existing output mismatch: {case} {dump}") + completed.add((case, dump)) + records = [row for row in previous if (row["case"], row["dump"]) in completed] + try: + # Both binaries receive the same argv[0], URL, environment and cwd. + for label, binary in binaries.items(): + run(binary, base + "/rows-500.html", "semantic_tree_text", False, + output / f"warmup-{label}.out", args.timeout, env) + with (output / "results.jsonl").open("w") as log: + for record in records: + log.write(json.dumps(record) + "\n") + log.flush() + for case in cases: + dumps = ("semantic_tree_text",) if case == "rows-200000" else ( + "semantic_tree_text", "semantic_tree" + ) + if case == "rows-40000": + dumps += ("markdown",) + for dump in dumps: + if (case, dump) in completed: + print(f"Reusing complete samples: {case} {dump}", flush=True) + continue + hashes = set() + for repeat in range(args.repeats): + order = ("before", "after") if repeat % 2 == 0 else ("after", "before") + for label in order: + destination = output / f"{case}-{dump}-{repeat}-{label}.out" + record = {"case": case, "dump": dump, "repeat": repeat, "binary": label, + **run(binaries[label], base + f"/{case}.html", dump, + case in ("frames", "mixed"), destination, args.timeout, env)} + hashes.add(record["sha256"]) + records.append(record) + line = json.dumps(record) + log.write(line + "\n") + log.flush() + print(line, flush=True) + if len(hashes) != 1: + raise AssertionError(f"output mismatch: {case} {dump}") + finally: + server.shutdown() + server.server_close() + + lines = ["# AX compact benchmark", "", + "Generated local fixtures; alternating runs; median of " + f"{args.repeats} runs; RSS in MiB. Before is the existing typed AX implementation.", "", + "| Case | Dump | RSS before | RSS after | RSS change | Time before | Time after | Time change |", + "|---|---|---:|---:|---:|---:|---:|---:|"] + for case, dump in dict.fromkeys((row["case"], row["dump"]) for row in records): + stats = {} + for label in binaries: + rows = [row for row in records if (row["case"], row["dump"], row["binary"]) == (case, dump, label)] + stats[label] = {key: statistics.median(row[key] for row in rows) + for key in ("maxrss_bytes", "wall_seconds")} + before, after = stats["before"], stats["after"] + rss_change = (after["maxrss_bytes"] / before["maxrss_bytes"] - 1) * 100 + wall_change = (after["wall_seconds"] / before["wall_seconds"] - 1) * 100 + lines.append(f"| {case} | {dump} | {before['maxrss_bytes']/2**20:.1f} | " + f"{after['maxrss_bytes']/2**20:.1f} | {rss_change:+.1f}% | " + f"{before['wall_seconds']:.3f}s | {after['wall_seconds']:.3f}s | {wall_change:+.1f}% |") + lines += ["", "Every output is byte-for-byte identical across binaries and repetitions.", "", + "Small-page timings include process startup and are sensitive to noise.", "", + "| Case | AX nodes | Ignored reasons | Values | Nonempty properties | Nodes with rare payload |", + "|---|---:|---:|---:|---:|---:|"] + distributions = {} + for case in cases: + path = output / f"{case}-semantic_tree-0-after.out" + if not path.exists(): + continue + stats = distributions[case] = distribution(path) + lines.append(f"| {case} | {stats['nodes']} | {stats['ignored_reasons']} | " + f"{stats['values']} | {stats['nonempty_properties']} | " + f"{stats['rare_payloads'] / max(stats['nodes'], 1):.1%} |") + (output / "node-distribution.json").write_text(json.dumps(distributions, indent=2) + "\n") + lines += ["", "Raw samples and binary hashes: `results.jsonl`, `manifest.json`.", ""] + report = "\n".join(lines) + (output / "report.md").write_text(report) + print(report) + + +if __name__ == "__main__": + main() diff --git a/scripts/benchmark-ax-native.py b/scripts/benchmark-ax-native.py new file mode 100644 index 0000000000..99079a4b4a --- /dev/null +++ b/scripts/benchmark-ax-native.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Benchmark typed AX and CDP payload construction using the same generated DOM. + +Build both revisions with `cargo build --release -p moli`, then pass their +separate target/release/deps directories. The standalone Rust harness uses the +system allocator and synthetic visible styles; the CLI benchmark measures the +complete browser pipeline with its default allocator and real layout. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform +import statistics +import subprocess +import time + +NATIVE_SOURCE = r"""use std::{io, time::Instant}; +use moli_dom::{NodeId, native::{DomHost, NativeDom}, accessibility::{AccessibilityStyle, AccessibilityStyleSource, AccessibilityInput, AccessibilityFrameState, AccessibilityRequest, accessibility_nodes_for_document, accessibility_payloads_for_document}}; +struct Styles; +impl AccessibilityStyleSource for Styles { + fn element_style(&mut self, _: NodeId) -> Option { + Some(AccessibilityStyle { display_none: false, visibility_visible: true, hides_contents: false, block_level: false }) + } +} +fn element(doc: &mut DomHost, parent: NodeId, tag: &str) -> NodeId { + let node = doc.create_element(tag); + assert!(doc.append_child(parent, node)); + node +} +fn text(doc: &mut DomHost, parent: NodeId, data: &str) { + let node = doc.create_text_node(data); + assert!(doc.append_child(parent, node)); +} +fn main() { + let args: Vec<_> = std::env::args().collect(); + let case = args[1].as_str(); + let path = args[2].as_str(); + let count: usize = args[3].parse().unwrap(); + let mut doc = DomHost::from_dom(NativeDom::new_html(url::Url::parse("https://example.test/").unwrap())); + let root = doc.document_node_id(); + let host = element(&mut doc, root, if case == "markers" { "ul" } else { "main" }); + for index in 0..count { + match case { + "rows" => { + let row = element(&mut doc, host, "div"); + let link = element(&mut doc, row, "a"); + doc.set_attribute(link, "href", &format!("/item/{index}")); + text(&mut doc, link, &format!("Item {index}")); + let span = element(&mut doc, row, "span"); + text(&mut doc, span, " description"); + } + "values" => { + let input = element(&mut doc, host, "input"); + doc.set_attribute(input, "aria-label", "Text input"); + doc.set_attribute(input, "value", "Input value"); + } + "ignored" => { + let div = element(&mut doc, host, "div"); + doc.set_attribute(div, "aria-hidden", "true"); + let button = element(&mut doc, div, "button"); + text(&mut doc, button, "Ignored action"); + } + "markers" => { + let item = element(&mut doc, host, "li"); + text(&mut doc, item, "List item"); + } + _ => panic!("unknown case"), + } + } + let mut styles = Styles; + let mut input = AccessibilityInput::new(&mut styles, AccessibilityFrameState::Active); + let mut backend_id = |node: NodeId| Some(u32::try_from(node.index() + 1).unwrap()); + let started = Instant::now(); + match path { + "typed" => { + let nodes = accessibility_nodes_for_document(&doc, &mut input, root, AccessibilityRequest::Tree { max_depth: None }, &mut backend_id).unwrap(); + eprintln!("{{\"nodes\":{},\"build_seconds\":{}}}", nodes.len(), started.elapsed().as_secs_f64()); + serde_json::to_writer(io::stdout().lock(), &nodes).unwrap(); + } + "protocol" => { + let nodes = accessibility_payloads_for_document(&doc, &mut input, root, AccessibilityRequest::Tree { max_depth: None }, &mut backend_id).unwrap(); + eprintln!("{{\"nodes\":{},\"build_seconds\":{}}}", nodes.len(), started.elapsed().as_secs_f64()); + serde_json::to_writer(io::stdout().lock(), &nodes).unwrap(); + } + _ => panic!("unknown path"), + } +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--before-deps', type=Path, required=True) + parser.add_argument('--after-deps', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--count', type=int, default=20000) + parser.add_argument('--repeats', type=int, default=5) + args = parser.parse_args() + if platform.system() != 'Linux': + parser.error('peak RSS reporting currently requires Linux') + if args.count < 1 or args.repeats < 1: + parser.error('--count and --repeats must be positive') + root = args.output.resolve() + root.mkdir(parents=True, exist_ok=True) + source = root / 'native-storage.rs' + source.write_text(NATIVE_SOURCE) + manifest = {'count': args.count, 'repeats': args.repeats, 'libraries': {}} + for label, deps in [('before', args.before_deps.resolve()), ('after', args.after_deps.resolve())]: + command = ['rustc', '--edition=2024', '--crate-name', 'native_ax_storage', str(source), + '-L', f'dependency={deps}', '-C', 'panic=abort', '-C', 'opt-level=3', + '-C', 'codegen-units=1', '-D', 'warnings', '-o', str(root/f'native-{label}')] + manifest['libraries'][label] = {} + for crate in ('moli_dom', 'serde_json', 'url'): + candidates = list(deps.glob(f'lib{crate}-*.rlib')) + if not candidates: + parser.error(f'missing {crate} release library in {deps}') + rlib = max(candidates, key=lambda path: path.stat().st_mtime) + with rlib.open('rb') as library: + digest = hashlib.file_digest(library, 'sha256').hexdigest() + manifest['libraries'][label][crate] = {'path': str(rlib), 'sha256': digest} + command += ['--extern', f'{crate}={rlib}'] + subprocess.run(command, check=True) + (root/'manifest.json').write_text(json.dumps(manifest, indent=2)+'\n') + records = [] + for label in ('before', 'after'): + subprocess.run(['ax-storage', 'rows', 'typed', '100'], + executable=str(root/f'native-{label}'), + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True) + with (root/'native-results.jsonl').open('w') as log: + for case in ('rows', 'values', 'ignored', 'markers'): + for path in ('typed', 'protocol'): + hashes = set() + for repeat in range(args.repeats): + order = ('before','after') if repeat % 2 == 0 else ('after','before') + for label in order: + dest = root/f'native-{case}-{path}-{repeat}-{label}.out' + meta = dest.with_suffix('.meta') + started = time.monotonic() + with dest.open('wb') as out, meta.open('wb') as err: + process = subprocess.Popen(['ax-storage',case,path,str(args.count)], + executable=str(root/f'native-{label}'), + stdout=out,stderr=err) + _, status, usage = os.wait4(process.pid,0) + process.returncode = os.waitstatus_to_exitcode(status) + if process.returncode: + raise RuntimeError(meta.read_text()) + wall = time.monotonic()-started + with dest.open('rb') as out: + digest = hashlib.file_digest(out,'sha256').hexdigest() + hashes.add(digest) + record = dict(case=case,path=path,repeat=repeat,binary=label, + maxrss_bytes=usage.ru_maxrss*1024,wall_seconds=wall, + sha256=digest,**json.loads(meta.read_text())) + records.append(record) + line=json.dumps(record) + log.write(line+'\n') + log.flush() + print(line,flush=True) + assert len(hashes)==1,(case,path,hashes) + lines=['# Native AX construction benchmark','', + 'Same generated DOM and style source; Rust system allocator; no V8 or real layout.', + f'Median of {args.repeats} alternating runs. Build time isolates the AX API call; peak RSS covers the process including DOM construction and streaming serialization.','', + '| Case | Path | Nodes | RSS before MiB | RSS after MiB | Build before | Build after | Build change |', + '|---|---|---:|---:|---:|---:|---:|---:|'] + for case,path in dict.fromkeys((r['case'],r['path']) for r in records): + data={label:[r for r in records if (r['case'],r['path'],r['binary'])==(case,path,label)] for label in ('before','after')} + rss={label:statistics.median(r['maxrss_bytes']/2**20 for r in rows) for label,rows in data.items()} + build={label:statistics.median(r['build_seconds'] for r in rows) for label,rows in data.items()} + lines.append(f"| {case} | {path} | {data['before'][0]['nodes']} | {rss['before']:.1f} | {rss['after']:.1f} | {build['before']:.4f}s | {build['after']:.4f}s | {(build['after']/build['before']-1)*100:+.1f}% |") + lines += ['', 'Before/after output SHA-256 matches for every case and path.', ''] + report='\n'.join(lines) + (root/'native-report.md').write_text(report) + print(report) + + +if __name__ == '__main__': + main()