mirror of
https://github.com/lexmount/moli.git
synced 2026-11-18 16:08:37 +00:00
329 lines
19 KiB
Python
329 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Compare the same Limo consumer with renderer DevTools enabled and disabled.
|
|
|
|
Uses this checkout's patches and local moli-core. Both variants share a lockfile,
|
|
target directory and release profile; the existing Limo executable is preserved.
|
|
Run from the Moli root: python3 tools/devtools-size-ab.py --limo-root ../limo
|
|
Standalone checks need no Limo checkout: python3 tools/devtools-size-ab.py --check-only --runtime-smoke
|
|
"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
|
|
|
|
def elf_sections(path):
|
|
data = path.read_bytes()
|
|
header = struct.unpack_from('<16sHHIQQQIHHHHHH', data)
|
|
assert header[0][:6] == b'\x7fELF\x02\x01', 'Expected little-endian ELF64'
|
|
sections = [struct.unpack_from('<IIQQQQIIQQ', data, header[6] + i * header[11])
|
|
for i in range(header[12])]
|
|
strings = sections[header[13]]
|
|
names = data[strings[4]:strings[4] + strings[5]]
|
|
return {names[s[0]:].split(b'\0', 1)[0].decode(): s[5] for s in sections}
|
|
|
|
|
|
def audit_symbols(path, devtools):
|
|
"""Check linked implementations, including drop glue, rather than strings."""
|
|
symbols = path.read_text()
|
|
frontend = (
|
|
'::initialize_inspector_session_after_attach',
|
|
'::prepare_frontend',
|
|
'::run_dom_inspector_edit',
|
|
'::service_worker_runtime::service::devtools_commands::',
|
|
'RendererDevToolsSessionOutputHost',
|
|
'WorkerInspectorSessionRetirement',
|
|
'WorkerInspectorOutboundState',
|
|
'::script_vm::input_dispatch::',
|
|
'::page_vm::frontend_node_bindings::',
|
|
'::page_vm::dom_mutations::',
|
|
'WorkerInspectorClient as v8::inspector::V8InspectorClientImpl>::run_message_loop_on_pause',
|
|
)
|
|
matches = {name: [line for line in symbols.splitlines() if name in line]
|
|
for name in frontend}
|
|
retained = {name: lines for name, lines in matches.items() if lines}
|
|
native_inspector = 'v8_inspector::V8Inspector::create(' in symbols
|
|
if not native_inspector:
|
|
raise RuntimeError('The native V8 Inspector backend must remain linked')
|
|
if not devtools and retained:
|
|
raise RuntimeError(f'Frontend implementations remain linked: {list(retained)}')
|
|
return {'native_v8_inspector': native_inspector,
|
|
'frontend_implementations': retained,
|
|
'checked_frontend_patterns': list(frontend)}
|
|
|
|
|
|
def runtime_fingerprint(moli):
|
|
paths = subprocess.check_output(
|
|
['git', 'ls-files', '--cached', '--others', '--exclude-standard', '--',
|
|
'Cargo.toml', 'Cargo.lock', 'moli-types', 'moli-core', 'moli-renderer-v8',
|
|
'moli-protocol/Cargo.toml'], cwd=moli, text=True).splitlines()
|
|
digest = hashlib.sha256()
|
|
for name in sorted(set(paths)):
|
|
path = moli / name
|
|
if path.is_file() and (path.suffix == '.rs' or path.name in ('Cargo.toml', 'Cargo.lock')):
|
|
digest.update(name.encode() + b'\0' + path.read_bytes() + b'\0')
|
|
return digest.hexdigest()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--limo-root', type=Path,
|
|
help='Limo checkout for release size and runtime comparison; optional with --check-only')
|
|
parser.add_argument('--output', type=Path)
|
|
parser.add_argument('--jobs', type=int, default=2)
|
|
parser.add_argument('--online', action='store_true')
|
|
parser.add_argument('--order', choices=('on-first', 'off-first'), default='off-first')
|
|
parser.add_argument('--skip-contracts', action='store_true', help='Reuse previously checked API contracts')
|
|
parser.add_argument('--check-only', action='store_true', help='Check feature/API contracts without linking')
|
|
parser.add_argument('--runtime-smoke', action='store_true',
|
|
help='Also build and run the standalone probe with DevTools disabled in check-only mode')
|
|
args = parser.parse_args()
|
|
if args.limo_root is None and not args.check_only:
|
|
parser.error('--limo-root is required for the release comparison; use --check-only for standalone API checks')
|
|
moli = Path(__file__).resolve().parent.parent
|
|
source_fingerprint = runtime_fingerprint(moli)
|
|
limo = args.limo_root.resolve() if args.limo_root else None
|
|
target = limo / 'target' if limo else Path(os.environ.get('CARGO_TARGET_DIR', moli / 'target'))
|
|
output = (args.output or target / 'devtools-ab').resolve()
|
|
app = output / 'app'
|
|
(app / 'src').mkdir(parents=True, exist_ok=True)
|
|
(output / 'tmp').mkdir(exist_ok=True)
|
|
if limo:
|
|
for source in ('main.rs', 'server.rs'):
|
|
shutil.copyfile(limo / 'src' / source, app / 'src' / source)
|
|
manifest = (limo / 'Cargo.toml').read_text().split('[patch.crates-io]')[0]
|
|
else:
|
|
(app / 'src/main.rs').write_text('fn main() {}\n')
|
|
manifest = '''[package]
|
|
name = "moli-devtools-contracts"
|
|
version = "0.0.0"
|
|
edition = "2024"
|
|
|
|
[dependencies]
|
|
moli-core = { default-features = false }
|
|
|
|
[profile.release]
|
|
panic = "abort"
|
|
codegen-units = 1
|
|
'''
|
|
lines = manifest.splitlines()
|
|
for i, line in enumerate(lines):
|
|
if line.startswith('moli-core = '):
|
|
lines[i] = f'moli-core = {{ path = "{moli / "moli-core"}", default-features = false }}'
|
|
# Append the dependency before the release profile rather than under it.
|
|
dependency_end = next(i for i, line in enumerate(lines) if line.startswith('[profile.'))
|
|
lines.insert(dependency_end, f'moli-command-types = {{ path = "{moli / "moli-types/moli-command-types"}", default-features = false }}\n')
|
|
lines.insert(dependency_end, f'moli-renderer-v8 = {{ path = "{moli / "moli-renderer-v8"}", default-features = false }}\n')
|
|
if not limo and args.runtime_smoke:
|
|
lines[dependency_end:dependency_end] = [
|
|
'anyhow = "1"',
|
|
'axum = "0.8"',
|
|
'tokio = { version = "1", features = ["net", "rt", "time"] }',
|
|
]
|
|
manifest = '\n'.join(lines) + '\n\n[workspace]\n\n[features]\ndefault = []\ndevtools = ["moli-core/devtools", "moli-command-types/devtools"]\ntypes-devtools-only = ["moli-command-types/devtools"]\nrenderer-devtools-only = ["moli-renderer-v8/devtools"]\nace-only = ["moli-core/ace-protocol"]\n\n'
|
|
patches = (moli / 'Cargo.toml').read_text()
|
|
patches = patches[patches.index('[patch.crates-io]'):patches.index('[workspace.lints.clippy]')]
|
|
patches = patches.replace('path = "vendor/', f'path = "{moli}/vendor/')
|
|
manifest += patches
|
|
if limo or args.runtime_smoke:
|
|
shutil.copyfile(moli / 'tools/devtools-size-probe.rs', app / 'src/devtools_probe.rs')
|
|
manifest += '\n[[bin]]\nname = "devtools-probe"\npath = "src/devtools_probe.rs"\n'
|
|
contracts = {
|
|
'runtime-contract': '''fn main() {
|
|
let _ = moli_core::page::Page::start_evaluate_runtime_expression;
|
|
let _ = moli_core::page::Page::start_capture_screenshot;
|
|
let _ = moli_core::page::Page::start_print_to_pdf;
|
|
let _ = moli_core::runtime::Browser::wait_for_selector;
|
|
let _ = moli_core::runtime::Browser::wait_for_script_truthy;
|
|
let _: Option<moli_command_types::RuntimeRealmInfo> = None;
|
|
}''',
|
|
'devtools-types-contract': '''use moli_command_types::devtools::{InspectorCommand, InspectorCommandHandles, NativeProtocolResponse};
|
|
struct Handles;
|
|
impl InspectorCommandHandles for Handles {
|
|
type ResponseSender = ();
|
|
type RuntimeOutput = ();
|
|
type SessionDetachGuard = ();
|
|
}
|
|
fn main() {
|
|
let _: InspectorCommand<Handles> = InspectorCommand::RuntimeEnableEvents;
|
|
let _ = NativeProtocolResponse::<()>::success(Default::default());
|
|
}''',
|
|
'devtools-api-contract': '''fn main() {
|
|
let _ = moli_core::RendererCdpCall::new;
|
|
let _ = moli_core::RendererDevToolsPageCommand::Inspector;
|
|
let _ = moli_core::RendererDevToolsPageCommand::Native;
|
|
let _ = moli_core::RendererPageCommand::dispatch_runtime_protocol_message;
|
|
let _ = moli_core::page::Page::start_runtime_protocol_message;
|
|
}''',
|
|
'devtools-dom-contract': '''fn main() {
|
|
let _ = moli_core::RendererPageCommand::DevTools(moli_core::RendererDevToolsPageCommand::DocumentBidiNodeBinding {
|
|
inspector_session_id: None,
|
|
shared_id: String::new(),
|
|
});
|
|
let _ = moli_core::RendererPageCommand::DevTools(moli_core::RendererDevToolsPageCommand::DocumentPerformSearch {
|
|
inspector_session_id: None,
|
|
query: String::new(),
|
|
include_user_agent_shadow_dom: false,
|
|
include_whitespace: false,
|
|
});
|
|
}''',
|
|
'devtools-worker-contract': '''struct Handles;
|
|
impl moli_command_types::devtools::WorkerCommandHandles for Handles {
|
|
type SessionLifetime = ();
|
|
type ResponseSender = ();
|
|
type ProtocolResponseSender = ();
|
|
}
|
|
fn main() {
|
|
let _: moli_command_types::devtools::WorkerCommand<Handles> =
|
|
moli_command_types::devtools::WorkerCommand::RunIfWaitingForDebugger;
|
|
}''',
|
|
}
|
|
removed_apis = {
|
|
'native-call': ('moli_core::RendererCdpCall::new', 'RendererCdpCall'),
|
|
'command-group': ('moli_core::RendererPageCommand::DevTools', 'DevTools'),
|
|
'reply-group': ('moli_core::RendererPageReply::DevTools', 'DevTools'),
|
|
'devtools-reply-type': ('moli_core::RendererDevToolsPageReply::NativeCommandPublished', 'RendererDevToolsPageReply'),
|
|
'inspector-command': ('moli_core::RendererDevToolsPageCommand::Inspector', 'RendererDevToolsPageCommand'),
|
|
'native-command': ('moli_core::RendererDevToolsPageCommand::Native', 'RendererDevToolsPageCommand'),
|
|
'raw-dispatch': ('moli_core::RendererPageCommand::dispatch_runtime_protocol_message', 'dispatch_runtime_protocol_message'),
|
|
'page-dispatch': ('moli_core::page::Page::start_runtime_protocol_message', 'start_runtime_protocol_message'),
|
|
'input': ('moli_core::page::Page::start_dispatch_key_event_with_location_outcome', 'start_dispatch_key_event_with_location_outcome'),
|
|
'network': ('moli_core::page::Page::start_set_fetch_subresource_interception', 'start_set_fetch_subresource_interception'),
|
|
'dedicated-worker-pause': ('moli_core::runtime::RendererBrowserContextRuntime::set_dedicated_worker_pause_on_start_for_devtools', 'set_dedicated_worker_pause_on_start_for_devtools'),
|
|
'shared-worker-pause': ('moli_core::runtime::RendererBrowserContextRuntime::set_shared_worker_pause_on_start_for_devtools', 'set_shared_worker_pause_on_start_for_devtools'),
|
|
'navigation-interception': ('moli_core::runtime::NavigationEngine::set_subresource_interception', 'set_subresource_interception'),
|
|
'commit-interception': ('|config: &moli_core::runtime::PreparedDocumentPageCommitConfiguration| config.fetch_subresource_interception', 'fetch_subresource_interception'),
|
|
'network-policy-interception': ('moli_renderer_v8::network::PageNetworkPolicy::set_subresource_interception', 'set_subresource_interception'),
|
|
'worker-policy-interception': ('moli_renderer_v8::network::ResourceInterceptionPolicy::set_interception', 'set_interception'),
|
|
}
|
|
for name, (expression, _) in removed_apis.items():
|
|
contracts[f'devtools-{name}-contract'] = f'fn main() {{ let _ = {expression}; }}'
|
|
for name, source in contracts.items():
|
|
(app / f'src/{name}.rs').write_text(source + '\n')
|
|
manifest += f'\n[[bin]]\nname = "{name}"\npath = "src/{name}.rs"\n'
|
|
(app / 'Cargo.toml').write_text(manifest)
|
|
if not (app / 'Cargo.lock').exists():
|
|
shutil.copyfile((limo or moli) / 'Cargo.lock', app / 'Cargo.lock')
|
|
|
|
scratch = os.environ.get('TMPDIR') or str(output / 'tmp')
|
|
env = dict(os.environ, CARGO_TARGET_DIR=str(target),
|
|
TMPDIR=scratch, TMP=scratch, TEMP=scratch,
|
|
no_proxy='127.0.0.1,localhost,::1',
|
|
NO_PROXY='127.0.0.1,localhost,::1')
|
|
if limo:
|
|
env['LIMO_TEST_ROOT'] = str(limo)
|
|
|
|
def run(command, name, timeout=None):
|
|
print(' '.join(str(x) for x in command), flush=True)
|
|
with (output / f'{name}.log').open('w') as log:
|
|
subprocess.run(command, cwd=moli, env=env, stdout=log,
|
|
stderr=subprocess.STDOUT, check=True, timeout=timeout)
|
|
|
|
common = ([] if args.check_only else ['--release']) + ['--manifest-path', str(app / 'Cargo.toml'),
|
|
'--no-default-features', '--jobs', str(args.jobs)]
|
|
if not args.online:
|
|
common += ['--offline']
|
|
if not args.skip_contracts:
|
|
run(['cargo', 'check', *common, '--bin', 'runtime-contract'], 'contract-runtime-off')
|
|
for feature in ['types-devtools-only', 'renderer-devtools-only', 'ace-only']:
|
|
run(['cargo', 'check', *common, '--bin', 'runtime-contract', '--features', feature],
|
|
f'contract-runtime-{feature}')
|
|
negative_contracts = [('devtools-types-contract', 'moli_command_types::devtools'),
|
|
('devtools-api-contract', 'RendererCdpCall'),
|
|
('devtools-dom-contract', 'DocumentBidiNodeBinding'),
|
|
('devtools-worker-contract', 'RunIfWaitingForDebugger')]
|
|
negative_contracts += [(f'devtools-{name}-contract', expected) for name, (_, expected) in removed_apis.items()]
|
|
for name, expected in negative_contracts:
|
|
run(['cargo', 'check', *common, '--bin', name, '--features', 'devtools'], f'contract-{name}-on')
|
|
with (output / f'contract-{name}-off.log').open('w') as log:
|
|
result = subprocess.run(['cargo', 'check', *common, '--bin', name], cwd=moli,
|
|
env=env, stdout=log, stderr=subprocess.STDOUT)
|
|
diagnostic = (output / f'contract-{name}-off.log').read_text()
|
|
if result.returncode == 0 or expected not in diagnostic:
|
|
raise RuntimeError(f'{name}: expected an unavailable DevTools API in the off build; see {output}')
|
|
if args.check_only:
|
|
if not args.skip_contracts:
|
|
print('Feature contracts passed. Runtime APIs compile, and DevTools APIs require the feature.')
|
|
if args.runtime_smoke:
|
|
run(['cargo', 'build', *common, '--bin', 'devtools-probe'], 'probe-build-off')
|
|
probe = 'devtools-probe.exe' if os.name == 'nt' else 'devtools-probe'
|
|
run([str(target.resolve() / 'debug' / probe)], 'probe-off', timeout=30)
|
|
print((output / 'probe-off.log').read_text().strip())
|
|
return
|
|
results = {}
|
|
order = ('off', 'on') if args.order == 'off-first' else ('on', 'off')
|
|
for mode in order:
|
|
flags = ['--features', 'devtools'] if mode == 'on' else []
|
|
tree = subprocess.check_output(
|
|
['cargo', 'tree', '--manifest-path', str(app / 'Cargo.toml'),
|
|
'--no-default-features', '--prefix', 'none', '--edges', 'normal,build',
|
|
'--format', '{p} {f}', *flags], cwd=moli, env=env, text=True)
|
|
(output / f'dependencies-{mode}.txt').write_text(tree)
|
|
if mode == 'off' and any(line.startswith('moli-protocol-cdp v') for line in tree.splitlines()):
|
|
raise RuntimeError('The off consumer still depends on moli-protocol-cdp')
|
|
unstripped = output / f'limo-{mode}.unstripped'
|
|
binary = output / f'limo-{mode}'
|
|
run(['cargo', 'rustc', *common, '--bin', 'limo', *flags, '--',
|
|
f'--emit=link={unstripped}', f'-C', f'link-arg=-Wl,-Map={output / f"limo-{mode}.map"}'],
|
|
f'build-{mode}')
|
|
if runtime_fingerprint(moli) != source_fingerprint:
|
|
raise RuntimeError('Runtime source changed during the build; rerun on a frozen checkout')
|
|
subprocess.run(['strip', '--strip-all', '-o', str(binary), str(unstripped)], check=True)
|
|
run(['cargo', 'build', *common, '--bin', 'devtools-probe', *flags], f'probe-build-{mode}')
|
|
shutil.copy2(limo / 'target/release/devtools-probe', output / f'probe-{mode}')
|
|
env['MOLI_PROBE_ARTIFACT_DIR'] = str(output / f'probe-artifacts-{mode}')
|
|
run([str(output / f'probe-{mode}')], f'probe-{mode}', timeout=30)
|
|
env['LIMO_TEST_BIN'] = str(binary)
|
|
run([sys.executable, str(moli / 'tools/devtools-size-smoke.py')], f'smoke-{mode}', timeout=180)
|
|
last = json.loads((output / f'smoke-{mode}.log').read_text().splitlines()[-1])
|
|
smoke = json.loads(Path(last['report']).read_text())
|
|
sections = elf_sections(binary)
|
|
with (output / f'symbols-{mode}.txt').open('w') as log:
|
|
subprocess.run(['nm', '-S', '--size-sort', '--demangle', str(unstripped)],
|
|
stdout=log, check=True)
|
|
audit = audit_symbols(output / f'symbols-{mode}.txt', mode == 'on')
|
|
results[mode] = {'bytes': binary.stat().st_size,
|
|
'mib': round(binary.stat().st_size / 2**20, 4),
|
|
'unstripped_bytes': unstripped.stat().st_size,
|
|
'sections': sections, 'smoke': smoke,
|
|
'sha256': hashlib.sha256(binary.read_bytes()).hexdigest(),
|
|
'symbol_audit': audit,
|
|
'probe': (output / f'probe-{mode}.log').read_text()}
|
|
(output / 'comparison.json').write_text(json.dumps(results, indent=2))
|
|
if runtime_fingerprint(moli) != source_fingerprint:
|
|
raise RuntimeError('Runtime source changed during the comparison; rerun on a frozen checkout')
|
|
png_hashes = {mode: hashlib.sha256((output / f'probe-artifacts-{mode}/probe.png').read_bytes()).hexdigest()
|
|
for mode in ('on', 'off')}
|
|
if len(set(png_hashes.values())) != 1:
|
|
raise RuntimeError('Screenshot output changed with DevTools disabled')
|
|
results['png_sha256'] = png_hashes
|
|
results['moli_runtime_source_sha256'] = source_fingerprint
|
|
results['moli_checkout'] = str(moli)
|
|
results['limo_revision'] = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=limo, text=True).strip()
|
|
results['limo_source_sha256'] = {name: hashlib.sha256((app / 'src' / name).read_bytes()).hexdigest()
|
|
for name in ('main.rs', 'server.rs', 'devtools_probe.rs')}
|
|
results['compiler'] = subprocess.check_output(['rustc', '--version'], cwd=moli, text=True).strip()
|
|
results['cargo_lock_sha256'] = hashlib.sha256((app / 'Cargo.lock').read_bytes()).hexdigest()
|
|
results['moli_patch_sha256'] = hashlib.sha256(subprocess.check_output(['git', 'diff', 'HEAD', '--', 'Cargo.toml', 'moli-types', 'moli-core', 'moli-renderer-v8', 'moli-protocol'], cwd=moli)).hexdigest()
|
|
results['scope'] = 'Compile-time external DevTools command types, dispatch and protocol adapters; V8 Inspector, host evaluation, screenshots and PDF are preserved.'
|
|
results['moli_revision'] = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=moli, text=True).strip()
|
|
results['saved_bytes'] = results['on']['bytes'] - results['off']['bytes']
|
|
results['saved_mib'] = round(results['saved_bytes'] / 2**20, 4)
|
|
results['saved_percent'] = round(100 * results['saved_bytes'] / results['on']['bytes'], 3)
|
|
results['behavior_differences'] = [a['name'] for a, b in zip(results['on']['smoke']['results'], results['off']['smoke']['results'])
|
|
if a['returncode'] != b['returncode'] or a['passed'] != b['passed']]
|
|
(output / 'comparison.json').write_text(json.dumps(results, indent=2))
|
|
if results['behavior_differences']:
|
|
raise SystemExit(f"Runtime regressions: {results['behavior_differences']}; see {output / 'comparison.json'}")
|
|
print(json.dumps({key: results[key] for key in ('moli_revision', 'saved_bytes', 'saved_mib', 'saved_percent', 'behavior_differences')}, indent=2))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|