diff --git a/moli-benchmark/scripts/analyze-executable-resident-pages.mjs b/moli-benchmark/scripts/analyze-executable-resident-pages.mjs new file mode 100644 index 0000000000..00da4b1a87 --- /dev/null +++ b/moli-benchmark/scripts/analyze-executable-resident-pages.mjs @@ -0,0 +1,150 @@ +#!/usr/bin/env node + +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; + +const artifactPath = process.argv[2]; +if (!artifactPath) { + throw new Error('usage: analyze-executable-resident-pages.mjs ARTIFACT.json'); +} + +function executableSegment(binary) { + const output = execFileSync('readelf', ['-lW', binary], { encoding: 'utf8' }); + for (const line of output.split('\n')) { + const fields = line.trim().split(/\s+/); + if (fields[0] !== 'LOAD' || !fields.includes('E')) continue; + return { + offset: Number.parseInt(fields[1], 16), + virtualAddress: Number.parseInt(fields[2], 16), + fileSize: Number.parseInt(fields[4], 16), + }; + } + throw new Error(`${binary}: executable LOAD segment not found`); +} + +function clusterSymbol(name, engine) { + if (/(^| )v8::|^(v8_|Builtins_|Builtins::|icu_|u[a-z]+_\d+|std::__Cr::)/.test(name)) { + return 'V8 / ICU / bundled libc++'; + } + if (/^(aws_lc_|SSL_|CRYPTO_|OPENSSL_|bssl::|ring::)/.test(name)) return 'TLS / crypto'; + if (/^(Curl_|curl_|nghttp2_)/.test(name)) return 'curl / HTTP2'; + if (engine === 'moli') { + if (name.includes('moli_renderer_v8::')) return 'moli-renderer-v8'; + if (name.includes('moli_protocol::')) return 'moli-protocol'; + if (name.includes('moli_core::')) return 'moli-core'; + if (/(moli_fetch|moli_network|moli_tls|moli_cookie)/.test(name)) return 'Moli fetch / network'; + if (/(^|<)(style::|selectors::|cssparser::|servo_arc::)|stylo_/.test(name)) return 'Servo style'; + if (name.includes('taffy::')) return 'Taffy layout'; + if (/(^|<)(tokio::|mio::|futures_|futures::|hyper::|h2::)/.test(name)) return 'Rust async / HTTP'; + if (/(^|<)(std::|core::|alloc::|hashbrown::|regex)/.test(name)) return 'Rust std / collections'; + if (/(^|<)moli_/.test(name)) return 'Other Moli crates'; + } else { + if (name.startsWith('browser.webapi.')) return 'Lightpanda Web APIs'; + if (name.startsWith('browser.js.')) return 'Lightpanda V8 bridge'; + if (name.startsWith('browser.')) return 'Lightpanda browser'; + if (name.startsWith('cdp.')) return 'Lightpanda CDP'; + if (/^(Io\.|json\.|array_|hash_map\.|multi_array_|mem\.|fmt\.|fs\.|debug\.|sort\.|Thread\.)/.test(name)) { + return 'Zig std / generated'; + } + } + return 'Other'; +} + +function add(map, key, bytes) { + map.set(key, (map.get(key) ?? 0) + bytes); +} + +function symbolPrefix(name) { + const rust = name.indexOf('::'); + const zig = name.indexOf('.'); + const end = [rust, zig].filter((value) => value > 0).sort((a, b) => a - b)[0]; + if (end !== undefined) return name.slice(0, end); + const c = name.indexOf('_'); + return c > 0 ? name.slice(0, c) : name.split(/[<( ]/, 1)[0]; +} + +function analyzeRun(run, binary) { + const pageMap = run.executableResidentPageMap; + if (!pageMap) throw new Error(`${run.label}: missing executableResidentPageMap`); + const pageSize = pageMap.mappings[0]?.pageSize ?? 4096; + const presentPages = new Set(); + for (const mapping of pageMap.mappings) { + for (const [start, end] of mapping.presentFilePageRanges) { + for (let page = start; page < end; page += 1) presentPages.add(page); + } + } + + const segment = executableSegment(binary); + const symbols = []; + const nm = execFileSync( + 'nm', + ['-S', '--size-sort', '--demangle', '--defined-only', binary], + { encoding: 'utf8', maxBuffer: 1024 * 1024 * 256 }, + ); + for (const line of nm.split('\n')) { + const match = line.match(/^([0-9a-f]+)\s+([0-9a-f]+)\s+([tTwW])\s+(.*)$/); + if (!match) continue; + const virtualAddress = Number.parseInt(match[1], 16); + const size = Number.parseInt(match[2], 16); + if (size === 0 || virtualAddress < segment.virtualAddress) continue; + const fileStart = segment.offset + virtualAddress - segment.virtualAddress; + const fileEnd = fileStart + size; + if (fileStart >= segment.offset + segment.fileSize) continue; + symbols.push({ + name: match[4], + fileStart, + fileEnd: Math.min(fileEnd, segment.offset + segment.fileSize), + }); + } + + const byCluster = new Map(); + const otherByPrefix = new Map(); + const bySymbol = []; + let symbolResidentBytes = 0; + for (const symbol of symbols) { + let residentBytes = 0; + const firstPage = Math.floor(symbol.fileStart / pageSize); + const lastPage = Math.floor((symbol.fileEnd - 1) / pageSize); + for (let page = firstPage; page <= lastPage; page += 1) { + if (!presentPages.has(page)) continue; + residentBytes += Math.max( + 0, + Math.min(symbol.fileEnd, (page + 1) * pageSize) - Math.max(symbol.fileStart, page * pageSize), + ); + } + if (residentBytes === 0) continue; + symbolResidentBytes += residentBytes; + const cluster = clusterSymbol(symbol.name, run.engine); + add(byCluster, cluster, residentBytes); + if (cluster === 'Other') add(otherByPrefix, symbolPrefix(symbol.name), residentBytes); + bySymbol.push({ name: symbol.name, cluster, residentBytes }); + } + + const residentBytes = presentPages.size * pageSize; + return { + label: run.label, + engine: run.engine, + binary: path.resolve(binary), + residentBytes, + smapsExecutablePssBytes: run.memoryBeforeDiagnostics.buckets.mainExecutable.pss, + symbolResidentBytes, + unattributedBytes: Math.max(0, residentBytes - symbolResidentBytes), + byCluster: [...byCluster.entries()] + .map(([name, bytes]) => ({ name, bytes })) + .sort((left, right) => right.bytes - left.bytes), + otherByPrefix: [...otherByPrefix.entries()] + .map(([name, bytes]) => ({ name, bytes })) + .sort((left, right) => right.bytes - left.bytes) + .slice(0, 50), + topSymbols: bySymbol + .sort((left, right) => right.residentBytes - left.residentBytes) + .slice(0, 50), + }; +} + +const artifact = JSON.parse(fs.readFileSync(artifactPath, 'utf8')); +for (const run of artifact.runs) { + const binary = run.engine === 'moli' ? artifact.binaries.moli.path : artifact.binaries.lightpanda.path; + process.stdout.write(`${JSON.stringify(analyzeRun(run, binary))}\n`); +} diff --git a/moli-benchmark/scripts/analyze-jemalloc-profile.mjs b/moli-benchmark/scripts/analyze-jemalloc-profile.mjs new file mode 100644 index 0000000000..ad0cf771d4 --- /dev/null +++ b/moli-benchmark/scripts/analyze-jemalloc-profile.mjs @@ -0,0 +1,364 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; + +function parseArgs(argv) { + const args = { binary: null, profiles: [] }; + for (let index = 0; index < argv.length; index += 1) { + const value = argv[index]; + if (value === '--binary') { + args.binary = path.resolve(argv[++index]); + } else { + args.profiles.push(path.resolve(value)); + } + } + if (!args.binary || args.profiles.length === 0) { + throw new Error('usage: analyze-jemalloc-profile.mjs --binary BIN PROFILE...'); + } + return args; +} + +function parseMapping(line) { + const match = line.match( + /^([0-9a-f]+)-([0-9a-f]+)\s+(\S+)\s+([0-9a-f]+)\s+\S+\s+\d+\s*(.*)$/, + ); + if (!match) { + return null; + } + return { + start: BigInt(`0x${match[1]}`), + end: BigInt(`0x${match[2]}`), + permissions: match[3], + offset: BigInt(`0x${match[4]}`), + mappedPath: match[5], + }; +} + +function adjustedSample(rawObjects, rawBytes, interval) { + const meanObjectSize = rawBytes / rawObjects; + const scale = 1 / -Math.expm1(-meanObjectSize / interval); + return { + adjustedBytes: rawBytes * scale, + adjustedObjects: rawObjects * scale, + scale, + }; +} + +function parseProfile(profilePath) { + const lines = fs.readFileSync(profilePath, 'utf8').split('\n'); + const intervalMatch = lines[0].match(/^heap_v2\/(\d+)$/); + if (!intervalMatch) { + throw new Error(`${profilePath}: unsupported jemalloc profile header ${lines[0]}`); + } + const interval = Number.parseInt(intervalMatch[1], 10); + const mappingsStart = lines.indexOf('MAPPED_LIBRARIES:'); + if (mappingsStart < 0) { + throw new Error(`${profilePath}: missing MAPPED_LIBRARIES section`); + } + + const headerMatch = lines + .slice(1, mappingsStart) + .find((line) => /^\s+t\*:/.test(line)) + ?.match(/^\s+t\*:\s+(\d+):\s+(\d+)/); + const samples = []; + for (let index = 1; index < mappingsStart; index += 1) { + if (!lines[index].startsWith('@ ')) { + continue; + } + const frames = lines[index].slice(2).trim().split(/\s+/).filter(Boolean); + const totalsMatch = lines[index + 1]?.match(/^\s+t\*:\s+(\d+):\s+(\d+)/); + if (!totalsMatch) { + throw new Error(`${profilePath}:${index + 2}: stack is missing its t* sample`); + } + const rawObjects = Number.parseInt(totalsMatch[1], 10); + const rawBytes = Number.parseInt(totalsMatch[2], 10); + if (rawObjects === 0 || rawBytes === 0) { + continue; + } + samples.push({ + frames, + rawObjects, + rawBytes, + ...adjustedSample(rawObjects, rawBytes, interval), + }); + } + + const mappings = lines + .slice(mappingsStart + 1) + .map(parseMapping) + .filter((mapping) => mapping !== null); + return { + profilePath, + interval, + header: headerMatch + ? { + sampledObjects: Number.parseInt(headerMatch[1], 10), + sampledBytes: Number.parseInt(headerMatch[2], 10), + } + : null, + samples, + mappings, + }; +} + +function mappingForAddress(mappings, address) { + const numericAddress = BigInt(address); + return mappings.find( + (mapping) => numericAddress >= mapping.start && numericAddress < mapping.end, + ); +} + +function findMainMapping(profile) { + const executablePaths = new Set( + profile.mappings + .filter((mapping) => mapping.permissions.includes('x') && mapping.mappedPath.startsWith('/')) + .map((mapping) => mapping.mappedPath), + ); + const pathScores = new Map([...executablePaths].map((mappedPath) => [mappedPath, 0])); + for (const sample of profile.samples) { + for (const frame of sample.frames) { + const mapping = mappingForAddress(profile.mappings, frame); + if (mapping && pathScores.has(mapping.mappedPath)) { + pathScores.set( + mapping.mappedPath, + pathScores.get(mapping.mappedPath) + sample.adjustedBytes, + ); + } + } + } + const [mainPath] = [...pathScores.entries()] + .sort((left, right) => right[1] - left[1])[0] ?? []; + if (!mainPath) { + throw new Error(`${profile.profilePath}: could not identify the main executable mapping`); + } + const baseMapping = profile.mappings.find( + (mapping) => mapping.mappedPath === mainPath && mapping.offset === 0n, + ); + if (!baseMapping) { + throw new Error(`${profile.profilePath}: main executable has no offset-zero mapping`); + } + return { mappedPath: mainPath, loadBias: baseMapping.start }; +} + +function normalizeMainAddress(address, mainMapping) { + const numericAddress = BigInt(address); + const relativeAddress = numericAddress - mainMapping.loadBias; + const callerAddress = relativeAddress > 0n ? relativeAddress - 1n : relativeAddress; + return `0x${callerAddress.toString(16)}`; +} + +function symbolize(binary, profiles) { + const normalizedAddresses = new Set(); + for (const profile of profiles) { + for (const sample of profile.samples) { + for (const address of sample.frames) { + const mapping = mappingForAddress(profile.mappings, address); + if (mapping?.mappedPath === profile.mainMapping.mappedPath) { + normalizedAddresses.add(normalizeMainAddress(address, profile.mainMapping)); + } + } + } + } + + const result = new Map(); + const unique = [...normalizedAddresses]; + for (let start = 0; start < unique.length; start += 500) { + const batch = unique.slice(start, start + 500); + const child = spawnSync('addr2line', ['-Cfpe', binary, ...batch], { + encoding: 'utf8', + maxBuffer: 32 * 1024 * 1024, + }); + if (child.status !== 0) { + throw new Error(`addr2line failed: ${child.stderr}`); + } + const lines = child.stdout.trimEnd().split('\n'); + if (lines.length !== batch.length) { + throw new Error(`addr2line returned ${lines.length} lines for ${batch.length} addresses`); + } + for (let index = 0; index < batch.length; index += 1) { + result.set(batch[index], lines[index]); + } + } + return result; +} + +function describeFrame(profile, symbols, address) { + const mapping = mappingForAddress(profile.mappings, address); + if (!mapping) { + return `${address} at (unmapped)`; + } + if (mapping.mappedPath === profile.mainMapping.mappedPath) { + const normalized = normalizeMainAddress(address, profile.mainMapping); + return symbols.get(normalized) ?? `${normalized} at ??`; + } + const location = BigInt(address) - mapping.start + mapping.offset; + const label = mapping.mappedPath + ? path.basename(mapping.mappedPath) + : `(anonymous ${mapping.permissions})`; + return `${label}+0x${location.toString(16)}`; +} + +function isAllocatorPlumbing(symbol) { + return symbol.includes('prof_backtrace') + || symbol.includes('je_prof_') + || symbol.includes('prof_alloc_') + || symbol.includes('prof_sample_') + || symbol.includes('prof_tctx_') + || symbol.includes('jemalloc') + || symbol.includes('_rjem_') + || symbol.includes('tikv_jemalloc') + || symbol.startsWith('operator new(') + || symbol.includes('OPENSSL_malloc') + || symbol.includes('OPENSSL_realloc') + || symbol.includes('mallocx') + || symbol.includes('imalloc') + || symbol.includes('ialloc') + || symbol.includes('arena_malloc') + || symbol.includes('arena_ralloc') + || symbol.includes('arena_dalloc') + || symbol.includes('tcache_alloc') + || symbol.includes('__rust_alloc') + || symbol.includes('__rdl_alloc') + || symbol.includes('alloc::alloc::') + || symbol.includes('alloc::raw_vec::RawVecInner') + || symbol.includes('alloc::raw_vec::RawVec<') + || symbol.includes('hashbrown::raw::RawTableInner::fallible_with_capacity') + || symbol.includes('hashbrown::raw::RawTable::reserve_rehash') + || symbol.includes('smallvec::SmallVec::try_grow') + || symbol.includes('thin_vec::ThinVec::reserve') + || symbol.includes('alloc::raw_vec::RawVecInner::try_allocate_in') + || symbol.includes('Allocator::allocate') + || symbol.includes('Allocator::grow'); +} + +function aggregate(samples, keyForSample) { + const totals = new Map(); + for (const sample of samples) { + const key = keyForSample(sample); + const current = totals.get(key) ?? { + adjustedBytes: 0, + adjustedObjects: 0, + sampledBytes: 0, + sampledObjects: 0, + stacks: 0, + }; + current.adjustedBytes += sample.adjustedBytes; + current.adjustedObjects += sample.adjustedObjects; + current.sampledBytes += sample.rawBytes; + current.sampledObjects += sample.rawObjects; + current.stacks += 1; + totals.set(key, current); + } + return [...totals.entries()] + .map(([key, totalsForKey]) => ({ key, ...totalsForKey })) + .sort((left, right) => right.adjustedBytes - left.adjustedBytes); +} + +function categoryForSample(sample) { + const stack = sample.consumerStack.join('\n'); + const matches = (pattern) => pattern.test(stack); + + if (matches(/SubresourceResponseBody|SubresourceNetworkRecord|ScriptNetworkOutput|TargetNetworkOutput|TargetSubresource|CapturedBodyWriter|renderer_network_observation|response::ResponseBody/)) { + return 'network observations / retained response bodies'; + } + if (matches(/style::|selectors::|cssparser::|Stylesheet|stylesheet|CSSRule|css_rule|css_stylesheet|style_engine/)) { + return 'CSS / Stylo / CSSOM'; + } + if (matches(/ModuleScript|module_script|module_runtime|script_planning|ScriptPreload|ImportMap|import_map|decode_classic_script|external_script_source/)) { + return 'scripts / modules / import maps'; + } + if (matches(/moli_dom::|NativeDom|native_bridge|document_runtime|html5ever|moli_parser/)) { + return 'DOM / bindings / parser'; + } + if (matches(/v8::|Builtins_|icu_|_uhash_|cppgc/)) { + return 'V8 / ICU native'; + } + if (matches(/aws_lc_|OPENSSL_|SSL_|bssl::|CRYPTO_/)) { + return 'TLS / crypto'; + } + if (matches(/Curl_|curl::|nghttp2_|http2\.c/)) { + return 'curl / HTTP2'; + } + if (matches(/tokio::|mio::|futures::|moli_local_executor|runtime::owner/)) { + return 'async runtime / scheduling'; + } + if (matches(/moli_protocol::|moli_protocol_server::/)) { + return 'protocol / CDP'; + } + if (matches(/moli_/)) { + return 'other Moli'; + } + return 'other'; +} + +function analyze(profile, symbols) { + const samples = profile.samples.map((sample) => { + const stack = sample.frames.map((address) => describeFrame(profile, symbols, address)); + const consumerStack = stack.filter((symbol) => !isAllocatorPlumbing(symbol)); + return { + ...sample, + stack, + consumer: consumerStack[0] ?? stack[0] ?? '(unknown)', + consumerStack, + }; + }); + const adjustedBytes = samples.reduce((total, sample) => total + sample.adjustedBytes, 0); + const adjustedObjects = samples.reduce((total, sample) => total + sample.adjustedObjects, 0); + const sampledBytes = samples.reduce((total, sample) => total + sample.rawBytes, 0); + const sampledObjects = samples.reduce((total, sample) => total + sample.rawObjects, 0); + const topCategories = aggregate(samples, categoryForSample).map((category) => ({ + ...category, + adjustedFraction: adjustedBytes === 0 ? 0 : category.adjustedBytes / adjustedBytes, + })); + + return { + profile: profile.profilePath, + interval: profile.interval, + mainMapping: { + path: profile.mainMapping.mappedPath, + loadBias: `0x${profile.mainMapping.loadBias.toString(16)}`, + }, + header: profile.header, + sampling: { + liveStackRecords: samples.length, + sampledBytes, + sampledObjects, + adjustedBytes, + adjustedObjects, + headerBytesMatch: profile.header?.sampledBytes === sampledBytes, + headerObjectsMatch: profile.header?.sampledObjects === sampledObjects, + }, + topCategories, + topCategoryConsumers: aggregate( + samples, + (sample) => `${categoryForSample(sample)}\t${sample.consumer}`, + ).slice(0, 100), + topConsumers: aggregate(samples, (sample) => sample.consumer).slice(0, 50), + topConsumerStacks: aggregate( + samples, + (sample) => sample.consumerStack.slice(0, 10).join(' <- ') || sample.consumer, + ).slice(0, 50), + largestSampledStacks: samples + .sort((left, right) => right.adjustedBytes - left.adjustedBytes) + .slice(0, 100) + .map((sample) => ({ + sampledObjects: sample.rawObjects, + sampledBytes: sample.rawBytes, + adjustedObjects: sample.adjustedObjects, + adjustedBytes: sample.adjustedBytes, + scale: sample.scale, + consumer: sample.consumer, + stack: sample.consumerStack.slice(0, 16), + })), + }; +} + +const args = parseArgs(process.argv.slice(2)); +const profiles = args.profiles.map(parseProfile); +for (const profile of profiles) { + profile.mainMapping = findMainMapping(profile); +} +const symbols = symbolize(args.binary, profiles); +console.log(JSON.stringify(profiles.map((profile) => analyze(profile, symbols)), null, 2)); diff --git a/moli-benchmark/scripts/analyze-lightpanda-pa-profile.mjs b/moli-benchmark/scripts/analyze-lightpanda-pa-profile.mjs new file mode 100644 index 0000000000..068394047b --- /dev/null +++ b/moli-benchmark/scripts/analyze-lightpanda-pa-profile.mjs @@ -0,0 +1,213 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; + +function parseArgs(argv) { + const args = { binary: null, profiles: [] }; + for (let index = 0; index < argv.length; index += 1) { + const value = argv[index]; + if (value === '--binary') { + args.binary = path.resolve(argv[++index]); + } else { + args.profiles.push(path.resolve(value)); + } + } + if (!args.binary || args.profiles.length === 0) { + throw new Error('usage: analyze-lightpanda-pa-profile.mjs --binary BIN PROFILE...'); + } + return args; +} + +function parseFields(line) { + return Object.fromEntries( + [...line.matchAll(/(\w+)=([^\s]+)/g)].map((match) => { + const raw = match[2]; + const value = raw.startsWith('0x') ? raw : Number.parseInt(raw, 10); + return [match[1], value]; + }), + ); +} + +function parseProfile(profilePath) { + const lines = fs.readFileSync(profilePath, 'utf8').trim().split('\n'); + const header = parseFields(lines.find((line) => line.startsWith('H ')) ?? ''); + const histogram = lines + .filter((line) => line.startsWith('B ')) + .map(parseFields); + const samples = lines + .filter((line) => line.startsWith('S ')) + .map((line) => { + const fields = parseFields(line); + return { + size: fields.size, + weight: fields.weight, + key: fields.key, + depth: fields.depth, + frames: [...line.matchAll(/\s(0x[0-9a-f]+)(?=\s|$)/g)] + .map((match) => match[1]), + }; + }); + return { profilePath, header, histogram, samples }; +} + +function symbolize(binary, addresses) { + const result = new Map(); + const unique = [...new Set(addresses)]; + for (let start = 0; start < unique.length; start += 500) { + const batch = unique.slice(start, start + 500); + const child = spawnSync('addr2line', ['-Cfpe', binary, ...batch], { + encoding: 'utf8', + maxBuffer: 16 * 1024 * 1024, + }); + if (child.status !== 0) { + throw new Error(`addr2line failed: ${child.stderr}`); + } + const lines = child.stdout.trimEnd().split('\n'); + if (lines.length !== batch.length) { + throw new Error(`addr2line returned ${lines.length} lines for ${batch.length} addresses`); + } + for (let index = 0; index < batch.length; index += 1) { + result.set(batch[index], lines[index]); + } + } + return result; +} + +function isProfilerFrame(symbol) { + return symbol.includes('pa_profile.cpp') + || symbol.includes('RecordAllocation') + || symbol.includes('DispatchAlloc'); +} + +function isAllocatorPlumbing(symbol) { + return isProfilerFrame(symbol) + || /^(malloc|calloc|realloc|memalign|aligned_alloc|operator new)/.test(symbol) + || symbol.includes('allocator_shim::') + || symbol.includes('Arena.rawAlloc') + || symbol.includes('ArenaAllocator.alloc') + || symbol.includes('mem.Allocator.rawAlloc') + || symbol.includes('mem.Allocator.allocBytesWithAlignment') + || symbol.includes('mem.Allocator.allocWithOptions'); +} + +function aggregate(samples, keyForSample) { + const totals = new Map(); + for (const sample of samples) { + const key = keyForSample(sample); + const current = totals.get(key) ?? { estimatedBytes: 0, sampledBytes: 0, samples: 0 }; + current.estimatedBytes += sample.size * sample.weight; + current.sampledBytes += sample.size; + current.samples += 1; + totals.set(key, current); + } + return [...totals.entries()] + .map(([key, totalsForKey]) => ({ key, ...totalsForKey })) + .sort((left, right) => right.estimatedBytes - left.estimatedBytes); +} + +function categoryForSample(sample) { + const stack = sample.consumerStack.join('\n'); + const matches = (pattern) => pattern.test(stack); + + if (matches(/Notification\.dispatch|HttpClient\.Transfer\.deliver|HttpClient\.drainInbox/)) { + return 'network response buffers / notifications'; + } + if (matches(/Timers\.|setTimeout|setInterval/)) return 'timers'; + if (matches(/FinalizerCallback|mapZigInstanceToJs|browser\.js\.Local/)) { + return 'JS wrapper finalizers'; + } + if (matches(/StyleManager|loadExternalStylesheet|stylesheet|CSSRule|cssparser|style::|selectors::/)) { + return 'CSS / Stylo / CSSOM'; + } + if (matches(/ScriptManager|ImportMap|resolveModule|browser\.js\.Module/)) { + return 'scripts / modules / import maps'; + } + if (matches(/node_factory|browser\.parser|slab\.Slab|html5ever/)) { + return 'DOM / bindings / parser'; + } + if (matches(/ArenaPool|ArenaAllocator/)) return 'arena pools'; + if (matches(/v8::|Builtins_|icu_|_uhash_|cppgc/)) return 'V8 / ICU native'; + if (matches(/OPENSSL_|SSL_|CRYPTO_|bssl::/)) return 'TLS / crypto'; + if (matches(/Curl_|nghttp2_|libcurl/)) return 'curl / HTTP2'; + if (matches(/browser\.|cdp\./)) return 'other Lightpanda browser / CDP'; + return 'other'; +} + +function analyze(profile, symbols) { + const samples = profile.samples.map((sample) => { + const stack = sample.frames.map((address) => symbols.get(address) ?? `${address} at ??`); + const externalStack = stack.filter((symbol) => !isProfilerFrame(symbol)); + const consumerStack = stack.filter((symbol) => !isAllocatorPlumbing(symbol)); + return { + ...sample, + stack, + leaf: externalStack[0] ?? '(profiler only)', + consumer: consumerStack[0] ?? externalStack[0] ?? '(unknown)', + consumerStack, + }; + }); + const estimatedRequestedBytes = samples.reduce( + (total, sample) => total + sample.size * sample.weight, + 0, + ); + const sampledRequestedBytes = samples.reduce((total, sample) => total + sample.size, 0); + const exactLargeSamples = samples.filter((sample) => sample.weight === 1); + const exactLargeRequestedBytes = exactLargeSamples.reduce( + (total, sample) => total + sample.size, + 0, + ); + const histogramLargeRequestedBytes = profile.histogram + .filter((bin) => bin.bin >= 17) + .reduce((total, bin) => total + bin.requested_bytes, 0); + const topCategories = aggregate(samples, categoryForSample).map((category) => ({ + ...category, + estimatedFraction: estimatedRequestedBytes === 0 + ? 0 + : category.estimatedBytes / estimatedRequestedBytes, + })); + + return { + profile: profile.profilePath, + header: profile.header, + histogram: profile.histogram, + sampling: { + liveSamples: samples.length, + sampledRequestedBytes, + estimatedRequestedBytes, + estimateToExactRatio: estimatedRequestedBytes / profile.header.live_requested_bytes, + exactLargeSamples: exactLargeSamples.length, + exactLargeRequestedBytes, + histogramLargeRequestedBytes, + exactLargeMatchesHistogram: exactLargeRequestedBytes === histogramLargeRequestedBytes, + }, + topCategories, + topCategoryConsumers: aggregate( + samples, + (sample) => `${categoryForSample(sample)}\t${sample.consumer}`, + ).slice(0, 100), + topConsumers: aggregate(samples, (sample) => sample.consumer).slice(0, 40), + topConsumerStacks: aggregate( + samples, + (sample) => sample.consumerStack.slice(0, 8).join(' <- ') || sample.consumer, + ).slice(0, 40), + largestLiveAllocations: exactLargeSamples + .sort((left, right) => right.size - left.size) + .slice(0, 100) + .map((sample) => ({ + size: sample.size, + key: sample.key, + consumer: sample.consumer, + stack: sample.consumerStack.slice(0, 12), + })), + }; +} + +const args = parseArgs(process.argv.slice(2)); +const profiles = args.profiles.map(parseProfile); +const symbols = symbolize( + args.binary, + profiles.flatMap((profile) => profile.samples.flatMap((sample) => sample.frames)), +); +console.log(JSON.stringify(profiles.map((profile) => analyze(profile, symbols)), null, 2)); diff --git a/moli-benchmark/scripts/analyze-v8-heapsnapshot.mjs b/moli-benchmark/scripts/analyze-v8-heapsnapshot.mjs new file mode 100644 index 0000000000..2fc5a6be40 --- /dev/null +++ b/moli-benchmark/scripts/analyze-v8-heapsnapshot.mjs @@ -0,0 +1,71 @@ +#!/usr/bin/env node + +import fs from 'node:fs'; +import path from 'node:path'; + +const inputs = process.argv.slice(2); +if (inputs.length === 0) { + throw new Error('usage: analyze-v8-heapsnapshot.mjs SNAPSHOT...'); +} + +function topEntries(map, limit) { + return [...map.entries()] + .map(([name, value]) => ({ name, ...value })) + .sort((left, right) => right.selfSize - left.selfSize || right.count - left.count) + .slice(0, limit); +} + +function add(map, key, selfSize) { + const value = map.get(key) ?? { count: 0, selfSize: 0 }; + value.count += 1; + value.selfSize += selfSize; + map.set(key, value); +} + +function analyze(input) { + const snapshot = JSON.parse(fs.readFileSync(input, 'utf8')); + const fields = snapshot.snapshot.meta.node_fields; + const fieldCount = fields.length; + const typeIndex = fields.indexOf('type'); + const nameIndex = fields.indexOf('name'); + const selfSizeIndex = fields.indexOf('self_size'); + if ([typeIndex, nameIndex, selfSizeIndex].includes(-1)) { + throw new Error(`${input}: unsupported node_fields`); + } + if (snapshot.nodes.length % fieldCount !== 0) { + throw new Error(`${input}: truncated nodes array`); + } + + const nodeTypes = snapshot.snapshot.meta.node_types[typeIndex]; + const byType = new Map(); + const byName = new Map(); + const byTypeAndName = new Map(); + let selfSize = 0; + + for (let offset = 0; offset < snapshot.nodes.length; offset += fieldCount) { + const type = nodeTypes[snapshot.nodes[offset + typeIndex]]; + const name = snapshot.strings[snapshot.nodes[offset + nameIndex]]; + const size = snapshot.nodes[offset + selfSizeIndex]; + selfSize += size; + add(byType, type, size); + add(byName, name, size); + add(byTypeAndName, `${type}\t${name}`, size); + } + + return { + path: path.resolve(input), + bytes: fs.statSync(input).size, + nodeCount: snapshot.nodes.length / fieldCount, + selfSize, + byType: topEntries(byType, byType.size), + topNames: topEntries(byName, 60), + topTypeNames: topEntries(byTypeAndName, 100).map((entry) => { + const [type, name] = entry.name.split('\t', 2); + return { type, name, count: entry.count, selfSize: entry.selfSize }; + }), + }; +} + +for (const input of inputs) { + process.stdout.write(`${JSON.stringify(analyze(input))}\n`); +}