mirror of
https://github.com/lexmount/moli.git
synced 2026-10-05 00:00:55 +00:00
chore(benchmark): add memory attribution analyzers
This commit is contained in:
ldm0
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
const artifactPath = process.argv[2];
|
||||
if (!artifactPath) {
|
||||
throw new Error('usage: analyze-executable-resident-pages.mjs ARTIFACT.json');
|
||||
}
|
||||
|
||||
function executableSegment(binary) {
|
||||
const output = execFileSync('readelf', ['-lW', binary], { encoding: 'utf8' });
|
||||
for (const line of output.split('\n')) {
|
||||
const fields = line.trim().split(/\s+/);
|
||||
if (fields[0] !== 'LOAD' || !fields.includes('E')) continue;
|
||||
return {
|
||||
offset: Number.parseInt(fields[1], 16),
|
||||
virtualAddress: Number.parseInt(fields[2], 16),
|
||||
fileSize: Number.parseInt(fields[4], 16),
|
||||
};
|
||||
}
|
||||
throw new Error(`${binary}: executable LOAD segment not found`);
|
||||
}
|
||||
|
||||
function clusterSymbol(name, engine) {
|
||||
if (/(^| )v8::|^(v8_|Builtins_|Builtins::|icu_|u[a-z]+_\d+|std::__Cr::)/.test(name)) {
|
||||
return 'V8 / ICU / bundled libc++';
|
||||
}
|
||||
if (/^(aws_lc_|SSL_|CRYPTO_|OPENSSL_|bssl::|ring::)/.test(name)) return 'TLS / crypto';
|
||||
if (/^(Curl_|curl_|nghttp2_)/.test(name)) return 'curl / HTTP2';
|
||||
if (engine === 'moli') {
|
||||
if (name.includes('moli_renderer_v8::')) return 'moli-renderer-v8';
|
||||
if (name.includes('moli_protocol::')) return 'moli-protocol';
|
||||
if (name.includes('moli_core::')) return 'moli-core';
|
||||
if (/(moli_fetch|moli_network|moli_tls|moli_cookie)/.test(name)) return 'Moli fetch / network';
|
||||
if (/(^|<)(style::|selectors::|cssparser::|servo_arc::)|stylo_/.test(name)) return 'Servo style';
|
||||
if (name.includes('taffy::')) return 'Taffy layout';
|
||||
if (/(^|<)(tokio::|mio::|futures_|futures::|hyper::|h2::)/.test(name)) return 'Rust async / HTTP';
|
||||
if (/(^|<)(std::|core::|alloc::|hashbrown::|regex)/.test(name)) return 'Rust std / collections';
|
||||
if (/(^|<)moli_/.test(name)) return 'Other Moli crates';
|
||||
} else {
|
||||
if (name.startsWith('browser.webapi.')) return 'Lightpanda Web APIs';
|
||||
if (name.startsWith('browser.js.')) return 'Lightpanda V8 bridge';
|
||||
if (name.startsWith('browser.')) return 'Lightpanda browser';
|
||||
if (name.startsWith('cdp.')) return 'Lightpanda CDP';
|
||||
if (/^(Io\.|json\.|array_|hash_map\.|multi_array_|mem\.|fmt\.|fs\.|debug\.|sort\.|Thread\.)/.test(name)) {
|
||||
return 'Zig std / generated';
|
||||
}
|
||||
}
|
||||
return 'Other';
|
||||
}
|
||||
|
||||
function add(map, key, bytes) {
|
||||
map.set(key, (map.get(key) ?? 0) + bytes);
|
||||
}
|
||||
|
||||
function symbolPrefix(name) {
|
||||
const rust = name.indexOf('::');
|
||||
const zig = name.indexOf('.');
|
||||
const end = [rust, zig].filter((value) => value > 0).sort((a, b) => a - b)[0];
|
||||
if (end !== undefined) return name.slice(0, end);
|
||||
const c = name.indexOf('_');
|
||||
return c > 0 ? name.slice(0, c) : name.split(/[<( ]/, 1)[0];
|
||||
}
|
||||
|
||||
function analyzeRun(run, binary) {
|
||||
const pageMap = run.executableResidentPageMap;
|
||||
if (!pageMap) throw new Error(`${run.label}: missing executableResidentPageMap`);
|
||||
const pageSize = pageMap.mappings[0]?.pageSize ?? 4096;
|
||||
const presentPages = new Set();
|
||||
for (const mapping of pageMap.mappings) {
|
||||
for (const [start, end] of mapping.presentFilePageRanges) {
|
||||
for (let page = start; page < end; page += 1) presentPages.add(page);
|
||||
}
|
||||
}
|
||||
|
||||
const segment = executableSegment(binary);
|
||||
const symbols = [];
|
||||
const nm = execFileSync(
|
||||
'nm',
|
||||
['-S', '--size-sort', '--demangle', '--defined-only', binary],
|
||||
{ encoding: 'utf8', maxBuffer: 1024 * 1024 * 256 },
|
||||
);
|
||||
for (const line of nm.split('\n')) {
|
||||
const match = line.match(/^([0-9a-f]+)\s+([0-9a-f]+)\s+([tTwW])\s+(.*)$/);
|
||||
if (!match) continue;
|
||||
const virtualAddress = Number.parseInt(match[1], 16);
|
||||
const size = Number.parseInt(match[2], 16);
|
||||
if (size === 0 || virtualAddress < segment.virtualAddress) continue;
|
||||
const fileStart = segment.offset + virtualAddress - segment.virtualAddress;
|
||||
const fileEnd = fileStart + size;
|
||||
if (fileStart >= segment.offset + segment.fileSize) continue;
|
||||
symbols.push({
|
||||
name: match[4],
|
||||
fileStart,
|
||||
fileEnd: Math.min(fileEnd, segment.offset + segment.fileSize),
|
||||
});
|
||||
}
|
||||
|
||||
const byCluster = new Map();
|
||||
const otherByPrefix = new Map();
|
||||
const bySymbol = [];
|
||||
let symbolResidentBytes = 0;
|
||||
for (const symbol of symbols) {
|
||||
let residentBytes = 0;
|
||||
const firstPage = Math.floor(symbol.fileStart / pageSize);
|
||||
const lastPage = Math.floor((symbol.fileEnd - 1) / pageSize);
|
||||
for (let page = firstPage; page <= lastPage; page += 1) {
|
||||
if (!presentPages.has(page)) continue;
|
||||
residentBytes += Math.max(
|
||||
0,
|
||||
Math.min(symbol.fileEnd, (page + 1) * pageSize) - Math.max(symbol.fileStart, page * pageSize),
|
||||
);
|
||||
}
|
||||
if (residentBytes === 0) continue;
|
||||
symbolResidentBytes += residentBytes;
|
||||
const cluster = clusterSymbol(symbol.name, run.engine);
|
||||
add(byCluster, cluster, residentBytes);
|
||||
if (cluster === 'Other') add(otherByPrefix, symbolPrefix(symbol.name), residentBytes);
|
||||
bySymbol.push({ name: symbol.name, cluster, residentBytes });
|
||||
}
|
||||
|
||||
const residentBytes = presentPages.size * pageSize;
|
||||
return {
|
||||
label: run.label,
|
||||
engine: run.engine,
|
||||
binary: path.resolve(binary),
|
||||
residentBytes,
|
||||
smapsExecutablePssBytes: run.memoryBeforeDiagnostics.buckets.mainExecutable.pss,
|
||||
symbolResidentBytes,
|
||||
unattributedBytes: Math.max(0, residentBytes - symbolResidentBytes),
|
||||
byCluster: [...byCluster.entries()]
|
||||
.map(([name, bytes]) => ({ name, bytes }))
|
||||
.sort((left, right) => right.bytes - left.bytes),
|
||||
otherByPrefix: [...otherByPrefix.entries()]
|
||||
.map(([name, bytes]) => ({ name, bytes }))
|
||||
.sort((left, right) => right.bytes - left.bytes)
|
||||
.slice(0, 50),
|
||||
topSymbols: bySymbol
|
||||
.sort((left, right) => right.residentBytes - left.residentBytes)
|
||||
.slice(0, 50),
|
||||
};
|
||||
}
|
||||
|
||||
const artifact = JSON.parse(fs.readFileSync(artifactPath, 'utf8'));
|
||||
for (const run of artifact.runs) {
|
||||
const binary = run.engine === 'moli' ? artifact.binaries.moli.path : artifact.binaries.lightpanda.path;
|
||||
process.stdout.write(`${JSON.stringify(analyzeRun(run, binary))}\n`);
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = { binary: null, profiles: [] };
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const value = argv[index];
|
||||
if (value === '--binary') {
|
||||
args.binary = path.resolve(argv[++index]);
|
||||
} else {
|
||||
args.profiles.push(path.resolve(value));
|
||||
}
|
||||
}
|
||||
if (!args.binary || args.profiles.length === 0) {
|
||||
throw new Error('usage: analyze-jemalloc-profile.mjs --binary BIN PROFILE...');
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function parseMapping(line) {
|
||||
const match = line.match(
|
||||
/^([0-9a-f]+)-([0-9a-f]+)\s+(\S+)\s+([0-9a-f]+)\s+\S+\s+\d+\s*(.*)$/,
|
||||
);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
start: BigInt(`0x${match[1]}`),
|
||||
end: BigInt(`0x${match[2]}`),
|
||||
permissions: match[3],
|
||||
offset: BigInt(`0x${match[4]}`),
|
||||
mappedPath: match[5],
|
||||
};
|
||||
}
|
||||
|
||||
function adjustedSample(rawObjects, rawBytes, interval) {
|
||||
const meanObjectSize = rawBytes / rawObjects;
|
||||
const scale = 1 / -Math.expm1(-meanObjectSize / interval);
|
||||
return {
|
||||
adjustedBytes: rawBytes * scale,
|
||||
adjustedObjects: rawObjects * scale,
|
||||
scale,
|
||||
};
|
||||
}
|
||||
|
||||
function parseProfile(profilePath) {
|
||||
const lines = fs.readFileSync(profilePath, 'utf8').split('\n');
|
||||
const intervalMatch = lines[0].match(/^heap_v2\/(\d+)$/);
|
||||
if (!intervalMatch) {
|
||||
throw new Error(`${profilePath}: unsupported jemalloc profile header ${lines[0]}`);
|
||||
}
|
||||
const interval = Number.parseInt(intervalMatch[1], 10);
|
||||
const mappingsStart = lines.indexOf('MAPPED_LIBRARIES:');
|
||||
if (mappingsStart < 0) {
|
||||
throw new Error(`${profilePath}: missing MAPPED_LIBRARIES section`);
|
||||
}
|
||||
|
||||
const headerMatch = lines
|
||||
.slice(1, mappingsStart)
|
||||
.find((line) => /^\s+t\*:/.test(line))
|
||||
?.match(/^\s+t\*:\s+(\d+):\s+(\d+)/);
|
||||
const samples = [];
|
||||
for (let index = 1; index < mappingsStart; index += 1) {
|
||||
if (!lines[index].startsWith('@ ')) {
|
||||
continue;
|
||||
}
|
||||
const frames = lines[index].slice(2).trim().split(/\s+/).filter(Boolean);
|
||||
const totalsMatch = lines[index + 1]?.match(/^\s+t\*:\s+(\d+):\s+(\d+)/);
|
||||
if (!totalsMatch) {
|
||||
throw new Error(`${profilePath}:${index + 2}: stack is missing its t* sample`);
|
||||
}
|
||||
const rawObjects = Number.parseInt(totalsMatch[1], 10);
|
||||
const rawBytes = Number.parseInt(totalsMatch[2], 10);
|
||||
if (rawObjects === 0 || rawBytes === 0) {
|
||||
continue;
|
||||
}
|
||||
samples.push({
|
||||
frames,
|
||||
rawObjects,
|
||||
rawBytes,
|
||||
...adjustedSample(rawObjects, rawBytes, interval),
|
||||
});
|
||||
}
|
||||
|
||||
const mappings = lines
|
||||
.slice(mappingsStart + 1)
|
||||
.map(parseMapping)
|
||||
.filter((mapping) => mapping !== null);
|
||||
return {
|
||||
profilePath,
|
||||
interval,
|
||||
header: headerMatch
|
||||
? {
|
||||
sampledObjects: Number.parseInt(headerMatch[1], 10),
|
||||
sampledBytes: Number.parseInt(headerMatch[2], 10),
|
||||
}
|
||||
: null,
|
||||
samples,
|
||||
mappings,
|
||||
};
|
||||
}
|
||||
|
||||
function mappingForAddress(mappings, address) {
|
||||
const numericAddress = BigInt(address);
|
||||
return mappings.find(
|
||||
(mapping) => numericAddress >= mapping.start && numericAddress < mapping.end,
|
||||
);
|
||||
}
|
||||
|
||||
function findMainMapping(profile) {
|
||||
const executablePaths = new Set(
|
||||
profile.mappings
|
||||
.filter((mapping) => mapping.permissions.includes('x') && mapping.mappedPath.startsWith('/'))
|
||||
.map((mapping) => mapping.mappedPath),
|
||||
);
|
||||
const pathScores = new Map([...executablePaths].map((mappedPath) => [mappedPath, 0]));
|
||||
for (const sample of profile.samples) {
|
||||
for (const frame of sample.frames) {
|
||||
const mapping = mappingForAddress(profile.mappings, frame);
|
||||
if (mapping && pathScores.has(mapping.mappedPath)) {
|
||||
pathScores.set(
|
||||
mapping.mappedPath,
|
||||
pathScores.get(mapping.mappedPath) + sample.adjustedBytes,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
const [mainPath] = [...pathScores.entries()]
|
||||
.sort((left, right) => right[1] - left[1])[0] ?? [];
|
||||
if (!mainPath) {
|
||||
throw new Error(`${profile.profilePath}: could not identify the main executable mapping`);
|
||||
}
|
||||
const baseMapping = profile.mappings.find(
|
||||
(mapping) => mapping.mappedPath === mainPath && mapping.offset === 0n,
|
||||
);
|
||||
if (!baseMapping) {
|
||||
throw new Error(`${profile.profilePath}: main executable has no offset-zero mapping`);
|
||||
}
|
||||
return { mappedPath: mainPath, loadBias: baseMapping.start };
|
||||
}
|
||||
|
||||
function normalizeMainAddress(address, mainMapping) {
|
||||
const numericAddress = BigInt(address);
|
||||
const relativeAddress = numericAddress - mainMapping.loadBias;
|
||||
const callerAddress = relativeAddress > 0n ? relativeAddress - 1n : relativeAddress;
|
||||
return `0x${callerAddress.toString(16)}`;
|
||||
}
|
||||
|
||||
function symbolize(binary, profiles) {
|
||||
const normalizedAddresses = new Set();
|
||||
for (const profile of profiles) {
|
||||
for (const sample of profile.samples) {
|
||||
for (const address of sample.frames) {
|
||||
const mapping = mappingForAddress(profile.mappings, address);
|
||||
if (mapping?.mappedPath === profile.mainMapping.mappedPath) {
|
||||
normalizedAddresses.add(normalizeMainAddress(address, profile.mainMapping));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const result = new Map();
|
||||
const unique = [...normalizedAddresses];
|
||||
for (let start = 0; start < unique.length; start += 500) {
|
||||
const batch = unique.slice(start, start + 500);
|
||||
const child = spawnSync('addr2line', ['-Cfpe', binary, ...batch], {
|
||||
encoding: 'utf8',
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
});
|
||||
if (child.status !== 0) {
|
||||
throw new Error(`addr2line failed: ${child.stderr}`);
|
||||
}
|
||||
const lines = child.stdout.trimEnd().split('\n');
|
||||
if (lines.length !== batch.length) {
|
||||
throw new Error(`addr2line returned ${lines.length} lines for ${batch.length} addresses`);
|
||||
}
|
||||
for (let index = 0; index < batch.length; index += 1) {
|
||||
result.set(batch[index], lines[index]);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function describeFrame(profile, symbols, address) {
|
||||
const mapping = mappingForAddress(profile.mappings, address);
|
||||
if (!mapping) {
|
||||
return `${address} at (unmapped)`;
|
||||
}
|
||||
if (mapping.mappedPath === profile.mainMapping.mappedPath) {
|
||||
const normalized = normalizeMainAddress(address, profile.mainMapping);
|
||||
return symbols.get(normalized) ?? `${normalized} at ??`;
|
||||
}
|
||||
const location = BigInt(address) - mapping.start + mapping.offset;
|
||||
const label = mapping.mappedPath
|
||||
? path.basename(mapping.mappedPath)
|
||||
: `(anonymous ${mapping.permissions})`;
|
||||
return `${label}+0x${location.toString(16)}`;
|
||||
}
|
||||
|
||||
function isAllocatorPlumbing(symbol) {
|
||||
return symbol.includes('prof_backtrace')
|
||||
|| symbol.includes('je_prof_')
|
||||
|| symbol.includes('prof_alloc_')
|
||||
|| symbol.includes('prof_sample_')
|
||||
|| symbol.includes('prof_tctx_')
|
||||
|| symbol.includes('jemalloc')
|
||||
|| symbol.includes('_rjem_')
|
||||
|| symbol.includes('tikv_jemalloc')
|
||||
|| symbol.startsWith('operator new(')
|
||||
|| symbol.includes('OPENSSL_malloc')
|
||||
|| symbol.includes('OPENSSL_realloc')
|
||||
|| symbol.includes('mallocx')
|
||||
|| symbol.includes('imalloc')
|
||||
|| symbol.includes('ialloc')
|
||||
|| symbol.includes('arena_malloc')
|
||||
|| symbol.includes('arena_ralloc')
|
||||
|| symbol.includes('arena_dalloc')
|
||||
|| symbol.includes('tcache_alloc')
|
||||
|| symbol.includes('__rust_alloc')
|
||||
|| symbol.includes('__rdl_alloc')
|
||||
|| symbol.includes('alloc::alloc::')
|
||||
|| symbol.includes('alloc::raw_vec::RawVecInner')
|
||||
|| symbol.includes('alloc::raw_vec::RawVec<')
|
||||
|| symbol.includes('hashbrown::raw::RawTableInner::fallible_with_capacity')
|
||||
|| symbol.includes('hashbrown::raw::RawTable<T,A>::reserve_rehash')
|
||||
|| symbol.includes('smallvec::SmallVec<A>::try_grow')
|
||||
|| symbol.includes('thin_vec::ThinVec<T>::reserve')
|
||||
|| symbol.includes('alloc::raw_vec::RawVecInner::try_allocate_in')
|
||||
|| symbol.includes('Allocator::allocate')
|
||||
|| symbol.includes('Allocator::grow');
|
||||
}
|
||||
|
||||
function aggregate(samples, keyForSample) {
|
||||
const totals = new Map();
|
||||
for (const sample of samples) {
|
||||
const key = keyForSample(sample);
|
||||
const current = totals.get(key) ?? {
|
||||
adjustedBytes: 0,
|
||||
adjustedObjects: 0,
|
||||
sampledBytes: 0,
|
||||
sampledObjects: 0,
|
||||
stacks: 0,
|
||||
};
|
||||
current.adjustedBytes += sample.adjustedBytes;
|
||||
current.adjustedObjects += sample.adjustedObjects;
|
||||
current.sampledBytes += sample.rawBytes;
|
||||
current.sampledObjects += sample.rawObjects;
|
||||
current.stacks += 1;
|
||||
totals.set(key, current);
|
||||
}
|
||||
return [...totals.entries()]
|
||||
.map(([key, totalsForKey]) => ({ key, ...totalsForKey }))
|
||||
.sort((left, right) => right.adjustedBytes - left.adjustedBytes);
|
||||
}
|
||||
|
||||
function categoryForSample(sample) {
|
||||
const stack = sample.consumerStack.join('\n');
|
||||
const matches = (pattern) => pattern.test(stack);
|
||||
|
||||
if (matches(/SubresourceResponseBody|SubresourceNetworkRecord|ScriptNetworkOutput|TargetNetworkOutput|TargetSubresource|CapturedBodyWriter|renderer_network_observation|response::ResponseBody/)) {
|
||||
return 'network observations / retained response bodies';
|
||||
}
|
||||
if (matches(/style::|selectors::|cssparser::|Stylesheet|stylesheet|CSSRule|css_rule|css_stylesheet|style_engine/)) {
|
||||
return 'CSS / Stylo / CSSOM';
|
||||
}
|
||||
if (matches(/ModuleScript|module_script|module_runtime|script_planning|ScriptPreload|ImportMap|import_map|decode_classic_script|external_script_source/)) {
|
||||
return 'scripts / modules / import maps';
|
||||
}
|
||||
if (matches(/moli_dom::|NativeDom|native_bridge|document_runtime|html5ever|moli_parser/)) {
|
||||
return 'DOM / bindings / parser';
|
||||
}
|
||||
if (matches(/v8::|Builtins_|icu_|_uhash_|cppgc/)) {
|
||||
return 'V8 / ICU native';
|
||||
}
|
||||
if (matches(/aws_lc_|OPENSSL_|SSL_|bssl::|CRYPTO_/)) {
|
||||
return 'TLS / crypto';
|
||||
}
|
||||
if (matches(/Curl_|curl::|nghttp2_|http2\.c/)) {
|
||||
return 'curl / HTTP2';
|
||||
}
|
||||
if (matches(/tokio::|mio::|futures::|moli_local_executor|runtime::owner/)) {
|
||||
return 'async runtime / scheduling';
|
||||
}
|
||||
if (matches(/moli_protocol::|moli_protocol_server::/)) {
|
||||
return 'protocol / CDP';
|
||||
}
|
||||
if (matches(/moli_/)) {
|
||||
return 'other Moli';
|
||||
}
|
||||
return 'other';
|
||||
}
|
||||
|
||||
function analyze(profile, symbols) {
|
||||
const samples = profile.samples.map((sample) => {
|
||||
const stack = sample.frames.map((address) => describeFrame(profile, symbols, address));
|
||||
const consumerStack = stack.filter((symbol) => !isAllocatorPlumbing(symbol));
|
||||
return {
|
||||
...sample,
|
||||
stack,
|
||||
consumer: consumerStack[0] ?? stack[0] ?? '(unknown)',
|
||||
consumerStack,
|
||||
};
|
||||
});
|
||||
const adjustedBytes = samples.reduce((total, sample) => total + sample.adjustedBytes, 0);
|
||||
const adjustedObjects = samples.reduce((total, sample) => total + sample.adjustedObjects, 0);
|
||||
const sampledBytes = samples.reduce((total, sample) => total + sample.rawBytes, 0);
|
||||
const sampledObjects = samples.reduce((total, sample) => total + sample.rawObjects, 0);
|
||||
const topCategories = aggregate(samples, categoryForSample).map((category) => ({
|
||||
...category,
|
||||
adjustedFraction: adjustedBytes === 0 ? 0 : category.adjustedBytes / adjustedBytes,
|
||||
}));
|
||||
|
||||
return {
|
||||
profile: profile.profilePath,
|
||||
interval: profile.interval,
|
||||
mainMapping: {
|
||||
path: profile.mainMapping.mappedPath,
|
||||
loadBias: `0x${profile.mainMapping.loadBias.toString(16)}`,
|
||||
},
|
||||
header: profile.header,
|
||||
sampling: {
|
||||
liveStackRecords: samples.length,
|
||||
sampledBytes,
|
||||
sampledObjects,
|
||||
adjustedBytes,
|
||||
adjustedObjects,
|
||||
headerBytesMatch: profile.header?.sampledBytes === sampledBytes,
|
||||
headerObjectsMatch: profile.header?.sampledObjects === sampledObjects,
|
||||
},
|
||||
topCategories,
|
||||
topCategoryConsumers: aggregate(
|
||||
samples,
|
||||
(sample) => `${categoryForSample(sample)}\t${sample.consumer}`,
|
||||
).slice(0, 100),
|
||||
topConsumers: aggregate(samples, (sample) => sample.consumer).slice(0, 50),
|
||||
topConsumerStacks: aggregate(
|
||||
samples,
|
||||
(sample) => sample.consumerStack.slice(0, 10).join(' <- ') || sample.consumer,
|
||||
).slice(0, 50),
|
||||
largestSampledStacks: samples
|
||||
.sort((left, right) => right.adjustedBytes - left.adjustedBytes)
|
||||
.slice(0, 100)
|
||||
.map((sample) => ({
|
||||
sampledObjects: sample.rawObjects,
|
||||
sampledBytes: sample.rawBytes,
|
||||
adjustedObjects: sample.adjustedObjects,
|
||||
adjustedBytes: sample.adjustedBytes,
|
||||
scale: sample.scale,
|
||||
consumer: sample.consumer,
|
||||
stack: sample.consumerStack.slice(0, 16),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const profiles = args.profiles.map(parseProfile);
|
||||
for (const profile of profiles) {
|
||||
profile.mainMapping = findMainMapping(profile);
|
||||
}
|
||||
const symbols = symbolize(args.binary, profiles);
|
||||
console.log(JSON.stringify(profiles.map((profile) => analyze(profile, symbols)), null, 2));
|
||||
@@ -0,0 +1,213 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = { binary: null, profiles: [] };
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const value = argv[index];
|
||||
if (value === '--binary') {
|
||||
args.binary = path.resolve(argv[++index]);
|
||||
} else {
|
||||
args.profiles.push(path.resolve(value));
|
||||
}
|
||||
}
|
||||
if (!args.binary || args.profiles.length === 0) {
|
||||
throw new Error('usage: analyze-lightpanda-pa-profile.mjs --binary BIN PROFILE...');
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function parseFields(line) {
|
||||
return Object.fromEntries(
|
||||
[...line.matchAll(/(\w+)=([^\s]+)/g)].map((match) => {
|
||||
const raw = match[2];
|
||||
const value = raw.startsWith('0x') ? raw : Number.parseInt(raw, 10);
|
||||
return [match[1], value];
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function parseProfile(profilePath) {
|
||||
const lines = fs.readFileSync(profilePath, 'utf8').trim().split('\n');
|
||||
const header = parseFields(lines.find((line) => line.startsWith('H ')) ?? '');
|
||||
const histogram = lines
|
||||
.filter((line) => line.startsWith('B '))
|
||||
.map(parseFields);
|
||||
const samples = lines
|
||||
.filter((line) => line.startsWith('S '))
|
||||
.map((line) => {
|
||||
const fields = parseFields(line);
|
||||
return {
|
||||
size: fields.size,
|
||||
weight: fields.weight,
|
||||
key: fields.key,
|
||||
depth: fields.depth,
|
||||
frames: [...line.matchAll(/\s(0x[0-9a-f]+)(?=\s|$)/g)]
|
||||
.map((match) => match[1]),
|
||||
};
|
||||
});
|
||||
return { profilePath, header, histogram, samples };
|
||||
}
|
||||
|
||||
function symbolize(binary, addresses) {
|
||||
const result = new Map();
|
||||
const unique = [...new Set(addresses)];
|
||||
for (let start = 0; start < unique.length; start += 500) {
|
||||
const batch = unique.slice(start, start + 500);
|
||||
const child = spawnSync('addr2line', ['-Cfpe', binary, ...batch], {
|
||||
encoding: 'utf8',
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
});
|
||||
if (child.status !== 0) {
|
||||
throw new Error(`addr2line failed: ${child.stderr}`);
|
||||
}
|
||||
const lines = child.stdout.trimEnd().split('\n');
|
||||
if (lines.length !== batch.length) {
|
||||
throw new Error(`addr2line returned ${lines.length} lines for ${batch.length} addresses`);
|
||||
}
|
||||
for (let index = 0; index < batch.length; index += 1) {
|
||||
result.set(batch[index], lines[index]);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function isProfilerFrame(symbol) {
|
||||
return symbol.includes('pa_profile.cpp')
|
||||
|| symbol.includes('RecordAllocation')
|
||||
|| symbol.includes('DispatchAlloc');
|
||||
}
|
||||
|
||||
function isAllocatorPlumbing(symbol) {
|
||||
return isProfilerFrame(symbol)
|
||||
|| /^(malloc|calloc|realloc|memalign|aligned_alloc|operator new)/.test(symbol)
|
||||
|| symbol.includes('allocator_shim::')
|
||||
|| symbol.includes('Arena.rawAlloc')
|
||||
|| symbol.includes('ArenaAllocator.alloc')
|
||||
|| symbol.includes('mem.Allocator.rawAlloc')
|
||||
|| symbol.includes('mem.Allocator.allocBytesWithAlignment')
|
||||
|| symbol.includes('mem.Allocator.allocWithOptions');
|
||||
}
|
||||
|
||||
function aggregate(samples, keyForSample) {
|
||||
const totals = new Map();
|
||||
for (const sample of samples) {
|
||||
const key = keyForSample(sample);
|
||||
const current = totals.get(key) ?? { estimatedBytes: 0, sampledBytes: 0, samples: 0 };
|
||||
current.estimatedBytes += sample.size * sample.weight;
|
||||
current.sampledBytes += sample.size;
|
||||
current.samples += 1;
|
||||
totals.set(key, current);
|
||||
}
|
||||
return [...totals.entries()]
|
||||
.map(([key, totalsForKey]) => ({ key, ...totalsForKey }))
|
||||
.sort((left, right) => right.estimatedBytes - left.estimatedBytes);
|
||||
}
|
||||
|
||||
function categoryForSample(sample) {
|
||||
const stack = sample.consumerStack.join('\n');
|
||||
const matches = (pattern) => pattern.test(stack);
|
||||
|
||||
if (matches(/Notification\.dispatch|HttpClient\.Transfer\.deliver|HttpClient\.drainInbox/)) {
|
||||
return 'network response buffers / notifications';
|
||||
}
|
||||
if (matches(/Timers\.|setTimeout|setInterval/)) return 'timers';
|
||||
if (matches(/FinalizerCallback|mapZigInstanceToJs|browser\.js\.Local/)) {
|
||||
return 'JS wrapper finalizers';
|
||||
}
|
||||
if (matches(/StyleManager|loadExternalStylesheet|stylesheet|CSSRule|cssparser|style::|selectors::/)) {
|
||||
return 'CSS / Stylo / CSSOM';
|
||||
}
|
||||
if (matches(/ScriptManager|ImportMap|resolveModule|browser\.js\.Module/)) {
|
||||
return 'scripts / modules / import maps';
|
||||
}
|
||||
if (matches(/node_factory|browser\.parser|slab\.Slab|html5ever/)) {
|
||||
return 'DOM / bindings / parser';
|
||||
}
|
||||
if (matches(/ArenaPool|ArenaAllocator/)) return 'arena pools';
|
||||
if (matches(/v8::|Builtins_|icu_|_uhash_|cppgc/)) return 'V8 / ICU native';
|
||||
if (matches(/OPENSSL_|SSL_|CRYPTO_|bssl::/)) return 'TLS / crypto';
|
||||
if (matches(/Curl_|nghttp2_|libcurl/)) return 'curl / HTTP2';
|
||||
if (matches(/browser\.|cdp\./)) return 'other Lightpanda browser / CDP';
|
||||
return 'other';
|
||||
}
|
||||
|
||||
function analyze(profile, symbols) {
|
||||
const samples = profile.samples.map((sample) => {
|
||||
const stack = sample.frames.map((address) => symbols.get(address) ?? `${address} at ??`);
|
||||
const externalStack = stack.filter((symbol) => !isProfilerFrame(symbol));
|
||||
const consumerStack = stack.filter((symbol) => !isAllocatorPlumbing(symbol));
|
||||
return {
|
||||
...sample,
|
||||
stack,
|
||||
leaf: externalStack[0] ?? '(profiler only)',
|
||||
consumer: consumerStack[0] ?? externalStack[0] ?? '(unknown)',
|
||||
consumerStack,
|
||||
};
|
||||
});
|
||||
const estimatedRequestedBytes = samples.reduce(
|
||||
(total, sample) => total + sample.size * sample.weight,
|
||||
0,
|
||||
);
|
||||
const sampledRequestedBytes = samples.reduce((total, sample) => total + sample.size, 0);
|
||||
const exactLargeSamples = samples.filter((sample) => sample.weight === 1);
|
||||
const exactLargeRequestedBytes = exactLargeSamples.reduce(
|
||||
(total, sample) => total + sample.size,
|
||||
0,
|
||||
);
|
||||
const histogramLargeRequestedBytes = profile.histogram
|
||||
.filter((bin) => bin.bin >= 17)
|
||||
.reduce((total, bin) => total + bin.requested_bytes, 0);
|
||||
const topCategories = aggregate(samples, categoryForSample).map((category) => ({
|
||||
...category,
|
||||
estimatedFraction: estimatedRequestedBytes === 0
|
||||
? 0
|
||||
: category.estimatedBytes / estimatedRequestedBytes,
|
||||
}));
|
||||
|
||||
return {
|
||||
profile: profile.profilePath,
|
||||
header: profile.header,
|
||||
histogram: profile.histogram,
|
||||
sampling: {
|
||||
liveSamples: samples.length,
|
||||
sampledRequestedBytes,
|
||||
estimatedRequestedBytes,
|
||||
estimateToExactRatio: estimatedRequestedBytes / profile.header.live_requested_bytes,
|
||||
exactLargeSamples: exactLargeSamples.length,
|
||||
exactLargeRequestedBytes,
|
||||
histogramLargeRequestedBytes,
|
||||
exactLargeMatchesHistogram: exactLargeRequestedBytes === histogramLargeRequestedBytes,
|
||||
},
|
||||
topCategories,
|
||||
topCategoryConsumers: aggregate(
|
||||
samples,
|
||||
(sample) => `${categoryForSample(sample)}\t${sample.consumer}`,
|
||||
).slice(0, 100),
|
||||
topConsumers: aggregate(samples, (sample) => sample.consumer).slice(0, 40),
|
||||
topConsumerStacks: aggregate(
|
||||
samples,
|
||||
(sample) => sample.consumerStack.slice(0, 8).join(' <- ') || sample.consumer,
|
||||
).slice(0, 40),
|
||||
largestLiveAllocations: exactLargeSamples
|
||||
.sort((left, right) => right.size - left.size)
|
||||
.slice(0, 100)
|
||||
.map((sample) => ({
|
||||
size: sample.size,
|
||||
key: sample.key,
|
||||
consumer: sample.consumer,
|
||||
stack: sample.consumerStack.slice(0, 12),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const profiles = args.profiles.map(parseProfile);
|
||||
const symbols = symbolize(
|
||||
args.binary,
|
||||
profiles.flatMap((profile) => profile.samples.flatMap((sample) => sample.frames)),
|
||||
);
|
||||
console.log(JSON.stringify(profiles.map((profile) => analyze(profile, symbols)), null, 2));
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
const inputs = process.argv.slice(2);
|
||||
if (inputs.length === 0) {
|
||||
throw new Error('usage: analyze-v8-heapsnapshot.mjs SNAPSHOT...');
|
||||
}
|
||||
|
||||
function topEntries(map, limit) {
|
||||
return [...map.entries()]
|
||||
.map(([name, value]) => ({ name, ...value }))
|
||||
.sort((left, right) => right.selfSize - left.selfSize || right.count - left.count)
|
||||
.slice(0, limit);
|
||||
}
|
||||
|
||||
function add(map, key, selfSize) {
|
||||
const value = map.get(key) ?? { count: 0, selfSize: 0 };
|
||||
value.count += 1;
|
||||
value.selfSize += selfSize;
|
||||
map.set(key, value);
|
||||
}
|
||||
|
||||
function analyze(input) {
|
||||
const snapshot = JSON.parse(fs.readFileSync(input, 'utf8'));
|
||||
const fields = snapshot.snapshot.meta.node_fields;
|
||||
const fieldCount = fields.length;
|
||||
const typeIndex = fields.indexOf('type');
|
||||
const nameIndex = fields.indexOf('name');
|
||||
const selfSizeIndex = fields.indexOf('self_size');
|
||||
if ([typeIndex, nameIndex, selfSizeIndex].includes(-1)) {
|
||||
throw new Error(`${input}: unsupported node_fields`);
|
||||
}
|
||||
if (snapshot.nodes.length % fieldCount !== 0) {
|
||||
throw new Error(`${input}: truncated nodes array`);
|
||||
}
|
||||
|
||||
const nodeTypes = snapshot.snapshot.meta.node_types[typeIndex];
|
||||
const byType = new Map();
|
||||
const byName = new Map();
|
||||
const byTypeAndName = new Map();
|
||||
let selfSize = 0;
|
||||
|
||||
for (let offset = 0; offset < snapshot.nodes.length; offset += fieldCount) {
|
||||
const type = nodeTypes[snapshot.nodes[offset + typeIndex]];
|
||||
const name = snapshot.strings[snapshot.nodes[offset + nameIndex]];
|
||||
const size = snapshot.nodes[offset + selfSizeIndex];
|
||||
selfSize += size;
|
||||
add(byType, type, size);
|
||||
add(byName, name, size);
|
||||
add(byTypeAndName, `${type}\t${name}`, size);
|
||||
}
|
||||
|
||||
return {
|
||||
path: path.resolve(input),
|
||||
bytes: fs.statSync(input).size,
|
||||
nodeCount: snapshot.nodes.length / fieldCount,
|
||||
selfSize,
|
||||
byType: topEntries(byType, byType.size),
|
||||
topNames: topEntries(byName, 60),
|
||||
topTypeNames: topEntries(byTypeAndName, 100).map((entry) => {
|
||||
const [type, name] = entry.name.split('\t', 2);
|
||||
return { type, name, count: entry.count, selfSize: entry.selfSize };
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
for (const input of inputs) {
|
||||
process.stdout.write(`${JSON.stringify(analyze(input))}\n`);
|
||||
}
|
||||
Reference in New Issue
Block a user