diff --git a/docs/reference/linux-memory-kill-attribution.md b/docs/reference/linux-memory-kill-attribution.md index 8b19c1d83a9..0eda06df355 100644 --- a/docs/reference/linux-memory-kill-attribution.md +++ b/docs/reference/linux-memory-kill-attribution.md @@ -48,7 +48,13 @@ up to 10 s before it (see `pre-gone-host-memory.ts`). | `systemMemoryStall{Some,Full}Avg{10,60}Pct` | `/proc/pressure/memory` | | `systemMemoryCgroupStall{Some,Full}Avg{10,60}Pct` | the cgroup's `memory.pressure` | -An absent row means "could not measure", never "unlimited" or "calm". cgroup v1 +An absent row means "could not measure", never "calm" — with one exception to +read carefully: `memory.max` and `memory.high` read the literal string `max` +when no ceiling is set, and that is reported as an absent `CgroupMaxMB` / +`CgroupHighMB`, not as a number. So the ceiling fields alone cannot separate "no +ceiling" from "unreadable"; `systemMemoryCgroupCurrentMB` is the tell. Present +means the cgroup was read and the missing ceiling really is unlimited; no +`Cgroup*` field at all means nothing was measurable. cgroup v1 is not read at all: its limit is not resolvable from `/proc/self/cgroup` without mount parsing, and a half-right ceiling is worse than none. diff --git a/src/main/crash-reporting/linux-cgroup-memory-limit.test.ts b/src/main/crash-reporting/linux-cgroup-memory-limit.test.ts index f77dbea369e..8b668c36aec 100644 --- a/src/main/crash-reporting/linux-cgroup-memory-limit.test.ts +++ b/src/main/crash-reporting/linux-cgroup-memory-limit.test.ts @@ -5,7 +5,9 @@ import { parseCgroupMemoryEvent, parseCgroupV2Path, readLinuxCgroupMemoryLimit, - setLinuxCgroupMemoryLimitReaderForTest + resolveCgroupV2MemoryDir, + setLinuxCgroupMemoryLimitReaderForTest, + setLinuxPseudoFileReaderForTest } from './linux-cgroup-memory-limit' import { setLinuxMemoryPressureStallReaderForTest } from './linux-memory-pressure-stall' import { getSystemMemoryDetails, setSystemMemoryInfoReaderForTest } from './system-memory-details' @@ -33,9 +35,87 @@ beforeEach(() => { afterEach(() => { setLinuxCgroupMemoryLimitReaderForTest(null) setLinuxMemoryPressureStallReaderForTest(null) + setLinuxPseudoFileReaderForTest(null) setSystemMemoryInfoReaderForTest(null) }) +/** Only the listed paths exist; anything else reads as an unreadable pseudo-file. */ +function fakeLinuxPseudoFiles(files: Record): void { + setLinuxPseudoFileReaderForTest((filePath) => files[filePath]) +} + +const SANDBOX_CGROUP_PATH = '0::/user.slice/user-1000.slice/app.slice/orca.scope\n' + +// Everything below the seam that the reader test double skips: which directory +// the sysfs reads actually land in, and whether an unresolvable one stays quiet. +describe('cgroup v2 memory directory resolution', () => { + it('reads the directory /proc/self/cgroup names when it exists', () => { + fakeLinuxPseudoFiles({ + '/proc/self/cgroup': SANDBOX_CGROUP_PATH, + '/sys/fs/cgroup/user.slice/user-1000.slice/app.slice/orca.scope/memory.current': '512\n' + }) + + expect(resolveCgroupV2MemoryDir()).toBe( + '/sys/fs/cgroup/user.slice/user-1000.slice/app.slice/orca.scope' + ) + }) + + it('falls back to the mount root, which is our own cgroup inside a namespace', () => { + // The sandbox case the header claims to cover: the reported path is a host + // path that does not exist in here, and the mount root IS our cgroup. + fakeLinuxPseudoFiles({ + '/proc/self/cgroup': SANDBOX_CGROUP_PATH, + '/sys/fs/cgroup/memory.current': '900000000\n', + '/sys/fs/cgroup/memory.max': '1073741824\n' + }) + + expect(resolveCgroupV2MemoryDir()).toBe('/sys/fs/cgroup') + expect(readLinuxCgroupMemoryLimit('linux')).toMatchObject({ + maxBytes: 1_073_741_824, + currentBytes: 900_000_000 + }) + }) + + it('claims nothing when neither candidate has memory.current', () => { + // A v1-only or hybrid host: the unified root exists but carries no memory + // controller, and the host root cgroup never has memory.current. + fakeLinuxPseudoFiles({ '/proc/self/cgroup': SANDBOX_CGROUP_PATH }) + + expect(resolveCgroupV2MemoryDir()).toBeUndefined() + expect(readLinuxCgroupMemoryLimit('linux')).toBeUndefined() + }) + + it('reads the ceiling, the throttle and the events off the resolved directory', () => { + fakeLinuxPseudoFiles({ + '/proc/self/cgroup': SANDBOX_CGROUP_PATH, + '/sys/fs/cgroup/user.slice/user-1000.slice/app.slice/orca.scope/memory.current': '4200000000', + '/sys/fs/cgroup/user.slice/user-1000.slice/app.slice/orca.scope/memory.max': '4294967296', + '/sys/fs/cgroup/user.slice/user-1000.slice/app.slice/orca.scope/memory.high': 'max\n', + '/sys/fs/cgroup/user.slice/user-1000.slice/app.slice/orca.scope/memory.events': + 'low 0\nhigh 7\nmax 2\noom 1\noom_kill 1\n' + }) + + expect(readLinuxCgroupMemoryLimit('linux')).toEqual({ + maxBytes: 4_294_967_296, + // `max` is no ceiling, and must not surface as one just because it was read. + highBytes: undefined, + currentBytes: 4_200_000_000, + oomKillCount: 1, + maxEventCount: 2, + highEventCount: 7 + }) + }) + + it('says nothing rather than a row of undefineds when the files are garbage', () => { + fakeLinuxPseudoFiles({ + '/proc/self/cgroup': SANDBOX_CGROUP_PATH, + '/sys/fs/cgroup/user.slice/user-1000.slice/app.slice/orca.scope/memory.current': 'max' + }) + + expect(readLinuxCgroupMemoryLimit('linux')).toBeUndefined() + }) +}) + describe('linux cgroup v2 memory limit', () => { it('takes the unified-hierarchy line, not a v1 controller line', () => { const procSelfCgroup = [ diff --git a/src/main/crash-reporting/linux-cgroup-memory-limit.ts b/src/main/crash-reporting/linux-cgroup-memory-limit.ts index 27ac14117e9..ae0466b0bdf 100644 --- a/src/main/crash-reporting/linux-cgroup-memory-limit.ts +++ b/src/main/crash-reporting/linux-cgroup-memory-limit.ts @@ -38,8 +38,10 @@ export type LinuxCgroupMemoryLimit = { type LinuxCgroupMemoryLimitReader = () => LinuxCgroupMemoryLimit | undefined +type LinuxPseudoFileReader = (filePath: string) => string | undefined + /** Absent is a normal answer here: hardened hosts, WSL and containers hide these. */ -export function readLinuxPseudoFile(filePath: string): string | undefined { +function readLinuxPseudoFileFromDisk(filePath: string): string | undefined { try { return readFileSync(filePath, 'utf8') } catch { @@ -47,6 +49,17 @@ export function readLinuxPseudoFile(filePath: string): string | undefined { } } +let pseudoFileReader: LinuxPseudoFileReader = readLinuxPseudoFileFromDisk + +/** The seam the sysfs resolution itself is tested through; PSI reads through it too. */ +export function setLinuxPseudoFileReaderForTest(next: LinuxPseudoFileReader | null): void { + pseudoFileReader = next ?? readLinuxPseudoFileFromDisk +} + +export function readLinuxPseudoFile(filePath: string): string | undefined { + return pseudoFileReader(filePath) +} + /** The unified-hierarchy line is the one with an empty controller list. */ export function parseCgroupV2Path(procSelfCgroup: string): string | undefined { for (const line of procSelfCgroup.split('\n')) { diff --git a/src/main/crash-reporting/linux-memory-pressure-stall.test.ts b/src/main/crash-reporting/linux-memory-pressure-stall.test.ts index 161ac48a805..45b999f15b5 100644 --- a/src/main/crash-reporting/linux-memory-pressure-stall.test.ts +++ b/src/main/crash-reporting/linux-memory-pressure-stall.test.ts @@ -5,7 +5,10 @@ import { readLinuxMemoryPressureStall, setLinuxMemoryPressureStallReaderForTest } from './linux-memory-pressure-stall' -import { setLinuxCgroupMemoryLimitReaderForTest } from './linux-cgroup-memory-limit' +import { + setLinuxCgroupMemoryLimitReaderForTest, + setLinuxPseudoFileReaderForTest +} from './linux-cgroup-memory-limit' import { getSystemMemoryDetails, setSystemMemoryInfoReaderForTest } from './system-memory-details' import { preGoneSystemMemoryDetails, @@ -36,9 +39,54 @@ beforeEach(() => { afterEach(() => { setLinuxCgroupMemoryLimitReaderForTest(null) setLinuxMemoryPressureStallReaderForTest(null) + setLinuxPseudoFileReaderForTest(null) setSystemMemoryInfoReaderForTest(null) }) +/** Only the listed paths exist; anything else reads as an unreadable pseudo-file. */ +function fakeLinuxPseudoFiles(files: Record): void { + setLinuxPseudoFileReaderForTest((filePath) => files[filePath]) +} + +const CALM_PRESSURE = 'some avg10=0.00 avg60=0.00 avg300=0.00 total=0\n' + +// Below the reader seam: which files the procfs reader actually opens, and +// whether an absent PSI stays absent instead of reading as a calm host. +describe('reading PSI off procfs and the cgroup', () => { + it("reads the host file and the resolved cgroup's own memory.pressure", () => { + fakeLinuxPseudoFiles({ + '/proc/self/cgroup': '0::/user.slice/orca.scope\n', + '/sys/fs/cgroup/user.slice/orca.scope/memory.current': '512\n', + '/proc/pressure/memory': PROC_PRESSURE_MEMORY, + '/sys/fs/cgroup/user.slice/orca.scope/memory.pressure': + 'some avg10=71.20 avg60=60.00 avg300=20.00 total=1\nfull avg10=58.90 avg60=41.30 avg300=9.00 total=2\n' + }) + + expect(readLinuxMemoryPressureStall('linux')).toEqual({ + host: { someAvg10: 61.4, someAvg60: 48.22, fullAvg10: 44.1, fullAvg60: 30.05 }, + cgroup: { someAvg10: 71.2, someAvg60: 60, fullAvg10: 58.9, fullAvg60: 41.3 } + }) + }) + + it('still reports the host file when the cgroup cannot be resolved', () => { + fakeLinuxPseudoFiles({ '/proc/pressure/memory': CALM_PRESSURE }) + + expect(readLinuxMemoryPressureStall('linux')).toEqual({ + host: { someAvg10: 0, someAvg60: 0 }, + cgroup: undefined + }) + }) + + it('stays silent on a kernel built without CONFIG_PSI', () => { + fakeLinuxPseudoFiles({ + '/proc/self/cgroup': '0::/user.slice/orca.scope\n', + '/sys/fs/cgroup/user.slice/orca.scope/memory.current': '512\n' + }) + + expect(readLinuxMemoryPressureStall('linux')).toBeUndefined() + }) +}) + describe('linux PSI memory stall', () => { it('takes avg10 and avg60 off both the some and full lines', () => { expect(parseMemoryPressureStall(PROC_PRESSURE_MEMORY)).toEqual({ diff --git a/src/main/crash-reporting/pre-gone-host-memory.test.ts b/src/main/crash-reporting/pre-gone-host-memory.test.ts index 0df13d4fee5..a67d212fee6 100644 --- a/src/main/crash-reporting/pre-gone-host-memory.test.ts +++ b/src/main/crash-reporting/pre-gone-host-memory.test.ts @@ -10,6 +10,8 @@ import { type SwapVolumeFreeSpace } from './swap-volume-free-space' import { samplePreGoneSystemMemory } from './pre-gone-host-memory' +import { setLinuxCgroupMemoryLimitReaderForTest } from './linux-cgroup-memory-limit' +import { setLinuxMemoryPressureStallReaderForTest } from './linux-memory-pressure-stall' import { buildProcessGoneCrashDetails, resetPreGoneCrashSamplingForTest, @@ -82,6 +84,10 @@ describe('pre-gone host memory', () => { resetPreGoneCrashSamplingForTest() setSystemMemoryInfoReaderForTest(null) setSwapVolumeFreeSpaceReaderForTest(null) + // Without these the linux readings below come from the CI host's own cgroup + // and PSI, so a capped or busy runner rewrites the label under test. + setLinuxCgroupMemoryLimitReaderForTest(() => undefined) + setLinuxMemoryPressureStallReaderForTest(() => undefined) appMetricsMock.mockClear() appMetricsMock.mockReturnValue(BROWSER_AND_RENDERER) })